diff --git a/.eslintrc.js b/.eslintrc.js index eb773fc..648aad3 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -2,23 +2,17 @@ module.exports = { env: { - "node": true + node: true, }, - extends: [ - "eslint:recommended", - "plugin:mozilla/recommended", - ], + extends: ["eslint:recommended", "plugin:mozilla/recommended"], - plugins: [ - "mozilla", - "json" - ], + plugins: ["mozilla", "json"], rules: { "babel/new-cap": "off", "comma-dangle": ["error", "always-multiline"], - "eqeqeq": "error", - "indent": ["warn", 2, {SwitchCase: 1}], + eqeqeq: "error", + indent: ["warn", 2, { SwitchCase: 1 }], "mozilla/no-aArgs": "warn", "mozilla/balanced-listeners": 0, "no-console": "warn", @@ -26,6 +20,6 @@ module.exports = { "no-shadow": ["error"], "no-unused-vars": "error", "prefer-const": "warn", - "semi": ["error", "always"], + semi: ["error", "always"], }, }; diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..dd9678b --- /dev/null +++ b/.prettierignore @@ -0,0 +1,3 @@ +package-lock.json +docs/css +package.json diff --git a/README.md b/README.md index 9314bcd..47cb20e 100644 --- a/README.md +++ b/README.md @@ -1,27 +1,34 @@ -| index | [example] | -|-------|---------------| +| index | [example] | +| ----- | --------- | # Shield Studies Addon Utils [![Build Status](https://travis-ci.org/tombell/travis-ci-status.svg?branch=master)](https://travis-ci.org/tombell/travis-ci-status) -- Instrument your Firefox Addon! -- Build Shield Study (Normandy) compatible addons without having to think very much. +* Instrument your Firefox Addon! +* Build Shield Study (Normandy) compatible addons without having to think very + much. ## Assumptions -- You are building a [LEGACY ADDONS](https://developer.mozilla.org/en-US/Add-ons/Legacy_add_ons). To deploy these after 57, you will need the magic special signing. -- Jetpack / addon-sdk is NOT SUPPORTED. -- WebExtensions are not strong enough +* You are building a + [LEGACY ADDONS](https://developer.mozilla.org/en-US/Add-ons/Legacy_add_ons). + To deploy these after 57, you will need the magic special signing. +* Jetpack / addon-sdk is NOT SUPPORTED. +* WebExtensions are not strong enough ## history of major versions -- v4.x: (proposed) additional functions for common cases -- v4: First `.jsm` release. Uses packet format for PACKET version 3. -- v3: Attempt to formalize on `shield-study` PACKET version 3. Jetpack based. Prototype used for `raymak/page-reload`. All work abandoned, and no formal npm release in this series. Work done at `v3-shield-packet-format` branch. LAST JETPACK (addon-sdk) RELEASE. -s v2: Code refactor to es6 `class` with event models. Added cli tooling. Packet format is still arbitrary and per-study. Jetpack based. Last used in studies in Q2 2017. -- v1: Initial work and thinking. Telemetry packets are rather arbitrary. Jetpack based. - - +* v4.x: (proposed) additional functions for common cases +* v4: First `.jsm` release. Uses packet format for PACKET version 3. +* v3: Attempt to formalize on `shield-study` PACKET version 3. Jetpack based. + Prototype used for `raymak/page-reload`. All work abandoned, and no formal npm + release in this series. Work done at `v3-shield-packet-format` branch. LAST + JETPACK (addon-sdk) RELEASE. +* v2: Code refactor to es6 `class` with event models. Added cli tooling. Packet + format is still arbitrary and per-study. Jetpack based. Last used in studies + in Q2 2017. +* v1: Initial work and thinking. Telemetry packets are rather arbitrary. Jetpack + based. ## install @@ -31,14 +38,15 @@ npm install --save-dev shield-studies-addon-utils Copy the file to somewhere useful in your addon. - ## Tutorial and Full Usage [fully worked tutorial - How To Shield Study](./howToShieldStudy.md) ## Example embedded web extension study -See [examples](https://github.com/mozilla/shield-studies-addon-utils/tree/master/examples) directory. +See +[examples](https://github.com/mozilla/shield-studies-addon-utils/tree/master/examples) +directory. ## Summary @@ -46,30 +54,27 @@ See [examples](https://github.com/mozilla/shield-studies-addon-utils/tree/master Your Study is: -- side-by-side variations (1 or more) +* side-by-side variations (1 or more) ### Benefits Using this, you get this analysis FOR FREE (and it's fast!) -- Branch x channel x VARIATION x experiment-id x PHASE (install, reject, alive etc) using UNIFIED TELEMETRY - +* Branch x channel x VARIATION x experiment-id x PHASE (install, reject, alive + etc) using UNIFIED TELEMETRY ## Development -- open an issue -- hack and file a PR - +* open an issue +* hack and file a PR ## Gotchas, Opinions, Side Effects, and Misfeatures -1. No handling of 'timers'. No saved state at all (including the variation name), unless you handle it yourself. - -2. No 'running' pings in v4 (yet). +1. No handling of 'timers'. No saved state at all (including the variation + name), unless you handle it yourself. -3. User disable also uninstalls (and cleans up) +2. No 'running' pings in v4 (yet). +3. User disable also uninstalls (and cleans up) [example]: examples/README.md - - diff --git a/examples/README.md b/examples/README.md index 5bd6cfa..4777714 100644 --- a/examples/README.md +++ b/examples/README.md @@ -2,6 +2,5 @@ Demonstrated in stages. -- stage 1: embedded web extension -- stage 2: instrumented shield study - +* stage 1: embedded web extension +* stage 2: instrumented shield study diff --git a/examples/stage-2-shield-study/.eslintrc.js b/examples/stage-2-shield-study/.eslintrc.js index 76ebb57..accf9c4 100644 --- a/examples/stage-2-shield-study/.eslintrc.js +++ b/examples/stage-2-shield-study/.eslintrc.js @@ -1,29 +1,22 @@ "use strict"; module.exports = { - env: { - }, - extends: [ - "eslint:recommended", - "plugin:mozilla/recommended", - ], + env: {}, + extends: ["eslint:recommended", "plugin:mozilla/recommended"], - plugins: [ - "mozilla", - "json" - ], + plugins: ["mozilla", "json"], rules: { "babel/new-cap": "off", "comma-dangle": ["error", "always-multiline"], - "eqeqeq": "error", - "indent": ["warn", 2, {SwitchCase: 1}], + eqeqeq: "error", + indent: ["warn", 2, { SwitchCase: 1 }], "mozilla/no-aArgs": "warn", "mozilla/balanced-listeners": 0, "no-console": "warn", "no-shadow": ["error"], "no-unused-vars": "error", "prefer-const": "warn", - "semi": ["error", "always"], + semi: ["error", "always"], }, }; diff --git a/examples/stage-2-shield-study/README.md b/examples/stage-2-shield-study/README.md index c105c7d..0c2acbf 100644 --- a/examples/stage-2-shield-study/README.md +++ b/examples/stage-2-shield-study/README.md @@ -2,41 +2,45 @@ ## install -`npm install` -`npm run build` - +```sh +npm install +npm run build +``` at second shell/prompt, watch files for changes to rebuild: `npm run watch` - ## in Firefox: -1. `about:debugging > [load temporary addon] > choose `dist/addon.xpi` +1. `about:debugging > [load temporary addon] > choose 'dist/addon.xpi'` 2. `tools > Web Developer > Browser Toolbox` ## Effects: -1. See a new button (with a 'puzzle piece') symbol. -2. to end early: Click on button multiple times until the 'too-popular' endpoint is reached. +1. See a new button (with a 'puzzle piece') symbol. +2. To end early: Click on button multiple times until the 'too-popular' endpoint + is reached. ## Further extensions / modifications -1. (TBD) +1. (TBD) ## Description of architecture -Embedded Web Extension (`/webextension/`) lives inside a restartless (`bootstrap.js`) extension. +Embedded Web Extension (`/webextension/`) lives inside a restartless +(`bootstrap.js`) extension. During `bootstrap.js:startup(data, reason)`: - a. `shieldUtils` imports and sets configuration from `Config.jsm` - b. `bootstrap.js:chooseVariation` explicitly and deterministically chooses a variation from `studyConfig.weightedVariations` - c. the WebExtension starts up - d. `boostrap.js` listens for requests from the `webExtension` that are study related: `["info", "telemetry", "endStudy"]` - e. `webExtension` (`background.js`) asks for `info` from `studyUtils` using `askShield` function. - f. Feature starts using the `variation` from that info. - g. Feature instruments user button to send `telemetry` and to `endStudy` if the button is clicked enough. - - +* `shieldUtils` imports and sets configuration from `Config.jsm`. +* `bootstrap.js:chooseVariation` explicitly and deterministically chooses a + variation from `studyConfig.weightedVariations`. +* the WebExtension starts up. +* `boostrap.js` listens for requests from the `webExtension` that are study + related: `["info", "telemetry", "endStudy"]`. +* `webExtension` (`background.js`) asks for `info` from `studyUtils` using + `askShield` function. +* Feature starts using the `variation` from that info. +* Feature instruments user button to send `telemetry` and to `endStudy` if the + button is clicked enough. diff --git a/examples/stage-2-shield-study/addon/Config.jsm b/examples/stage-2-shield-study/addon/Config.jsm index 072799f..f42214c 100644 --- a/examples/stage-2-shield-study/addon/Config.jsm +++ b/examples/stage-2-shield-study/addon/Config.jsm @@ -13,72 +13,79 @@ var EXPORTED_SYMBOLS = ["config"]; // var slug = "shield-example-addon"; // should match chrome.manifest; var config = { - "study": { - "studyName": "mostImportantExperiment", // no spaces, for all the reasons - "variation": { - "name": "kittens", + study: { + studyName: "mostImportantExperiment", // no spaces, for all the reasons + variation: { + name: "kittens", }, // optional, use to override/decide - "weightedVariations": [ - {"name": "control", - "weight": 1}, - {"name": "kittens", - "weight": 1.5}, - {"name": "puppers", - "weight": 2}, // we want more puppers in our sample + weightedVariations: [ + { + name: "control", + weight: 1, + }, + { + name: "kittens", + weight: 1.5, + }, + { + name: "puppers", + weight: 2, + }, // we want more puppers in our sample ], /** **endings** - * - keys indicate the 'endStudy' even that opens these. - * - urls should be static (data) or external, because they have to - * survive uninstall - * - If there is no key for an endStudy reason, no url will open. - * - usually surveys, orientations, explanations - */ - "endings": { + * - keys indicate the 'endStudy' even that opens these. + * - urls should be static (data) or external, because they have to + * survive uninstall + * - If there is no key for an endStudy reason, no url will open. + * - usually surveys, orientations, explanations + */ + endings: { /** standard endings */ "user-disable": { - "baseUrl": "data:,You uninstalled", + baseUrl: "data:,You uninstalled", }, - "ineligible": { - "baseUrl": "http://www.example.com/?reason=ineligible", + ineligible: { + baseUrl: "http://www.example.com/?reason=ineligible", }, - "expired": { - "baseUrl": "http://www.example.com/?reason=expired", + expired: { + baseUrl: "http://www.example.com/?reason=expired", }, /** User defined endings */ "too-popular": { // data uri made using `datauri-cli` - "baseUrl": "data:text/html;base64,PGh0bWw+CiAgPGJvZHk+CiAgICA8cD5Zb3UgYXJlIHVzaW5nIHRoaXMgZmVhdHVyZSA8c3Ryb25nPlNPIE1VQ0g8L3N0cm9uZz4gdGhhdCB3ZSBrbm93IHlvdSBsb3ZlIGl0IQogICAgPC9wPgogICAgPHA+VGhlIEV4cGVyaW1lbnQgaXMgb3ZlciBhbmQgd2UgYXJlIFVOSU5TVEFMTElORwogICAgPC9wPgogIDwvYm9keT4KPC9odG1sPgo=", - "study_state": "ended-positive", // neutral is default + baseUrl: + "data:text/html;base64,PGh0bWw+CiAgPGJvZHk+CiAgICA8cD5Zb3UgYXJlIHVzaW5nIHRoaXMgZmVhdHVyZSA8c3Ryb25nPlNPIE1VQ0g8L3N0cm9uZz4gdGhhdCB3ZSBrbm93IHlvdSBsb3ZlIGl0IQogICAgPC9wPgogICAgPHA+VGhlIEV4cGVyaW1lbnQgaXMgb3ZlciBhbmQgd2UgYXJlIFVOSU5TVEFMTElORwogICAgPC9wPgogIDwvYm9keT4KPC9odG1sPgo=", + study_state: "ended-positive", // neutral is default }, "a-non-url-opening-ending": { - "study_state": "ended-neutral", - "baseUrl": null, + study_state: "ended-neutral", + baseUrl: null, }, }, - "telemetry": { - "send": true, // assumed false. Actually send pings? - "removeTestingFlag": false, // Marks pings as testing, set true for actual release + telemetry: { + send: true, // assumed false. Actually send pings? + removeTestingFlag: false, // Marks pings as testing, set true for actual release // TODO "onInvalid": "throw" // invalid packet for schema? throw||log }, - "studyUtilsPath": `./StudyUtils.jsm`, + studyUtilsPath: `./StudyUtils.jsm`, }, - "isEligible": async function() { + isEligible: async function() { // get whatever prefs, addons, telemetry, anything! // Cu.import can see 'firefox things', but not package things. return true; }, // addon-specific modules to load/unload during `startup`, `shutdown` - "modules": [ + modules: [ // can use ${slug} here for example ], // sets the logging for BOTH the bootstrap file AND shield-study-utils - "log": { + log: { // Fatal: 70, Error: 60, Warn: 50, Info: 40, Config: 30, Debug: 20, Trace: 10, All: -1, - "bootstrap": { - "level": "Debug", + bootstrap: { + level: "Debug", }, - "studyUtils": { - "level": "Trace", + studyUtils: { + level: "Trace", }, }, }; diff --git a/examples/stage-2-shield-study/addon/bootstrap.js b/examples/stage-2-shield-study/addon/bootstrap.js index 2367b55..c120f90 100644 --- a/examples/stage-2-shield-study/addon/bootstrap.js +++ b/examples/stage-2-shield-study/addon/bootstrap.js @@ -1,17 +1,18 @@ "use strict"; - /* global __SCRIPT_URI_SPEC__ */ /* eslint no-unused-vars: ["error", { "varsIgnorePattern": "(startup|shutdown|install|uninstall)" }]*/ -const {utils: Cu} = Components; +const { utils: Cu } = Components; const CONFIGPATH = `${__SCRIPT_URI_SPEC__}/../Config.jsm`; const { config } = Cu.import(CONFIGPATH, {}); const studyConfig = config.study; Cu.import("resource://gre/modules/Console.jsm"); -const log = createLog(studyConfig.studyName, config.log.bootstrap.level); // defined below. +const log = createLog(studyConfig.studyName, config.log.bootstrap.level); // defined below. -const STUDYUTILSPATH = `${__SCRIPT_URI_SPEC__}/../${studyConfig.studyUtilsPath}`; +const STUDYUTILSPATH = `${__SCRIPT_URI_SPEC__}/../${ + studyConfig.studyUtilsPath +}`; const { studyUtils } = Cu.import(STUDYUTILSPATH, {}); async function startup(addonData, reason) { @@ -20,7 +21,7 @@ async function startup(addonData, reason) { studyUtils.setup({ studyName: studyConfig.studyName, endings: studyConfig.endings, - addon: {id: addonData.id, version: addonData.version}, + addon: { id: addonData.id, version: addonData.version }, telemetry: studyConfig.telemetry, }); studyUtils.setLoggingLevel(config.log.studyUtils.level); @@ -29,32 +30,33 @@ async function startup(addonData, reason) { Jsm.import(config.modules); - if ((REASONS[reason]) === "ADDON_INSTALL") { - studyUtils.firstSeen(); // sends telemetry "enter" + if (REASONS[reason] === "ADDON_INSTALL") { + studyUtils.firstSeen(); // sends telemetry "enter" const eligible = await config.isEligible(); // addon-specific if (!eligible) { // uses config.endings.ineligible.url if any, // sends UT for "ineligible" // then uninstalls addon - await studyUtils.endStudy({reason: "ineligible"}); + await studyUtils.endStudy({ reason: "ineligible" }); return; } } - await studyUtils.startup({reason}); + await studyUtils.startup({ reason }); console.log(`info ${JSON.stringify(studyUtils.info())}`); // if you have code to handle expiration / long-timers, it could go here. const webExtension = addonData.webExtension; webExtension.startup().then(api => { - const {browser} = api; + const { browser } = api; // messages intended for shieldn: {shield:true,msg=[info|endStudy|telemetry],data=data} - browser.runtime.onMessage.addListener(studyUtils.respondToWebExtensionMessage); + browser.runtime.onMessage.addListener( + studyUtils.respondToWebExtensionMessage + ); // other message handlers from your addon, if any }); // studyUtils.endStudy("user-disable"); } - function shutdown(addonData, reason) { console.log("shutdown", REASONS[reason] || reason); // are we uninstalling? @@ -64,11 +66,11 @@ function shutdown(addonData, reason) { if (!studyUtils._isEnding) { // we are the first requestors, must be user action. console.log("user requested shutdown"); - studyUtils.endStudy({reason: "user-disable"}); + studyUtils.endStudy({ reason: "user-disable" }); return; } - // normal shutdown, or 2nd attempts + // normal shutdown, or 2nd attempts console.log("Jsms unloading"); Jsm.unload(config.modules); Jsm.unload([CONFIGPATH, STUDYUTILSPATH]); @@ -88,16 +90,18 @@ function install(addonData, reason) { // addon state change reasons const REASONS = { - APP_STARTUP: 1, // The application is starting up. - APP_SHUTDOWN: 2, // The application is shutting down. - ADDON_ENABLE: 3, // The add-on is being enabled. - ADDON_DISABLE: 4, // The add-on is being disabled. (Also sent during uninstallation) - ADDON_INSTALL: 5, // The add-on is being installed. - ADDON_UNINSTALL: 6, // The add-on is being uninstalled. - ADDON_UPGRADE: 7, // The add-on is being upgraded. - ADDON_DOWNGRADE: 8, // The add-on is being downgraded. + APP_STARTUP: 1, // The application is starting up. + APP_SHUTDOWN: 2, // The application is shutting down. + ADDON_ENABLE: 3, // The add-on is being enabled. + ADDON_DISABLE: 4, // The add-on is being disabled. (Also sent during uninstallation) + ADDON_INSTALL: 5, // The add-on is being installed. + ADDON_UNINSTALL: 6, // The add-on is being uninstalled. + ADDON_UPGRADE: 7, // The add-on is being upgraded. + ADDON_DOWNGRADE: 8, // The add-on is being downgraded. }; -for (const r in REASONS) { REASONS[REASONS[r]] = r; } +for (const r in REASONS) { + REASONS[REASONS[r]] = r; +} // logging function createLog(name, levelWord) { @@ -119,7 +123,10 @@ async function chooseVariation() { source = "weightedVariation"; // this is the standard arm choosing method const clientId = await studyUtils.getTelemetryId(); - const hashFraction = await sample.hashFraction(studyConfig.studyName + clientId, 12); + const hashFraction = await sample.hashFraction( + studyConfig.studyName + clientId, + 12 + ); toSet = sample.chooseWeighted(studyConfig.weightedVariations, hashFraction); } log.debug(`variation: ${toSet} source:${source}`); diff --git a/examples/stage-2-shield-study/addon/lib/AddonPrefs.jsm b/examples/stage-2-shield-study/addon/lib/AddonPrefs.jsm index 47da3c5..930a848 100644 --- a/examples/stage-2-shield-study/addon/lib/AddonPrefs.jsm +++ b/examples/stage-2-shield-study/addon/lib/AddonPrefs.jsm @@ -38,5 +38,6 @@ function set(key, type, value) { } var AddonPrefs = { - get, set, + get, + set, }; diff --git a/examples/stage-2-shield-study/addon/webextension/.eslintrc.json b/examples/stage-2-shield-study/addon/webextension/.eslintrc.json index 1d71c50..5c7cc9e 100644 --- a/examples/stage-2-shield-study/addon/webextension/.eslintrc.json +++ b/examples/stage-2-shield-study/addon/webextension/.eslintrc.json @@ -1,13 +1,11 @@ { - "env": { - "browser": true, - "es6": true, - "webextensions": true - }, - "extends": [ - "eslint:recommended" - ], - "rules": { - "no-console": "warn" - } + "env": { + "browser": true, + "es6": true, + "webextensions": true + }, + "extends": ["eslint:recommended"], + "rules": { + "no-console": "warn" + } } diff --git a/examples/stage-2-shield-study/addon/webextension/background.js b/examples/stage-2-shield-study/addon/webextension/background.js index c097984..7b75102 100644 --- a/examples/stage-2-shield-study/addon/webextension/background.js +++ b/examples/stage-2-shield-study/addon/webextension/background.js @@ -1,12 +1,13 @@ "use strict"; -function telemetry (data) { - function throwIfInvalid (obj) { +function telemetry(data) { + function throwIfInvalid(obj) { // simple, check is all keys and values are strings for (const k in obj) { - if (typeof k !== 'string') throw new Error(`key ${k} not a string`); - if (typeof obj[k] !== 'string') throw new Error(`value ${k} ${obj[k]} not a string`); + if (typeof k !== "string") throw new Error(`key ${k} not a string`); + if (typeof obj[k] !== "string") + throw new Error(`value ${k} ${obj[k]} not a string`); } - return true + return true; } throwIfInvalid(data); msgStudy("telemetry", data); @@ -15,9 +16,10 @@ function telemetry (data) { // template code for talking to `studyUtils` using `browser.runtime` async function msgStudy(msg, data) { const allowed = ["endStudy", "telemetry", "info"]; - if (!allowed.includes(msg)) throw new Error(`shieldUtils doesn't know ${msg}, only knows ${allowed}`); + if (!allowed.includes(msg)) + throw new Error(`shieldUtils doesn't know ${msg}, only knows ${allowed}`); try { - const ans = await browser.runtime.sendMessage({shield: true, msg, data}); + const ans = await browser.runtime.sendMessage({ shield: true, msg, data }); return ans; } catch (e) { console.log("OHNO", e); @@ -25,27 +27,27 @@ async function msgStudy(msg, data) { } class Feature { - constructor({variation}) { + constructor({ variation }) { console.log("init", variation.name); this.times = 0; // do variations specific work. // browser.browserAction.setIcon({path: `icons/${variation}.png`}); - browser.browserAction.setTitle({title: variation.name}); + browser.browserAction.setTitle({ title: variation.name }); browser.browserAction.onClicked.addListener(() => this.handleClick()); } handleClick() { // note: doesn't persist across a session this.times += 1; console.log("got a click", this.times); - browser.browserAction.setBadgeText({text: this.times.toString()}); + browser.browserAction.setBadgeText({ text: this.times.toString() }); if (this.times == 1) { - telemetry({"evt": "first-click-in-session"}); + telemetry({ evt: "first-click-in-session" }); } - telemetry({"evt": "click", times: ""+this.times}); + telemetry({ evt: "click", times: "" + this.times }); // addon-initiated ending, 5 times in a session if (this.times >= 3) { - msgStudy("endStudy", {reason: "too-popular"}); + msgStudy("endStudy", { reason: "too-popular" }); } } } @@ -53,4 +55,4 @@ class Feature { // initialize the feature, using our specific variation // this isn't robust to race conditions at all, // should keep retrying until it gets an answer -msgStudy("info").then(({variation}) => new Feature({variation})); +msgStudy("info").then(({ variation }) => new Feature({ variation })); diff --git a/examples/stage-2-shield-study/package.json b/examples/stage-2-shield-study/package.json index 9dd1db9..b169dc9 100644 --- a/examples/stage-2-shield-study/package.json +++ b/examples/stage-2-shield-study/package.json @@ -6,10 +6,12 @@ "scripts": { "test": "echo \"Error: no test specified\" && exit 1", "postinstall": "ln -s ../../.. node_modules/shield-studies-addon-utils", - "prebuild": "cp node_modules/shield-studies-addon-utils/dist/StudyUtils.jsm addon/", + "prebuild": + "cp node_modules/shield-studies-addon-utils/dist/StudyUtils.jsm addon/", "build": "XPI=shield-embedded-web-extension.xpi bash ./bin/xpi.sh", "eslint": "eslint addon --ext jsm --ext js --ext json --", - "watch": "onchange 'addon/**' 'package.json' 'template/**' -e addon/install.rdf -e addon/chrome.manifest -e addon/StudyUtils.jsm -- npm run build -- '{{event}} {{changed}} $(date)'" + "watch": + "onchange 'addon/**' 'package.json' 'template/**' -e addon/install.rdf -e addon/chrome.manifest -e addon/StudyUtils.jsm -- npm run build -- '{{event}} {{changed}} $(date)'" }, "keywords": [], "author": "Gregg Lind ", diff --git a/howToShieldStudy.md b/howToShieldStudy.md index 45c8007..69f238a 100644 --- a/howToShieldStudy.md +++ b/howToShieldStudy.md @@ -1,39 +1,46 @@ -# How To Shield Study +# How To Shield Study **WARNING THIS INFORMATION IS OUT OF DATE WITH V4** - ## Key Ideas - Knowing Answers Using Studies -- :crystal_ball: Shield Studies are addons with instrumentation to help you [see the future](#knowing) and make decisions about features -- :bar_chart: Shield Studies compare user response to feature variations -- :seedling: This tutorial builds a study addon from the ground up -- :floppy_disk: See [full tutorial code below](#final-code) -- :bomb: [`shield`, the Shield Study CLI Tool](https://github.com/gregglind/shield-study-cli) -- :pencil: [Shield Studies Addon Template](https://github.com/gregglind/shield-studies-addon-template), includes common build, linting, and deploy machinery +* :crystal_ball: Shield Studies are addons with instrumentation to help you + [see the future](#knowing) and make decisions about + features +* :bar_chart: Shield Studies compare user response to feature variations +* :seedling: This tutorial builds a study addon from the ground up +* :floppy_disk: See [full tutorial code below](#final-code) +* :bomb: + [`shield`, the Shield Study CLI Tool](https://github.com/gregglind/shield-study-cli) +* :pencil: + [Shield Studies Addon Template](https://github.com/gregglind/shield-studies-addon-template), + includes common build, linting, and deploy machinery ## Example: Puppies Up In Your Browser :dog: :arrow_up: :computer: -[Fake!] VP of Firefox has a [pet theory](#notsorry) that users love theming their browser with puppy images. +[Fake!] VP of Firefox has a +[pet theory](#notsorry) that +users love theming their browser with puppy images. You want to please the VP of Firefox. So, you build an addon to get some puppies up in yer browser. +1. Install some needful tools. -1. Install some needful tools. + ```shell - ```shell - # useful global cli tools - npm install -g shield-study-cli jpm - ``` + # useful global cli tools + + npm install -g shield-study-cli jpm +```` 2. Start a project. **Low level (raw)** - ```shell - # a directory + ```shell + # a directory mkdir theme-shield-study && cd $_ jpm init npm install --save-dev jpm @@ -85,9 +92,9 @@ Is it better? 1. For **Release** users, does theming lead to - 1. **better retention** - 2. **higher usage** - 3. **more statisfaction** +1. **better retention** +2. **higher usage** +3. **more statisfaction** 2. Do users want **puppies or kittens** more? @@ -127,16 +134,16 @@ Let's expand the choice of themes. Organize all the choices into a `variations` /** theme.js */ function theme (choice) { - // do theme work - console.log("theme is", choice); +// do theme work +console.log("theme is", choice); } exports.variations = { - "notheme": () {}, - "puppies": () => theme("puppies"), - "kittens": () => theme("kittens") +"notheme": () {}, +"puppies": () => theme("puppies"), +"kittens": () => theme("kittens") } -``` +```` **index.js** @@ -144,156 +151,164 @@ exports.variations = { /** index.js */ /* give a theme to the user */ -require("./theme").variations.puppies() +require("./theme").variations.puppies(); ``` ### More Themes Makes New Problems -:crying_cat_face: New problem: in the current `index.js`, all the users get the same theme – _puppies_. +:crying*cat_face: New problem: in the current `index.js`, all the users get the +same theme – _puppies*. -Shield-studies are **single-blinded** experiments. Ideally, we want a paricular user to be assigned one of the choices **at random**, with equal probabality. +Shield-studies are **single-blinded** experiments. Ideally, we want a paricular +user to be assigned one of the choices **at random**, with equal probabality. -Bad solution: We **could** write our own randomization function to tackle this. *And* store the choice in a preference, so that the user **always** gets the same variation. +Bad solution: We **could** write our own randomization function to tackle this. +_And_ store the choice in a preference, so that the user **always** gets the +same variation. -Better solution: use `shield-studies-addon-utils`, which handles these headaches – randomization, persistence (and more) – for you. +Better solution: use `shield-studies-addon-utils`, which handles these headaches +– randomization, persistence (and more) – for you. -1. Add `shield-studies-addon-utils` to the project. +1. Add `shield-studies-addon-utils` to the project. - ```shell - npm install --save-dev shield-studies-addon-utils - ``` + ```shell + npm install --save-dev shield-studies-addon-utils + ``` -2. Use `shield-studies-addon-utils` in the project code. +2. Use `shield-studies-addon-utils` in the project code. - **index.js** - - ```js - /** index.js */ - const self = require("sdk/self"); + **index.js** - const shield = require("shield-studies-addon-utils"); - const studyConfig = require("./theme"); - const thisStudy = new shield.Study(studyConfig); - thisStudy.startup(self.loadReason); - ``` + ```js + /** index.js */ + const self = require("sdk/self"); -Now `thisStudy.startup` **decides** which branch the user receives, by setting these prefs (if unset): + const shield = require("shield-studies-addon-utils"); + const studyConfig = require("./theme"); + const thisStudy = new shield.Study(studyConfig); + thisStudy.startup(self.loadReason); + ``` - - `.shield.variation` - which variation user will get - - `.shield.firstrun` - epoch when study began +Now `thisStudy.startup` **decides** which branch the user receives, by setting +these prefs (if unset): -:gift: **Shield Bonus**: `Study` **re-uses** these choices during subsequent startups. +* `.shield.variation` - which variation user will get +* `.shield.firstrun` - epoch when study began -:gift: **Shield Bonus**: There are ways of specifying which variation the user gets for testing, using the `Study.decide` method. Overriding this method can also do non-random or unequal assignment to `variation`. +:gift: **Shield Bonus**: `Study` **re-uses** these choices during subsequent +startups. +:gift: **Shield Bonus**: There are ways of specifying which variation the user +gets for testing, using the `Study.decide` method. Overriding this method can +also do non-random or unequal assignment to `variation`. ### Pick a Theme, Any Theme -Use the `shield` cli command to run a particular variation for testing and demonstration purposes. - +Use the `shield` cli command to run a particular variation for testing and +demonstration purposes. 1. demo "kittens" variation on Aurora (Developer Edition) - ``` - $ shield . kittens -- -b Aurora - ``` + ``` + $ shield . kittens -- -b Aurora + ``` :gift: **Shield Bonus**: Arguments after the `--` are passed along to `jpm`. - ### Housekeeping: Isolate the Study Code -I don't like tangling up my feature code with study code, so let's spread it out a bit more. +I don't like tangling up my feature code with study code, so let's spread it out +a bit more. Let's generalize `theme.js => feature.js`. While we are here, we give the study a `config.name`. -1. Rename `theme.js` to `feature.js`, change the function names. +1. Rename `theme.js` to `feature.js`, change the function names. - **feature.js (`theme.js`)** + **feature.js (`theme.js`)** - ```js - /** feature.js **/ + ```js + /** feature.js **/ - function feature (choice) { - // do feature work - console.log("feature is", choice); - } - exports.feature = feature; - ``` + function feature(choice) { + // do feature work + console.log("feature is", choice); + } + exports.feature = feature; + ``` -2. Add `config.name`, use `feature`. +2. Add `config.name`, use `feature`. - **study.js** + **study.js** - ```js - /** study.js **/ + ```js + /** study.js **/ - const self = require("sdk/self"); - const shield = require("shield-studies-addon-utils"); + const self = require("sdk/self"); + const shield = require("shield-studies-addon-utils"); - const { feature } = require("./feature"); + const { feature } = require("./feature"); - const studyConfig = { - name: self.addonId, - variations = { - "notheme": () => feature("notheme"), - "puppies": () => feature("puppies"), - "kittens": () => feature("kittens") - } - } + const studyConfig = { + name: self.addonId, + variations = { + "notheme": () => feature("notheme"), + "puppies": () => feature("puppies"), + "kittens": () => feature("kittens") + } + } - const thisStudy = new shield.Study(studyConfig); + const thisStudy = new shield.Study(studyConfig); - exports.study = thisStudy; - ``` + exports.study = thisStudy; + ``` 3. Use the `study` in `index.js` - **index.js (final form)** + **index.js (final form)** + + ```js + /** index.js **/ - ```js - /** index.js **/ const self = require("sdk/self"); - require("./study").study.startup(self.loadReason); - ``` + require("./study").study.startup(self.loadReason); ``` -4. Let's fancy up the study by subclassing `shield.Study`: +4. Let's fancy up the study by subclassing `shield.Study`: - **study.js** + **study.js** - ```js - /** study.js **/ - const self = require("sdk/self"); - const shield = require("shield-studies-addon-utils"); - - const { feature } = require("./feature"); - - const studyConfig = { - name: self.addonId, - variations: { - "notheme": () => feature("notheme"), - "puppies": () => feature("puppies"), - "kittens": () => feature("kittens") - } - } - } + ```js + /** study.js **/ + const self = require("sdk/self"); + const shield = require("shield-studies-addon-utils"); - class OurStudy extends shield.Study { - constructor (config) { - super(config); - } - } + const { feature } = require("./feature"); - const thisStudy = new OurStudy(studyConfig); + const studyConfig = { + name: self.addonId, + variations: { + "notheme": () => feature("notheme"), + "puppies": () => feature("puppies"), + "kittens": () => feature("kittens") + } + } + } - exports.study = thisStudy; - ``` + class OurStudy extends shield.Study { + constructor (config) { + super(config); + } + } + + const thisStudy = new OurStudy(studyConfig); -:gift: **Shield Bonus**: `shield.Study` has a lot of overridable methods. [Full Shield.Study Api]() + exports.study = thisStudy; + ``` +:gift: **Shield Bonus**: `shield.Study` has a lot of overridable methods. +[Full Shield.Study Api]() ## Telemetry For Free @@ -301,27 +316,32 @@ By using a `shield.Study`, we get some Telemetry for free. The `Study` sends [shield-study telemetry packets]() for these lifecycle events: -- `install`: once -- `startup`: every Firefox startup -- `shutdown`: every end of session -- `running`: once per day -- `end-of-study`: study expires (is complete) -- `user-ended-study`: user disabled or uninstalled -- `ineligible`: user was not eligilbe +* `install`: once +* `startup`: every Firefox startup +* `shutdown`: every end of session +* `running`: once per day +* `end-of-study`: study expires (is complete) +* `user-ended-study`: user disabled or uninstalled +* `ineligible`: user was not eligilbe These data are enough to answer Research Questions 1.1, 1.2. -For Question 1.3 – _more satisfication_ – we need to _ask_ users about their perceptions. +For Question 1.3 – _more satisfication_ – we need to _ask_ users +about their perceptions. -## What does the user think? Surveys +## What does the user think? Surveys -Configure `surveyUrls`, so we can ask questions to users about their experience. Surveys can be called when the user experiences one of these 3 triggers: +Configure `surveyUrls`, so we can ask questions to users about their experience. +Surveys can be called when the user experiences one of these 3 triggers: -- `user-ended-study`: disable or uninstall -- `end-of-study`: the study expired naturally -- `ineligible`: user was ineligible +* `user-ended-study`: disable or uninstall +* `end-of-study`: the study expired naturally +* `ineligible`: user was ineligible -At these 3 times, the `study` will open a background tab to that survey url. The `shield.survey` function appends a `reason` queryArg and some other fields to the `surveyUrl`. One could use the same url for all 3. If any of these 3 are `null`, the `survey` will NOT be opened at that time. +At these 3 times, the `study` will open a background tab to that survey url. The +`shield.survey` function appends a `reason` queryArg and some other fields to +the `surveyUrl`. One could use the same url for all 3. If any of these 3 are +`null`, the `survey` will NOT be opened at that time. Get these urls and survey help from Firefox Strategy + Insights Team. @@ -343,8 +363,8 @@ const studyConfig = { } ``` -:gift: **Shield Bonus**: Lint `surveyUrls` with `Study.lint()` or the `shield` cli tool. - +:gift: **Shield Bonus**: Lint `surveyUrls` with `Study.lint()` or the `shield` +cli tool. ## Study Lifecycle @@ -357,17 +377,25 @@ Our Study works™, in that a user: For some studies, this is enough. -Other studies want a little more UI during the study life-cycle to handle orientation, cleanup, and ineligibility. They may also want to add their own measures for feature-specific usage. +Other studies want a little more UI during the study life-cycle to handle +orientation, cleanup, and ineligibility. They may also want to add their own +measures for feature-specific usage. -:crying_cat_face: Nobody wants to test that this UI works properly across restarts and under different study life-cycle conditions. +:crying_cat_face: Nobody wants to test that this UI works properly across +restarts and under different study life-cycle conditions. -:gift: **Shield Bonus**: handling life-cycle conditions is kind of the point of Shield :clap: :+1: +:gift: **Shield Bonus**: handling life-cycle conditions is kind of the point of +Shield :clap: :+1: ### Beginnings: Ineligible Users -While we are testing the addon internally, we realize that some users already have applied puppy (or other) themes. We don't want to enroll these users, because their data doesn't apply to the feature rollout questions. +While we are testing the addon internally, we realize that some users already +have applied puppy (or other) themes. We don't want to enroll these users, +because their data doesn't apply to the feature rollout questions. -By default, if they are ineligible, the addon will silently uninstall. We can augment that if we wish, by overriding the specially named `whenIneligible` function. +By default, if they are ineligible, the addon will silently uninstall. We can +augment that if we wish, by overriding the specially named `whenIneligible` +function. So, let's check if they are eligible, and handle them differently: @@ -376,14 +404,14 @@ So, let's check if they are eligible, and handle them differently: ```js /** feature.js **/ -const tabs = require('sdk/tabs'); +const tabs = require("sdk/tabs"); const prefSvc = require("sdk/preferences/service"); // ... -exports.isEligible = function () { - return !prefSvc.isSet('some.pref.somewhere'); -} +exports.isEligible = function() { + return !prefSvc.isSet("some.pref.somewhere"); +}; ``` **study.js** @@ -408,29 +436,31 @@ class OurStudy extends shield.Study { } ``` - ### Beginnings: Orientation -It would be nice to explain to an enrolled user what the heck is going on, and how to turn this experiment off. +It would be nice to explain to an enrolled user what the heck is going on, and +how to turn this experiment off. **feature.js** ```js /** feature.js **/ -const tabs = require('sdk/tabs'); +const tabs = require("sdk/tabs"); // ... -exports.orientation = function orientation (choice) { - return tabs.open(`data:text/html,You are on choice {choice}. Stop by, use by etc`) -} +exports.orientation = function orientation(choice) { + return tabs.open( + `data:text/html,You are on choice {choice}. Stop by, use by etc` + ); +}; // ... ``` -We have a special issue that the 'control' variation shouldn't have orientation. Let's handle that better. - +We have a special issue that the 'control' variation shouldn't have orientation. +Let's handle that better. **study.js** @@ -446,8 +476,8 @@ class OurStudy extends shield.Study { } ``` -:gift: **Shield bonus**: the `Study.variation` property knows the user's assigned variation. - +:gift: **Shield bonus**: the `Study.variation` property knows the user's +assigned variation. ### All Good Things Must End @@ -459,26 +489,27 @@ We need to tell `study.js` to handle endings (shutdown): const { when: unload } = require("sdk/system/unload"); //... -unload((reason) => thisStudy.shutdown(reason)) +unload(reason => thisStudy.shutdown(reason)); ``` There are 3 possible endings -- natural study expiration (completion) -- user uninstall or disable -- user ineligible (never install), described previously. - +* natural study expiration (completion) +* user uninstall or disable +* user ineligible (never install), described previously. #### Completing the Study (Natural Expiration) -Let's decide how long we want to collect data. By default, it is 7 days. After that period: +Let's decide how long we want to collect data. By default, it is 7 days. After +that period: -- the addon phones home as `end-of-study` -- a survey url opens (if provided) indicating `end-of-study` -- cleanup happens -- the study addon uninstalls +* the addon phones home as `end-of-study` +* a survey url opens (if provided) indicating `end-of-study` +* cleanup happens +* the study addon uninstalls -We decide to collect data for 14 days, to measure longer term retention and satisfaction. +We decide to collect data for 14 days, to measure longer term retention and +satisfaction. **study.js** @@ -497,25 +528,28 @@ const studyConfig = { #### User Uninstalls or Disables -Shield Studies treat disable and uninstall as equivalent events for these reasons: +Shield Studies treat disable and uninstall as equivalent events for these +reasons: -- (science): Disable, then re-enable is an unclear statement of intent, and hard to interpret -- (technical): The SDK has a hard time distinguishing them +* (science): Disable, then re-enable is an unclear statement of intent, and hard + to interpret +* (technical): The SDK has a hard time distinguishing them During `uninstall` or `disable`: -- the addon phones home as `user-ended-study` -- a survey url opens (if provided) indicating `user-ended-study` -- cleanup happens -- the study addon uninstalls - - -:gift: **Shield Bonus**: when the user disables or uninstalls the Study addon, it phones home! We can count that too, for each variation, to measure opt-out rate, by variation. +* the addon phones home as `user-ended-study` +* a survey url opens (if provided) indicating `user-ended-study` +* cleanup happens +* the study addon uninstalls +:gift: **Shield Bonus**: when the user disables or uninstalls the Study addon, +it phones home! We can count that too, for each variation, to measure opt-out +rate, by variation. ### Extra Measurements -You can add extra probes using `study.Report`. Let's instrument that the users saw orientation. +You can add extra probes using `study.Report`. Let's instrument that the users +saw orientation. **study.js** @@ -538,7 +572,6 @@ class OurStudy extends shield.Study { This will report to Telemetry as: ```js - // usual telemetry environment is also included // specific payload { @@ -549,27 +582,27 @@ This will report to Telemetry as: "study_name": 'the name' //... } - ``` -:gift: **Shield Bonus**: You can call `shield.report` directly if needed. `Study.report` is a static convenience method. - +:gift: **Shield Bonus**: You can call `shield.report` directly if needed. +`Study.report` is a static convenience method. ### Debugging and watching the Study in action -Add some listeners for telemetry and Study state changes. This might help explain what `Shield` is doing, and convince you that it is doing it correctly. +Add some listeners for telemetry and Study state changes. This might help +explain what `Shield` is doing, and convince you that it is doing it correctly. **study.js** ```js -prefsSvc.set('shield.study.debug', true) +prefsSvc.set("shield.study.debug", true); ``` WOULD turn on debugging: ```js -shield.Reporter.on("report",(d)=>console.log("telemetry", d)); -thisStudy.on("change",(newState)=>console.log("newState:", newState)); +shield.Reporter.on("report", d => console.log("telemetry", d)); +thisStudy.on("change", newState => console.log("newState:", newState)); ``` :gift: **Shield Bonus**: `shield run . --debug` does the same thing. @@ -578,155 +611,166 @@ thisStudy.on("change",(newState)=>console.log("newState:", newState)); **TODO** -- in progress cli tools -- in progress shield-study-addon-template +* in progress cli tools +* in progress shield-study-addon-template ### shield-study-addon-template -1. Check out [shield-studies-addon-template](https://github.com/gregglind/shield-studies-addon-template) +1. Check out + [shield-studies-addon-template](https://github.com/gregglind/shield-studies-addon-template) - ``` - URL=https://github.com/gregglind/shield-studies-addon-template - git clone --depth 1 $URL "your-addon_name" and && cd $_ - rm -rf .git - git init - # setup git stuff for your branch + ``` + URL=https://github.com/gregglind/shield-studies-addon-template + git clone --depth 1 $URL "your-addon_name" and && cd $_ + rm -rf .git + git init + # setup git stuff for your branch - ## get to work! - ``` + ## get to work! + ``` - This is also `shield init my-project`, that does the same thing. + This is also `shield init my-project`, that does the same thing. -2. Modify the things in these files +2. Modify the things in these files - - `lib/index.js` - - `lib/study.js` - - `lib/feature` - - `package.json` + * `lib/index.js` + * `lib/study.js` + * `lib/feature` + * `package.json` -3. fix tests -4. profit. +3. fix tests +4. profit. ### shield-studies-cli -0. `npm install -g shield-studies-cli` -1. use the `shield` cli tool - - - `shield run ./ some-variation` - - `shield survey` +0. `npm install -g shield-studies-cli` +1. use the `shield` cli tool +* `shield run ./ some-variation` +* `shield survey` ### Deploy -- host addon publically on amo -- needs an AMO static page -- needs all the bugs file [TODO] -- get a normany recipes +* host addon publically on amo +* needs an AMO static page +* needs all the bugs file [TODO] +* get a normany recipes ## Answer Your Research Questions -Contact your local data science wizard to get the data back from Unified Telemetry. We will have a dashboard-feeding summarization job going Real Soon. +Contact your local data science wizard to get the data back from Unified +Telemetry. We will have a dashboard-feeding summarization job going Real Soon. ## Other magical magic -1. Some useful prefs: +1. Some useful prefs: - - `shield.study.fakedie`: won't uninstall - - `shield.study.debug`: more debugging + * `shield.study.fakedie`: won't uninstall + * `shield.study.debug`: more debugging -2. Every `Study` objects is an `EventTarget`. You can listen / emit on it directly. +2. Every `Study` objects is an `EventTarget`. You can listen / emit on it + directly. - ```js - aStudy.once('installed', function onceInstalled () {}) - ``` + ```js + aStudy.once("installed", function onceInstalled() {}); + ``` -3. Choosing a variation for a demo or QA run. +3. Choosing a variation for a demo or QA run. - `shield . variationName` + `shield . variationName` -4. Unequal (or complex) assignment variations +4. Unequal (or complex) assignment variations - Override `decideVariation`. + Override `decideVariation`. - ```js - unequalVariations (study, rng=Math.random()) { - // always return an rng - if (rng < .3) { - return 'a'; - elif (rng < .9) { - return 'b'; - } else { - return 'c' - } - } + ```js + unequalVariations (study, rng=Math.random()) { + // always return an rng + if (rng < .3) { + return 'a'; + elif (rng < .9) { + return 'b'; + } else { + return 'c' + } + } - class UnequalAssigment extends shield.Study { - // only used during first `install` - decideVariation () { - return unequalVariations(this) - } - } - ``` + class UnequalAssigment extends shield.Study { + // only used during first `install` + decideVariation () { + return unequalVariations(this) + } + } + ``` ## Full Study Api ### Useful Overridable Methods in `Study` -- `cleanup`: what happens during uninstall (any reason). -- `decideVariation`: choose which variation user gets on install. -- `isEligible`: boolean for 'should study even install?' -- `showSurvey`: how to show surveys, append on query args, etc. -- `surveyQueryArgs`: what queryArgs to append onto surveys. -- `whenIneligible`: if user is NOT eligible, should we do anything, like explain why, or inform them? -- `whenInstalled`: after successful install, now what? Orientation? -- `whenComplete`: when study completes (expires naturally), should anything unusual happen? - -See the [source code](https://github.com/mozilla/shield-studies-addon-utils/blob/master/lib/index.js) for more details. +* `cleanup`: what happens during uninstall (any reason). +* `decideVariation`: choose which variation user gets on install. +* `isEligible`: boolean for 'should study even install?' +* `showSurvey`: how to show surveys, append on query args, etc. +* `surveyQueryArgs`: what queryArgs to append onto surveys. +* `whenIneligible`: if user is NOT eligible, should we do anything, like explain + why, or inform them? +* `whenInstalled`: after successful install, now what? Orientation? +* `whenComplete`: when study completes (expires naturally), should anything + unusual happen? + +See the +[source code](https://github.com/mozilla/shield-studies-addon-utils/blob/master/lib/index.js) +for more details. ### Listenable signals One can also listen directly on a running study for signals. -See the [source code](https://github.com/mozilla/shield-studies-addon-utils/blob/master/lib/index.js) for more details. - - +See the +[source code](https://github.com/mozilla/shield-studies-addon-utils/blob/master/lib/index.js) +for more details. ## Final Code ### feature.js + ```js /** feature.js **/ -const tabs = require('sdk/tabs'); +const tabs = require("sdk/tabs"); const prefSvc = require("sdk/preferences/service"); -exports.which = function whichFeature (choice) { +exports.which = function whichFeature(choice) { // do feature work console.log("feature is", choice); -} +}; -exports.orientation = function orientation (choice) { - return tabs.open(`data:text/html,You are on choice {choice}. Stop by, use by etc`) -} +exports.orientation = function orientation(choice) { + return tabs.open( + `data:text/html,You are on choice {choice}. Stop by, use by etc` + ); +}; -exports.isEligible = function () { - return !prefSvc.isSet('some.pref.somewhere'); -} +exports.isEligible = function() { + return !prefSvc.isSet("some.pref.somewhere"); +}; ``` + ### index.js + ```js /** index.js **/ const self = require("sdk/self"); require("./study").study.startup(self.loadReason); - ``` ### study.js + ```js /** study.js **/ const self = require("sdk/self"); const shield = require("shield-studies-addon-utils"); -const tabs = require('sdk/tabs'); +const tabs = require("sdk/tabs"); const { when: unload } = require("sdk/system/unload"); const feature = require("./feature"); @@ -734,51 +778,54 @@ const feature = require("./feature"); const studyConfig = { name: self.addonId, duration: 14, - surveyUrls: { - 'end-of-study': 'some/url', - 'user-ended-study': 'some/url', - 'ineligible': null + surveyUrls: { + "end-of-study": "some/url", + "user-ended-study": "some/url", + ineligible: null, }, variations: { - "notheme": () => feature.which("notheme"), - "puppies": () => feature.which("puppies"), - "kittens": () => feature.which("kittens") - } -} + notheme: () => feature.which("notheme"), + puppies: () => feature.which("puppies"), + kittens: () => feature.which("kittens"), + }, +}; class OurStudy extends shield.Study { - constructor (config) { + constructor(config) { super(config); } - isEligible () { + isEligible() { // bool Already Has the feature. Stops install if true - return super.isEligible() && feature.isEligible() + return super.isEligible() && feature.isEligible(); } - whenIneligible () { + whenIneligible() { super.whenIneligible(); // additional actions for 'user isn't eligible' - tabs.open(`data:text/html,Uninstalling, you are not eligible for this study`) + tabs.open( + `data:text/html,Uninstalling, you are not eligible for this study` + ); } - whenInstalled () { + whenInstalled() { super.whenInstalled(); // orientation, unless our branch is 'notheme' - if (this.variation == 'notheme') {} + if (this.variation == "notheme") { + } feature.orientation(this.variation); } - cleanup (reason) { - super.cleanup(); // cleanup simple-prefs, simple-storage + cleanup(reason) { + super.cleanup(); // cleanup simple-prefs, simple-storage // do things, maybe depending on reason, branch } - whenComplete () { + whenComplete() { // when the study is naturally complete after this.days - super.whenComplete(); // calls survey, uninstalls + super.whenComplete(); // calls survey, uninstalls } - whenUninstalled () { + whenUninstalled() { // user uninstall super.whenUninstalled(); } - decideVariation () { - return super.decideVariation() // chooses at random + decideVariation() { + return super.decideVariation(); // chooses at random // unequal or non random allocation for example } } @@ -792,18 +839,15 @@ exports.studyConfig = studyConfig; // for use by index.js exports.study = thisStudy; -unload((reason) => thisStudy.shutdown(reason)) - - +unload(reason => thisStudy.shutdown(reason)); ``` ## Enjoy Kittens and Puppies Example Studies: -- [kittens and puppies example](https://github.com/mozilla/shield-studies-addon-utils/blob/master/examples/example-theme-study) -- [simpler pref-flipping study](https://github.com/mozilla/shield-studies-addon-utils/blob/master/examples/example-pref-flip) - +* [kittens and puppies example](https://github.com/mozilla/shield-studies-addon-utils/blob/master/examples/example-theme-study) +* [simpler pref-flipping study](https://github.com/mozilla/shield-studies-addon-utils/blob/master/examples/example-pref-flip) ## Author @@ -813,41 +857,45 @@ Core tutorial format idea: @Osmose. Readers / feedback / wit: -- Cathy Beil -- @Osmose (Michael Kelly) -- @raymak (Kamyar Ardekani - +* Cathy Beil +* @Osmose (Michael Kelly) +* @raymak (Kamyar Ardekani ## Endnotes -1. Insert a long, long digression about epistimology, the nature of knowledge, and statistic methods, perhaps summarized as: +1. Insert a long, long digression about epistimology, the + nature of knowledge, and statistic methods, perhaps summarized as: - "By looking at a sample of people like the ones who will use the feature, we can infer what is *likely* to happen if deploy it for real." + "By looking at a sample of people like the ones who will use the feature, we can infer what is *likely* to happen if deploy it for real." - For this to be true we need: + For this to be true we need: 1. **random sample** ascertiained (enrolled) at random - 2. **from similar popuation** a population "alike-enough" to the "real" popuation - 3. **sample size** of a large enough size such that observed effects are likely to not be due to chance - 4. **fidelity of experience** measured using an experience that is "close enough" to how it will be "for real" - 5. **detectable effect** *but* where the variations are *different enough* to be different from each other and from a control (observe-only) experience – the effect size must be big enough to observe - 6. **blinded, random assignment** where each user is *assigned at random* (and blinded to which variation they received) + 2. **from similar popuation** a population "alike-enough" to the "real" + popuation + 3. **sample size** of a large enough size such that observed effects are + likely to not be due to chance + 4. **fidelity of experience** measured using an experience that is "close + enough" to how it will be "for real" + 5. **detectable effect** _but_ where the variations are _different enough_ to + be different from each other and from a control (observe-only) experience + – the effect size must be big enough to observe + 6. **blinded, random assignment** where each user is _assigned at random_ + (and blinded to which variation they received) 7. **control group** with a control group to see "normal usage" - The more our experiment differs from these assumptions, the less well it predicts the future. + The more our experiment differs from these assumptions, the less well it predicts the future. - Much of the Shield Studies future work is about improving the fidelity of the sample to (real) Release users, and fixing things like: + Much of the Shield Studies future work is about improving the fidelity of the sample to (real) Release users, and fixing things like: - - Ascertainment bias (recruitment issues) - - blinding + - Ascertainment bias (recruitment issues) + - blinding ⇧ - 2. Sorry, not sorry. ⇧ -3. The Firefox Strategy + Insights "Weird Science" Subteam - Gregg Lind and friends. ⇧ - - +3. The Firefox Strategy + Insights "Weird Science" + Subteam - Gregg Lind and friends. ⇧ diff --git a/package.json b/package.json index 21cbb2d..0aee3dd 100644 --- a/package.json +++ b/package.json @@ -21,21 +21,13 @@ "eslint-plugin-json": "^1.2.0", "eslint-plugin-mozilla": "^0.4.0", "fixpack": "^2.3.1", + "prettier": "1.8.2", "shield-study-schemas": "^0.8.3", "webpack": "^2.6.1" }, - "files": [ - "dist" - ], + "files": ["dist"], "homepage": "https://github.com/mozilla/shield-studies-addon-utils#readme", - "keywords": [ - "addon", - "jsm", - "mozilla", - "normandy", - "shield", - "shield-study" - ], + "keywords": ["addon", "jsm", "mozilla", "normandy", "shield", "shield-study"], "license": "MPL-2.0", "main": "lib/index.js", "repository": { @@ -46,9 +38,12 @@ "build-test-addon-xpi": "./bin/make_xpi.sh", "dist": "webpack", "eslint": "eslint src --ext jsm --ext js --ext json", + "format": "prettier '**/*.{css,js,jsm,json,md}' --trailing-comma=es5 --write", + "postformat": "npm run eslint -- --fix", "predist": "npm run eslint", "prepack": "fixpack && npm run dist", "pretest": "npm run dist && npm run build-test-addon-xpi", - "test": "export FIREFOX_BINARY=firefox && XPI_NAME=test-addon/test-addon.xpi mocha test" + "test": + "export FIREFOX_BINARY=firefox && XPI_NAME=test-addon/test-addon.xpi mocha test" } } diff --git a/shield-study-helper-addon/addon/bootstrap.js b/shield-study-helper-addon/addon/bootstrap.js index 7db1e9b..75b97a6 100644 --- a/shield-study-helper-addon/addon/bootstrap.js +++ b/shield-study-helper-addon/addon/bootstrap.js @@ -3,15 +3,15 @@ /* global __SCRIPT_URI_SPEC__, Feature, studyUtils, config */ /* eslint no-unused-vars: ["error", { "varsIgnorePattern": "(startup|shutdown|install|uninstall)" }]*/ -async function getTelemetryPings (options) { +async function getTelemetryPings(options) { // type is String or Array - const {type, n, timestamp, headersOnly} = options; + const { type, n, timestamp, headersOnly } = options; Components.utils.import("resource://gre/modules/TelemetryArchive.jsm"); // {type, id, timestampCreated} let pings = await TelemetryArchive.promiseArchivedPingList(); if (type) { if (!(type instanceof Array)) { - type = [type]; // Array-ify if it's a string + type = [type]; // Array-ify if it's a string } } if (type) pings = pings.filter(p => type.includes(p.type)); @@ -19,23 +19,26 @@ async function getTelemetryPings (options) { pings.sort((a, b) => b.timestampCreated - a.timestampCreated); if (n) pings = pings.slice(0, n); - const pingData = headersOnly ? pings : pings.map(ping => TelemetryArchive.promiseArchivedPingById(ping.id)); - return Promise.all(pingData) + const pingData = headersOnly + ? pings + : pings.map(ping => TelemetryArchive.promiseArchivedPingById(ping.id)); + return Promise.all(pingData); } -async function pingsReport () { +async function pingsReport() { async function getPings() { const ar = ["shield-study", "shield-study-addon"]; - return getTelemetryPings({type: ["shield-study", "shield-study-addon"]}); + return getTelemetryPings({ type: ["shield-study", "shield-study-addon"] }); } const pings = (await getPings()).reverse(); if (pings.length == 0) { - return {"report": "No pings found"} + return { report: "No pings found" }; } const p0 = pings[0].payload; // print common fields - const report = ` + const report = + ` // common fields branch ${p0.branch} // should describe Question text @@ -43,20 +46,23 @@ study_name ${p0.study_name} addon_version ${p0.addon_version} version ${p0.version} -` + pings.map((p,i)=>`${i} ${p.creationDate} ${p.payload.type} -${JSON.stringify(p.payload.data,null,2)} +` + + pings + .map( + (p, i) => `${i} ${p.creationDate} ${p.payload.type} +${JSON.stringify(p.payload.data, null, 2)} -`).join('\n'); +` + ) + .join("\n"); - return {"report": report}; + return { report: report }; //pings.forEach(p=>{ // console.log(p.creationDate, p.payload.type); // console.log(JSON.stringify(p.payload.data,null,2)) //}) } - - async function listenFromWebExtension(msg, sender, sendResponse) { //await pingsReport(); console.log(`got ${msg}`); @@ -76,12 +82,11 @@ async function listenFromWebExtension(msg, sender, sendResponse) { return false; } - async function startup(addonData, reason) { - console.log('starting up debugger') + console.log("starting up debugger"); const webExtension = addonData.webExtension; webExtension.startup().then(api => { - const {browser} = api; + const { browser } = api; // messages intended for shieldn: {shield:true,msg=[info|endStudy|telemetry],data=data} browser.runtime.onMessage.addListener(listenFromWebExtension); // other message handlers from your addon, if any diff --git a/shield-study-helper-addon/addon/webextension/.eslintrc.json b/shield-study-helper-addon/addon/webextension/.eslintrc.json index 1d71c50..5c7cc9e 100644 --- a/shield-study-helper-addon/addon/webextension/.eslintrc.json +++ b/shield-study-helper-addon/addon/webextension/.eslintrc.json @@ -1,13 +1,11 @@ { - "env": { - "browser": true, - "es6": true, - "webextensions": true - }, - "extends": [ - "eslint:recommended" - ], - "rules": { - "no-console": "warn" - } + "env": { + "browser": true, + "es6": true, + "webextensions": true + }, + "extends": ["eslint:recommended"], + "rules": { + "no-console": "warn" + } } diff --git a/shield-study-helper-addon/addon/webextension/qa.js b/shield-study-helper-addon/addon/webextension/qa.js index ef036a1..d9f3142 100644 --- a/shield-study-helper-addon/addon/webextension/qa.js +++ b/shield-study-helper-addon/addon/webextension/qa.js @@ -7,26 +7,25 @@ has a bunch of pings and stuff `; -function printReport (text) { +function printReport(text) { console.log(`about to replace: ${text}`); - document.querySelector('#timestamp').textContent=`${new Date()}`; - document.querySelector('#qa').textContent=text; + document.querySelector("#timestamp").textContent = `${new Date()}`; + document.querySelector("#qa").textContent = text; } - -async function tryReportFromFirefox () { - console.log(`has browser runtime? ${browser.runtime}`) +async function tryReportFromFirefox() { + console.log(`has browser runtime? ${browser.runtime}`); if (browser.runtime) { - const reply = await browser.runtime.sendMessage("qa-report") + const reply = await browser.runtime.sendMessage("qa-report"); console.log("got reply!", reply); if (reply) { printReport(reply.report); //console.log("response from legacy add-on: " + reply.content); - }; + } } } -function startup () { +function startup() { printReport(PLACEHOLDER); console.log("asking firefox"); tryReportFromFirefox(); @@ -39,5 +38,4 @@ page starts up. - once it arrives, insert it. */ - -document.addEventListener('DOMContentLoaded', startup); +document.addEventListener("DOMContentLoaded", startup); diff --git a/shield-study-helper-addon/run-firefox.js b/shield-study-helper-addon/run-firefox.js index c784775..d72ca7e 100644 --- a/shield-study-helper-addon/run-firefox.js +++ b/shield-study-helper-addon/run-firefox.js @@ -10,7 +10,6 @@ console.log("Starting up firefox"); - require("geckodriver"); const firefox = require("selenium-webdriver/firefox"); const cmd = require("selenium-webdriver/lib/command"); @@ -60,11 +59,11 @@ async function promiseActualBinary(binary) { } } -promiseSetupDriver = async() => { +promiseSetupDriver = async () => { const profile = new firefox.Profile(); // TODO, allow 'actually send telemetry' here. - Object.keys(FIREFOX_PREFERENCES).forEach((key) => { + Object.keys(FIREFOX_PREFERENCES).forEach(key => { profile.setPreference(key, FIREFOX_PREFERENCES[key]); }); @@ -76,7 +75,9 @@ promiseSetupDriver = async() => { .forBrowser("firefox") .setFirefoxOptions(options); - const binaryLocation = await promiseActualBinary(process.env.FIREFOX_BINARY || "nightly"); + const binaryLocation = await promiseActualBinary( + process.env.FIREFOX_BINARY || "nightly" + ); await options.setBinary(new firefox.Binary(binaryLocation)); const driver = await builder.build(); // Firefox will be started up by now @@ -85,21 +86,28 @@ promiseSetupDriver = async() => { return driver; }; -installAddon = async(driver, fileLocation) => { +installAddon = async (driver, fileLocation) => { // references: // https://bugzilla.mozilla.org/show_bug.cgi?id=1298025 // https://github.com/mozilla/geckodriver/releases/tag/v0.17.0 const executor = driver.getExecutor(); - executor.defineCommand("installAddon", "POST", "/session/:sessionId/moz/addon/install"); + executor.defineCommand( + "installAddon", + "POST", + "/session/:sessionId/moz/addon/install" + ); const installCmd = new cmd.Command("installAddon"); const session = await driver.getSession(); - installCmd.setParameters({ sessionId: session.getId(), path: fileLocation, temporary: true }); + installCmd.setParameters({ + sessionId: session.getId(), + path: fileLocation, + temporary: true, + }); return executor.execute(installCmd); }; - -(async() => { +(async () => { try { const driver = await promiseSetupDriver(); @@ -114,7 +122,6 @@ installAddon = async(driver, fileLocation) => { // navigate to a regular page driver.setContext(Context.CONTENT); driver.get("about:debugging"); - } catch (e) { console.error(e); // eslint-disable-line no-console } diff --git a/src/StudyUtils.in.jsm b/src/StudyUtils.in.jsm index 802cbf9..9ed2515 100644 --- a/src/StudyUtils.in.jsm +++ b/src/StudyUtils.in.jsm @@ -5,7 +5,7 @@ const EXPORTED_SYMBOLS = ["studyUtils"]; const UTILS_VERSION = require("../package.json").version; const PACKET_VERSION = 3; -const {utils: Cu} = Components; +const { utils: Cu } = Components; Cu.import("resource://gre/modules/AddonManager.jsm"); Cu.import("resource://gre/modules/Services.jsm"); Cu.import("resource://gre/modules/XPCOMUtils.jsm"); @@ -15,8 +15,14 @@ let log; // telemetry utils const CID = Cu.import("resource://gre/modules/ClientID.jsm", null); -const { TelemetryController } = Cu.import("resource://gre/modules/TelemetryController.jsm", null); -const { TelemetryEnvironment } = Cu.import("resource://gre/modules/TelemetryEnvironment.jsm", null); +const { TelemetryController } = Cu.import( + "resource://gre/modules/TelemetryController.jsm", + null +); +const { TelemetryEnvironment } = Cu.import( + "resource://gre/modules/TelemetryEnvironment.jsm", + null +); async function getTelemetryId() { const id = TelemetryController.clientID; @@ -34,11 +40,13 @@ const ajv = new Ajv(); var jsonschema = { validate(data, schema) { var valid = ajv.validate(schema, data); - return {valid, errors: ajv.errors || []}; + return { valid, errors: ajv.errors || [] }; }, validateOrThrow(data, schema) { const valid = ajv.validate(schema, data); - if (!valid) { throw new Error(JSON.stringify((ajv.errors))); } + if (!valid) { + throw new Error(JSON.stringify(ajv.errors)); + } return true; }, }; @@ -66,14 +74,20 @@ var jsonschema = { */ function merge(source) { // get object's own property Symbols and/or Names, including nonEnumerables by default - function getOwnPropertyIdentifiers(object, options = { names: true, symbols: true, nonEnumerables: true }) { - const symbols = !options.symbols ? [] : - Object.getOwnPropertySymbols(object); + function getOwnPropertyIdentifiers( + object, + options = { names: true, symbols: true, nonEnumerables: true } + ) { + const symbols = !options.symbols + ? [] + : Object.getOwnPropertySymbols(object); // eslint-disable-next-line - const names = !options.names ? [] : - options.nonEnumerables ? Object.getOwnPropertyNames(object) : - Object.keys(object); + const names = !options.names + ? [] + : options.nonEnumerables + ? Object.getOwnPropertyNames(object) + : Object.keys(object); return [...names, ...symbols]; } const descriptor = {}; @@ -81,11 +95,13 @@ function merge(source) { // `Boolean` converts the first parameter to a boolean value. Any object is // converted to `true` where `null` and `undefined` becames `false`. Therefore // the `filter` method will keep only objects that are defined and not null. - Array.slice(arguments, 1).filter(Boolean).forEach(function onEach(properties) { - getOwnPropertyIdentifiers(properties).forEach(function(name) { - descriptor[name] = Object.getOwnPropertyDescriptor(properties, name); + Array.slice(arguments, 1) + .filter(Boolean) + .forEach(function onEach(properties) { + getOwnPropertyIdentifiers(properties).forEach(function(name) { + descriptor[name] = Object.getOwnPropertyDescriptor(properties, name); + }); }); - }); return Object.defineProperties(source, descriptor); } @@ -100,7 +116,7 @@ function mergeQueryArgs(url, ...args) { const merged = merge({}, ...args); // get user info. - Object.keys(merged).forEach((k) => { + Object.keys(merged).forEach(k => { log.debug(q.get(k), k, merged[k]); q.set(k, merged[k]); }); @@ -114,12 +130,17 @@ async function sha256(message) { const msgBuffer = new TextEncoder("utf-8").encode(message); // encode as UTF-8 const hashBuffer = await crypto.subtle.digest("SHA-256", msgBuffer); // hash the message const hashArray = Array.from(new Uint8Array(hashBuffer)); // convert ArrayBuffer to Array - const hashHex = hashArray.map(b => ("00" + b.toString(16)).slice(-2)).join(""); // convert bytes to hex string + const hashHex = hashArray + .map(b => ("00" + b.toString(16)).slice(-2)) + .join(""); // convert bytes to hex string return hashHex; } function cumsum(arr) { - return arr.reduce(function(r, c, i) { r.push((r[i - 1] || 0) + c); return r; }, [] ); + return arr.reduce(function(r, c, i) { + r.push((r[i - 1] || 0) + c); + return r; + }, []); } function chooseWeighted(weightedVariations, fraction = Math.random()) { @@ -151,19 +172,27 @@ async function hashFraction(saltedString, bits = 12) { class StudyUtils { constructor(config) { // TODO glind Answer: no. see if you can merge the construtor and setup and export the class, rather than a singleton - this.respondToWebExtensionMessage = function({shield, msg, data}, sender, sendResponse) { + this.respondToWebExtensionMessage = function( + { shield, msg, data }, + sender, + sendResponse + ) { // shield: boolean, if present, request is for shield if (!shield) return true; const allowedMethods = ["endStudy", "telemetry", "info"]; if (!allowedMethods.includes(msg)) { - throw new Error(`respondToWebExtensionMessage: "${msg}" is not in allowed studyUtils methods: ${allowedMethods}`); + throw new Error( + `respondToWebExtensionMessage: "${ + msg + }" is not in allowed studyUtils methods: ${allowedMethods}` + ); } // handle async Promise.resolve(this[msg](data)).then( function(ans) { log.debug("respondingTo", msg, ans); sendResponse(ans); - }, + } // function error eventually ); return true; @@ -185,7 +214,10 @@ class StudyUtils { this.REASONS = REASONS; } throwIfNotSetup(name = "unknown") { - if (!this._isSetup) throw new Error(name + ": this method can't be used until `setup` is called"); + if (!this._isSetup) + throw new Error( + name + ": this method can't be used until `setup` is called" + ); } setup(config) { log = createLog("shield-study-utils", config.log.studyUtils.level); @@ -210,7 +242,9 @@ class StudyUtils { // Wait for the window to be opened await new Promise(resolve => setTimeout(resolve, 30000)); } - Services.wm.getMostRecentWindow("navigator:browser").gBrowser.addTab(url, params); + Services.wm + .getMostRecentWindow("navigator:browser") + .gBrowser.addTab(url, params); } async getTelemetryId() { return await getTelemetryId(); @@ -232,7 +266,10 @@ class StudyUtils { let fraction = rng; if (fraction === null) { const clientId = await this.getTelemetryId(); - fraction = await this.sample.hashFraction(this.config.study.studyName + clientId, 12); + fraction = await this.sample.hashFraction( + this.config.study.studyName + clientId, + 12 + ); } return this.sample.chooseWeighted(weightedVariations, fraction); } @@ -259,18 +296,29 @@ class StudyUtils { firstSeen() { log.debug(`firstSeen`); this.throwIfNotSetup("firstSeen uses telemetry."); - this._telemetry({study_state: "enter"}, "shield-study"); + this._telemetry({ study_state: "enter" }, "shield-study"); } setActive() { this.throwIfNotSetup("setActive uses telemetry."); const info = this.info(); - log.debug("marking TelemetryEnvironment", info.studyName, info.variation.name); - TelemetryEnvironment.setExperimentActive(info.studyName, info.variation.name); + log.debug( + "marking TelemetryEnvironment", + info.studyName, + info.variation.name + ); + TelemetryEnvironment.setExperimentActive( + info.studyName, + info.variation.name + ); } unsetActive() { this.throwIfNotSetup("unsetActive uses telemetry."); const info = this.info(); - log.debug("unmarking TelemetryEnvironment", info.studyName, info.variation.name); + log.debug( + "unmarking TelemetryEnvironment", + info.studyName, + info.variation.name + ); TelemetryEnvironment.setExperimentInactive(info.studyName); } uninstall(id) { @@ -281,15 +329,15 @@ class StudyUtils { log.debug(`about to uninstall ${id}`); AddonManager.getAddonByID(id, addon => addon.uninstall()); } - async startup({reason}) { + async startup({ reason }) { this.throwIfNotSetup("startup"); log.debug(`startup ${reason}`); this.setActive(); if (reason === REASONS.ADDON_INSTALL) { - this._telemetry({study_state: "installed"}, "shield-study"); + this._telemetry({ study_state: "installed" }, "shield-study"); } } - async endStudy({reason, fullname}) { + async endStudy({ reason, fullname }) { this.throwIfNotSetup("endStudy"); if (this._isEnding) { log.debug("endStudy, already ending!"); @@ -302,7 +350,7 @@ class StudyUtils { // TODO glind, think about race conditions for endings, ensure only one exit const ending = this.config.study.endings[reason]; if (ending) { - const {baseUrl, exactUrl} = ending; + const { baseUrl, exactUrl } = ending; if (exactUrl) { this.openTab(exactUrl); } else if (baseUrl) { @@ -321,14 +369,17 @@ class StudyUtils { case "ended-positive": case "ended-neutral": case "ended-negative": - this._telemetry({study_state: reason, fullname}, "shield-study"); + this._telemetry({ study_state: reason, fullname }, "shield-study"); break; default: - this._telemetry({study_state: "ended-neutral", study_state_fullname: reason}, "shield-study"); - // unless we know better TODO grl + this._telemetry( + { study_state: "ended-neutral", study_state_fullname: reason }, + "shield-study" + ); + // unless we know better TODO grl } // these are all exits - this._telemetry({study_state: "exit"}, "shield-study"); + this._telemetry({ study_state: "exit" }, "shield-study"); this.uninstall(); // TODO glind. should be controllable by arg? } @@ -355,14 +406,14 @@ class StudyUtils { this.throwIfNotSetup("_telemetry"); const info = this.info(); const payload = { - version: PACKET_VERSION, - study_name: info.studyName, - branch: info.variation.name, - addon_version: info.addon.version, + version: PACKET_VERSION, + study_name: info.studyName, + branch: info.variation.name, + addon_version: info.addon.version, shield_version: UTILS_VERSION, - type: bucket, + type: bucket, data, - testing: !this.telemetryConfig.removeTestingFlag, + testing: !this.telemetryConfig.removeTestingFlag, }; let validation; @@ -377,10 +428,10 @@ class StudyUtils { if (validation.errors.length) { const errorReport = { - "error_id": "jsonschema-validation", - "error_source": "addon", - "severity": "fatal", - "message": JSON.stringify(validation.errors), + error_id: "jsonschema-validation", + error_source: "addon", + severity: "fatal", + message: JSON.stringify(validation.errors), }; if (bucket === "shield-study-error") { // log: if it's a warn or error, it breaks jpm test @@ -393,7 +444,7 @@ class StudyUtils { log.debug(`telemetry: ${JSON.stringify(payload)}`); // FIXME marcrowo: addClientId makes the ping not appear in test? // seems like a problem with Telemetry, not the shield-study-utils library - const telOptions = {addClientId: true, addEnvironment: true}; + const telOptions = { addClientId: true, addEnvironment: true }; if (!this.telemetryConfig.send) { log.debug("NOT sending. `telemetryConfig.send` is false"); return false; @@ -416,7 +467,6 @@ class StudyUtils { setLoggingLevel(descriptor) { log.level = Log.Level[descriptor]; } - } function createLog(name, levelWord) { @@ -429,16 +479,18 @@ function createLog(name, levelWord) { } /** addon state change reasons */ const REASONS = { - APP_STARTUP: 1, // The application is starting up. - APP_SHUTDOWN: 2, // The application is shutting down. - ADDON_ENABLE: 3, // The add-on is being enabled. - ADDON_DISABLE: 4, // The add-on is being disabled. (Also sent during uninstallation) - ADDON_INSTALL: 5, // The add-on is being installed. - ADDON_UNINSTALL: 6, // The add-on is being uninstalled. - ADDON_UPGRADE: 7, // The add-on is being upgraded. - ADDON_DOWNGRADE: 8, // The add-on is being downgraded. + APP_STARTUP: 1, // The application is starting up. + APP_SHUTDOWN: 2, // The application is shutting down. + ADDON_ENABLE: 3, // The add-on is being enabled. + ADDON_DISABLE: 4, // The add-on is being disabled. (Also sent during uninstallation) + ADDON_INSTALL: 5, // The add-on is being installed. + ADDON_UNINSTALL: 6, // The add-on is being uninstalled. + ADDON_UPGRADE: 7, // The add-on is being upgraded. + ADDON_DOWNGRADE: 8, // The add-on is being downgraded. }; -for (const r in REASONS) { REASONS[REASONS[r]] = r; } +for (const r in REASONS) { + REASONS[REASONS[r]] = r; +} // Actually create the singleton. var studyUtils = new StudyUtils(); diff --git a/src/schema.studySetup.json b/src/schema.studySetup.json index a061b96..ef4616a 100644 --- a/src/schema.studySetup.json +++ b/src/schema.studySetup.json @@ -1,10 +1,10 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "title": "study setup", + "title": "study setup", "type": "object", "definitions": { "idString": { - "description": "between 1,100 chars, no spaces, unicode ok.", + "description": "between 1,100 chars, no spaces, unicode ok.", "type": "string", "pattern": "^\\S+$", "minLength": 1, @@ -27,7 +27,8 @@ }, "study_state_fullname": { "type": "string", - "description": "Second part of name of state, if any. Study-specific for study-defined endings." + "description": + "Second part of name of state, if any. Study-specific for study-defined endings." }, "url": { "type": "string" @@ -60,23 +61,13 @@ }, "onInvalid": { "type": "string", - "enum": [ - "throw", - "log" - ] + "enum": ["throw", "log"] } }, - "required": [ - "removeTestingFlag", - "send" - ] + "required": ["removeTestingFlag", "send"] } }, - "required": [ - "studyName", - "endings", - "telemetry" - ] + "required": ["studyName", "endings", "telemetry"] }, "addon": { "type": "object", @@ -90,14 +81,8 @@ "description": "Semantic version of the addon." } }, - "required": [ - "id", - "version" - ] + "required": ["id", "version"] } }, - "required": [ - "study", - "addon" - ] -} \ No newline at end of file + "required": ["study", "addon"] +} diff --git a/src/schema.webExtensionMsg.json b/src/schema.webExtensionMsg.json index 5c5814f..c5df40b 100644 --- a/src/schema.webExtensionMsg.json +++ b/src/schema.webExtensionMsg.json @@ -8,18 +8,11 @@ }, "msg": { "type": "string", - "enum": [ - "endStudy", - "telemetry", - "info" - ] + "enum": ["endStudy", "telemetry", "info"] }, "data": { "type": "object" } }, - "required": [ - "shield", - "msg" - ] + "required": ["shield", "msg"] } diff --git a/src/schema.weightedVariations.json b/src/schema.weightedVariations.json index f6b7cd5..3c7c809 100644 --- a/src/schema.weightedVariations.json +++ b/src/schema.weightedVariations.json @@ -9,10 +9,7 @@ "type": "number", "minimum": 0 }, - "required": [ - "name", - "weight" - ] + "required": ["name", "weight"] } }, "items": { diff --git a/src/schemas.js b/src/schemas.js index 27ecb3d..ef9056e 100644 --- a/src/schemas.js +++ b/src/schemas.js @@ -1,8 +1,8 @@ module.exports = { "shield-study": require("shield-study-schemas/schemas-client/shield-study.schema.json"), - "shield-study-addon": require("shield-study-schemas/schemas-client/shield-study-addon.schema.json"), - "shield-study-error": require("shield-study-schemas/schemas-client/shield-study-error.schema.json"), - "studySetup": require("./schema.studySetup.json"), - "webExtensionMsg": require("./schema.webExtensionMsg.json"), - "weightedVariations": require("./schema.weightedVariations.json"), + "shield-study-addon": require("shield-study-schemas/schemas-client/shield-study-addon.schema.json"), + "shield-study-error": require("shield-study-schemas/schemas-client/shield-study-error.schema.json"), + studySetup: require("./schema.studySetup.json"), + webExtensionMsg: require("./schema.webExtensionMsg.json"), + weightedVariations: require("./schema.weightedVariations.json"), }; diff --git a/test-addon/README.md b/test-addon/README.md index 40af997..deca6fa 100644 --- a/test-addon/README.md +++ b/test-addon/README.md @@ -1,3 +1,4 @@ This is an example add-on that is used to test the shield-study-utils library. -We include the library by obtaining a copy of dist/StudyUtils.jsm (created by `npm run dist` = `webpack`). +We include the library by obtaining a copy of dist/StudyUtils.jsm (created by +`npm run dist` = `webpack`). diff --git a/test-addon/bootstrap.js b/test-addon/bootstrap.js index 4bccfe5..062b13d 100644 --- a/test-addon/bootstrap.js +++ b/test-addon/bootstrap.js @@ -1,6 +1,10 @@ const { classes: Cc, interfaces: Ci, utils: Cu } = Components; Cu.import("resource://gre/modules/XPCOMUtils.jsm"); -XPCOMUtils.defineLazyModuleGetter(this, "studyUtils", "resource://test-addon/StudyUtils.jsm"); +XPCOMUtils.defineLazyModuleGetter( + this, + "studyUtils", + "resource://test-addon/StudyUtils.jsm" +); this.install = function(data, reason) {}; diff --git a/test-addon/utils.jsm b/test-addon/utils.jsm index 1472083..c4e7ffb 100644 --- a/test-addon/utils.jsm +++ b/test-addon/utils.jsm @@ -1,6 +1,9 @@ /* eslint no-unused-vars: "off" */ -const { studyUtils } = Components.utils.import("resource://test-addon/StudyUtils.jsm", {}); +const { studyUtils } = Components.utils.import( + "resource://test-addon/StudyUtils.jsm", + {} +); Components.utils.import("resource://gre/modules/TelemetryArchive.jsm"); Components.utils.import("resource://gre/modules/Console.jsm"); @@ -11,8 +14,8 @@ function fakeSetup() { study: { studyName: "shield-utils-test", endings: { - "expired": { - "baseUrl": "http://www.example.com/?reason=expired", + expired: { + baseUrl: "http://www.example.com/?reason=expired", }, }, telemetry: { send: true, removeTestingFlag: false }, @@ -23,7 +26,7 @@ function fakeSetup() { level: "Warn", }, }, - addon: {id: "1", version: "1"}, + addon: { id: "1", version: "1" }, }); studyUtils.setVariation({ name: "puppers", weight: "2" }); } @@ -36,6 +39,8 @@ async function getMostRecentPingsByType(type) { const filteredPings = pings.filter(p => p.type === type); filteredPings.sort((a, b) => b.timestampCreated - a.timestampCreated); - const pingData = filteredPings.map(ping => TelemetryArchive.promiseArchivedPingById(ping.id)); + const pingData = filteredPings.map(ping => + TelemetryArchive.promiseArchivedPingById(ping.id) + ); return Promise.all(pingData); } diff --git a/test/shield_utils_test.js b/test/shield_utils_test.js index d54b2ec..2627946 100644 --- a/test/shield_utils_test.js +++ b/test/shield_utils_test.js @@ -14,7 +14,7 @@ describe("Shield Study Utils Functional Tests", function() { let driver; - before(async() => { + before(async () => { driver = await utils.promiseSetupDriver(); // install the addon (note: returns addon id) await utils.installAddon(driver); @@ -22,56 +22,84 @@ describe("Shield Study Utils Functional Tests", function() { after(() => driver.quit()); - it("should return the correct variation", async() => { - const variation = await driver.executeAsyncScript(async(callback) => { - const { studyUtils } = Components.utils.import("resource://test-addon/StudyUtils.jsm", {}); + it("should return the correct variation", async () => { + const variation = await driver.executeAsyncScript(async callback => { + const { studyUtils } = Components.utils.import( + "resource://test-addon/StudyUtils.jsm", + {} + ); // TODO move this to a Config.jsm file const studyConfig = { studyName: "shieldStudyUtilsTest", - "weightedVariations": [ - {"name": "control", - "weight": 1}, - {"name": "kittens", - "weight": 1.5}, - {"name": "puppers", - "weight": 2}, // we want more puppers in our sample + weightedVariations: [ + { + name: "control", + weight: 1, + }, + { + name: "kittens", + weight: 1.5, + }, + { + name: "puppers", + weight: 2, + }, // we want more puppers in our sample ], }; const sample = studyUtils.sample; const hashFraction = await sample.hashFraction("test"); - const chosenVariation = await sample.chooseWeighted(studyConfig.weightedVariations, hashFraction); + const chosenVariation = await sample.chooseWeighted( + studyConfig.weightedVariations, + hashFraction + ); callback(chosenVariation); }); assert(variation.name === "puppers"); }); - it("telemetry should be working", async() => { - const shieldTelemetryPing = await driver.executeAsyncScript(async(callback) => { - const { fakeSetup, getMostRecentPingsByType } = Components.utils.import("resource://test-addon/utils.jsm", {}); - const { studyUtils } = Components.utils.import("resource://test-addon/StudyUtils.jsm", {}); - Components.utils.import("resource://gre/modules/TelemetryArchive.jsm"); + it("telemetry should be working", async () => { + const shieldTelemetryPing = await driver.executeAsyncScript( + async callback => { + const { fakeSetup, getMostRecentPingsByType } = Components.utils.import( + "resource://test-addon/utils.jsm", + {} + ); + const { studyUtils } = Components.utils.import( + "resource://test-addon/StudyUtils.jsm", + {} + ); + Components.utils.import("resource://gre/modules/TelemetryArchive.jsm"); - fakeSetup(); + fakeSetup(); - await studyUtils.telemetry({ "foo": "bar" }); + await studyUtils.telemetry({ foo: "bar" }); - // TODO Fix this hackiness; caused by addClientId option in submitExternalPing - // The ping seems to be sending (appears in about:telemetry) but does not appear - // in the pings array - await new Promise(resolve => setTimeout(resolve, 1000)); + // TODO Fix this hackiness; caused by addClientId option in submitExternalPing + // The ping seems to be sending (appears in about:telemetry) but does not appear + // in the pings array + await new Promise(resolve => setTimeout(resolve, 1000)); - const shieldPings = await getMostRecentPingsByType("shield-study-addon"); - callback(shieldPings[0]); - }); + const shieldPings = await getMostRecentPingsByType( + "shield-study-addon" + ); + callback(shieldPings[0]); + } + ); assert(shieldTelemetryPing.payload.data.attributes.foo === "bar"); }); - describe("test the library's \"startup\" process", function() { - it("should send the correct ping on first seen", async() => { - const firstSeenPing = await driver.executeAsyncScript(async(callback) => { - const { fakeSetup, getMostRecentPingsByType } = Components.utils.import("resource://test-addon/utils.jsm", {}); - const { studyUtils } = Components.utils.import("resource://test-addon/StudyUtils.jsm", {}); + describe('test the library\'s "startup" process', function() { + it("should send the correct ping on first seen", async () => { + const firstSeenPing = await driver.executeAsyncScript(async callback => { + const { fakeSetup, getMostRecentPingsByType } = Components.utils.import( + "resource://test-addon/utils.jsm", + {} + ); + const { studyUtils } = Components.utils.import( + "resource://test-addon/StudyUtils.jsm", + {} + ); Components.utils.import("resource://gre/modules/TelemetryArchive.jsm"); fakeSetup(); @@ -84,30 +112,46 @@ describe("Shield Study Utils Functional Tests", function() { assert(firstSeenPing.payload.data.study_state === "enter"); }); - it("should set the experiment to active in Telemetry", async() => { - const activeExperiments = await driver.executeAsyncScript(async(callback) => { - const { fakeSetup } = Components.utils.import("resource://test-addon/utils.jsm", {}); - const { studyUtils } = Components.utils.import("resource://test-addon/StudyUtils.jsm", {}); - Components.utils.import("resource://gre/modules/TelemetryEnvironment.jsm"); - - fakeSetup(); - - studyUtils.setActive(); - - callback(TelemetryEnvironment.getActiveExperiments()); - }); + it("should set the experiment to active in Telemetry", async () => { + const activeExperiments = await driver.executeAsyncScript( + async callback => { + const { fakeSetup } = Components.utils.import( + "resource://test-addon/utils.jsm", + {} + ); + const { studyUtils } = Components.utils.import( + "resource://test-addon/StudyUtils.jsm", + {} + ); + Components.utils.import( + "resource://gre/modules/TelemetryEnvironment.jsm" + ); + + fakeSetup(); + + studyUtils.setActive(); + + callback(TelemetryEnvironment.getActiveExperiments()); + } + ); assert(activeExperiments.hasOwnProperty("shield-utils-test")); }); - it("should send the correct telemetry ping on first install", async() => { - const installedPing = await driver.executeAsyncScript(async(callback) => { - const { fakeSetup, getMostRecentPingsByType } = Components.utils.import("resource://test-addon/utils.jsm", {}); - const { studyUtils } = Components.utils.import("resource://test-addon/StudyUtils.jsm", {}); + it("should send the correct telemetry ping on first install", async () => { + const installedPing = await driver.executeAsyncScript(async callback => { + const { fakeSetup, getMostRecentPingsByType } = Components.utils.import( + "resource://test-addon/utils.jsm", + {} + ); + const { studyUtils } = Components.utils.import( + "resource://test-addon/StudyUtils.jsm", + {} + ); Components.utils.import("resource://gre/modules/TelemetryArchive.jsm"); fakeSetup(); - await studyUtils.startup({reason: 5}); // ADDON_INSTALL = 5 + await studyUtils.startup({ reason: 5 }); // ADDON_INSTALL = 5 const studyPings = await getMostRecentPingsByType("shield-study"); callback(studyPings[0]); @@ -117,37 +161,50 @@ describe("Shield Study Utils Functional Tests", function() { }); describe("test the library's endStudy() function", function() { - before(async() => { - await driver.executeAsyncScript(async(callback) => { - const { fakeSetup } = Components.utils.import("resource://test-addon/utils.jsm", {}); - const { studyUtils } = Components.utils.import("resource://test-addon/StudyUtils.jsm", {}); + before(async () => { + await driver.executeAsyncScript(async callback => { + const { fakeSetup } = Components.utils.import( + "resource://test-addon/utils.jsm", + {} + ); + const { studyUtils } = Components.utils.import( + "resource://test-addon/StudyUtils.jsm", + {} + ); fakeSetup(); // TODO add tests for other reasons (?) - await studyUtils.endStudy({reason: "expired", fullname: "TEST_FULLNAME"}); + await studyUtils.endStudy({ + reason: "expired", + fullname: "TEST_FULLNAME", + }); callback(); }); }); - it("should set the experiment as inactive", async() => { - const activeExperiments = await driver.executeAsyncScript(async(callback) => { - Components.utils.import("resource://gre/modules/TelemetryEnvironment.jsm"); - callback(TelemetryEnvironment.getActiveExperiments()); - }); + it("should set the experiment as inactive", async () => { + const activeExperiments = await driver.executeAsyncScript( + async callback => { + Components.utils.import( + "resource://gre/modules/TelemetryEnvironment.jsm" + ); + callback(TelemetryEnvironment.getActiveExperiments()); + } + ); assert(!activeExperiments.hasOwnProperty("shield-utils-test")); }); describe("test the opening of a URL at the end of the study", function() { - it("should open a new tab", async() => { - const newTabOpened = await driver.wait(async() => { + it("should open a new tab", async () => { + const newTabOpened = await driver.wait(async () => { const handles = await driver.getAllWindowHandles(); return handles.length === 2; // opened a new tab }, 3000); assert(newTabOpened); }); - it("should open a new tab to the correct URL", async() => { + it("should open a new tab to the correct URL", async () => { const currentHandle = await driver.getWindowHandle(); driver.setContext(Context.CONTENT); // Find the new window handle. @@ -158,27 +215,35 @@ describe("Shield Study Utils Functional Tests", function() { newWindowHandle = handle; } } - const correctURLOpened = await driver.wait(async() => { + const correctURLOpened = await driver.wait(async () => { await driver.switchTo().window(newWindowHandle); const currentURL = await driver.getCurrentUrl(); - return currentURL.startsWith("http://www.example.com/?reason=expired"); + return currentURL.startsWith( + "http://www.example.com/?reason=expired" + ); }); assert(correctURLOpened); }); }); - it("should send the correct reason telemetry", async() => { - const pings = await driver.executeAsyncScript(async(callback) => { - const { getMostRecentPingsByType } = Components.utils.import("resource://test-addon/utils.jsm", {}); + it("should send the correct reason telemetry", async () => { + const pings = await driver.executeAsyncScript(async callback => { + const { getMostRecentPingsByType } = Components.utils.import( + "resource://test-addon/utils.jsm", + {} + ); const studyPings = await getMostRecentPingsByType("shield-study"); callback(studyPings[1]); // ping before the most recent ping }); assert(pings.payload.data.study_state === "expired"); }); - it("should send the uninstall telemetry", async() => { - const pings = await driver.executeAsyncScript(async(callback) => { - const { getMostRecentPingsByType } = Components.utils.import("resource://test-addon/utils.jsm", {}); + it("should send the uninstall telemetry", async () => { + const pings = await driver.executeAsyncScript(async callback => { + const { getMostRecentPingsByType } = Components.utils.import( + "resource://test-addon/utils.jsm", + {} + ); const studyPings = await getMostRecentPingsByType("shield-study"); callback(studyPings[0]); }); diff --git a/test/utils.js b/test/utils.js index f06a237..4fa4b85 100644 --- a/test/utils.js +++ b/test/utils.js @@ -27,7 +27,7 @@ const FIREFOX_PREFERENCES = { "devtools.chrome.enabled": true, "devtools.debugger.remote-enabled": true, "devtools.debugger.prompt-connection": false, - "general.warnOnAboutConfig": false + "general.warnOnAboutConfig": false, }; // useful if we need to test on a specific version of Firefox @@ -44,10 +44,10 @@ async function promiseActualBinary(binary) { } } -module.exports.promiseSetupDriver = async() => { +module.exports.promiseSetupDriver = async () => { const profile = new firefox.Profile(); - Object.keys(FIREFOX_PREFERENCES).forEach((key) => { + Object.keys(FIREFOX_PREFERENCES).forEach(key => { profile.setPreference(key, FIREFOX_PREFERENCES[key]); }); @@ -58,30 +58,44 @@ module.exports.promiseSetupDriver = async() => { .forBrowser("firefox") .setFirefoxOptions(options); - const binaryLocation = await promiseActualBinary(process.env.FIREFOX_BINARY || "firefox"); + const binaryLocation = await promiseActualBinary( + process.env.FIREFOX_BINARY || "firefox" + ); await options.setBinary(new firefox.Binary(binaryLocation)); const driver = await builder.build(); driver.setContext(Context.CHROME); return driver; }; -module.exports.installAddon = async(driver) => { +module.exports.installAddon = async driver => { // references: // https://bugzilla.mozilla.org/show_bug.cgi?id=1298025 // https://github.com/mozilla/geckodriver/releases/tag/v0.17.0 const fileLocation = path.join(process.cwd(), process.env.XPI_NAME); const executor = driver.getExecutor(); - executor.defineCommand("installAddon", "POST", "/session/:sessionId/moz/addon/install"); + executor.defineCommand( + "installAddon", + "POST", + "/session/:sessionId/moz/addon/install" + ); const installCmd = new cmd.Command("installAddon"); const session = await driver.getSession(); - installCmd.setParameters({ sessionId: session.getId(), path: fileLocation, temporary: true }); + installCmd.setParameters({ + sessionId: session.getId(), + path: fileLocation, + temporary: true, + }); return executor.execute(installCmd); }; -module.exports.uninstallAddon = async(driver, id) => { +module.exports.uninstallAddon = async (driver, id) => { const executor = driver.getExecutor(); - executor.defineCommand("uninstallAddon", "POST", "/session/:sessionId/moz/addon/uninstall"); + executor.defineCommand( + "uninstallAddon", + "POST", + "/session/:sessionId/moz/addon/uninstall" + ); const uninstallCmd = new cmd.Command("uninstallAddon"); const session = await driver.getSession(); diff --git a/webpack.config.js b/webpack.config.js index c888164..ccc134a 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,11 +1,11 @@ -var path = require('path'); -const webpack = require('webpack'); //to access built-in plugins +var path = require("path"); +const webpack = require("webpack"); //to access built-in plugins module.exports = { - entry: './src/StudyUtils.in.jsm', + entry: "./src/StudyUtils.in.jsm", output: { - path: path.resolve(__dirname, 'dist'), - filename: 'StudyUtils.jsm', - libraryTarget: 'this' // Possible value - amd, commonjs, commonjs2, commonjs-module, this, var - } + path: path.resolve(__dirname, "dist"), + filename: "StudyUtils.jsm", + libraryTarget: "this", // Possible value - amd, commonjs, commonjs2, commonjs-module, this, var + }, };