[wip] data crunching in wasm

tools/rum/cruncher.js

@@ -3,6 +3,10 @@
  * filtering, aggregating, and summarizing the data.
  */
 /* eslint-disable max-classes-per-file */
+// eslint-disable-next-line camelcase
+import init, { t_test } from './enigma.js';
+
+await init();
 /**
  * @typedef {Object} RawEvent - a raw RUM event
  * @property {string} checkpoint - the name of the event that happened
@@ -48,24 +52,30 @@
  * @returns {Bundle} a bundle with additional properties
  */
 export function addCalculatedProps(bundle) {
-  bundle.events.forEach((e) => {
+  for (let i = 0; i < bundle.events.length; i += 1) {
+    const e = bundle.events[i];
     if (e.checkpoint === 'enter') {
       bundle.visit = true;
       if (e.source === '') e.source = '(direct)';
+      break;
     }
     if (e.checkpoint === 'cwv-inp') {
       bundle.cwvINP = e.value;
+      break;
     }
     if (e.checkpoint === 'cwv-lcp') {
       bundle.cwvLCP = Math.max(e.value || 0, bundle.cwvLCP || 0);
+      break;
     }
     if (e.checkpoint === 'cwv-cls') {
       bundle.cwvCLS = Math.max(e.value || 0, bundle.cwvCLS || 0);
+      break;
     }
     if (e.checkpoint === 'cwv-ttfb') {
       bundle.cwvTTFB = e.value;
+      break;
     }
-  });
+  }
   return bundle;
 }
 
@@ -234,6 +244,26 @@ function erf(x1) {
   return sign * y;
 }
 
+function compute(data) {
+  let sum = 0;
+  let variance = 0;
+
+  // Calculate sum
+  for (let i = 0; i < data.length; i += 1) {
+    sum += data[i];
+  }
+
+  const mean = sum / data.length;
+
+  // Calculate variance
+  for (let i = 0; i < data.length; i += 1) {
+    variance += (data[i] - mean) ** 2;
+  }
+
+  variance /= data.length;
+
+  return { mean, variance };
+}
 /**
  * Performs a significance test on the data. The test assumes
  * that the data is normally distributed and will calculate
@@ -243,18 +273,27 @@ function erf(x1) {
  * @returns {number} the p-value, a value between 0 and 1
  */
 export function tTest(left, right) {
-  const meanLeft = left.reduce((acc, value) => acc + value, 0) / left.length;
-  const meanRight = right.reduce((acc, value) => acc + value, 0) / right.length;
-  const varianceLeft = left.reduce((acc, value) => acc + (value - meanLeft) ** 2, 0) / left.length;
-  const varianceRight = right
-    .reduce((acc, value) => acc + (value - meanRight) ** 2, 0) / right.length;
+  const { mean: meanLeft, variance: varianceLeft } = compute(left);
+  const { mean: meanRight, variance: varianceRight } = compute(right);
   const pooledVariance = (varianceLeft + varianceRight) / 2;
   const tValue = (meanLeft - meanRight) / Math
     .sqrt(pooledVariance * (1 / left.length + 1 / right.length));
   const p = 1 - (0.5 + 0.5 * erf(tValue / Math.sqrt(2)));
   return p;
 }
 
+/**
+ * Performs a significance test on the data. The test assumes
+ * that the data is normally distributed and will calculate
+ * the p-value for the difference between the two data sets.
+ * @param {number[]} left the first data set
+ * @param {number[]} right the second data set
+ * @returns {number} the p-value, a value between 0 and 1
+ */
+export function tTestWasm(left, right) {
+  return t_test(new Uint32Array(left), new Uint32Array(right));
+}
+
 class Facet {
   constructor(parent, value, name) {
     this.parent = parent;

tools/rum/elements/list-facet.js

@@ -1,13 +1,13 @@
 import {
   computeConversionRate, escapeHTML, scoreCWV, toHumanReadable,
 } from '../utils.js';
-import { tTest, zTestTwoProportions } from '../cruncher.js';
+import { tTestWasm, zTestTwoProportions } from '../cruncher.js';
 
 async function addSignificanceFlag(element, metric, baseline) {
   let p = 1;
   if (Array.isArray(metric.values) && Array.isArray(baseline.values)) {
     // for two arrays of values, we use a t-test
-    p = tTest(metric.values, baseline.values);
+    p = tTestWasm(metric.values, baseline.values);
   } else if (
     typeof metric.total === 'number'
     && typeof metric.conversions === 'number'

tools/rum/enigma.js

@@ -0,0 +1,129 @@
+let wasm;
+
+let cachedUint32Memory0 = null;
+
+function getUint32Memory0() {
+  if (cachedUint32Memory0 === null || cachedUint32Memory0.byteLength === 0) {
+    cachedUint32Memory0 = new Uint32Array(wasm.memory.buffer);
+  }
+  return cachedUint32Memory0;
+}
+
+let WASM_VECTOR_LEN = 0;
+
+function passArray32ToWasm0(arg, malloc) {
+  // eslint-disable-next-line no-bitwise
+  const ptr = malloc(arg.length * 4, 4) >>> 0;
+  getUint32Memory0().set(arg, ptr / 4);
+  WASM_VECTOR_LEN = arg.length;
+  return ptr;
+}
+/**
+* @param {Uint32Array} left
+* @param {Uint32Array} right
+* @returns {number}
+*/
+// eslint-disable-next-line camelcase
+export function t_test(left, right) {
+  // eslint-disable-next-line no-underscore-dangle
+  const ptr0 = passArray32ToWasm0(left, wasm.__wbindgen_malloc);
+  const len0 = WASM_VECTOR_LEN;
+  // eslint-disable-next-line no-underscore-dangle
+  const ptr1 = passArray32ToWasm0(right, wasm.__wbindgen_malloc);
+  const len1 = WASM_VECTOR_LEN;
+  const ret = wasm.t_test(ptr0, len0, ptr1, len1);
+  return ret;
+}
+
+// eslint-disable-next-line camelcase,no-underscore-dangle
+async function __wbg_load(module, imports) {
+  if (typeof Response === 'function' && module instanceof Response) {
+    if (typeof WebAssembly.instantiateStreaming === 'function') {
+      try {
+        return await WebAssembly.instantiateStreaming(module, imports);
+      } catch (e) {
+        if (module.headers.get('Content-Type') !== 'application/wasm') {
+          console.warn('`WebAssembly.instantiateStreaming` failed because your server does not serve wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n', e);
+        } else {
+          throw e;
+        }
+      }
+    }
+
+    const bytes = await module.arrayBuffer();
+    // eslint-disable-next-line no-return-await
+    return await WebAssembly.instantiate(bytes, imports);
+  }
+  const instance = await WebAssembly.instantiate(module, imports);
+
+  if (instance instanceof WebAssembly.Instance) {
+    return { instance, module };
+  }
+  return instance;
+}
+
+// eslint-disable-next-line camelcase,no-underscore-dangle
+function __wbg_get_imports() {
+  const imports = {};
+  imports.wbg = {};
+
+  return imports;
+}
+
+// eslint-disable-next-line no-underscore-dangle,camelcase,no-unused-vars
+function __wbg_init_memory(imports, maybe_memory) {
+
+}
+
+// eslint-disable-next-line camelcase,no-underscore-dangle
+function __wbg_finalize_init(instance, module) {
+  wasm = instance.exports;
+  // eslint-disable-next-line camelcase,no-underscore-dangle,no-use-before-define
+  __wbg_init.__wbindgen_wasm_module = module;
+  cachedUint32Memory0 = null;
+
+  return wasm;
+}
+
+function initSync(module) {
+  if (wasm !== undefined) return wasm;
+
+  const imports = __wbg_get_imports();
+
+  __wbg_init_memory(imports);
+
+  if (!(module instanceof WebAssembly.Module)) {
+    // eslint-disable-next-line no-param-reassign
+    module = new WebAssembly.Module(module);
+  }
+
+  const instance = new WebAssembly.Instance(module, imports);
+
+  return __wbg_finalize_init(instance, module);
+}
+
+// eslint-disable-next-line no-underscore-dangle,camelcase
+async function __wbg_init(input) {
+  if (wasm !== undefined) return wasm;
+
+  if (typeof input === 'undefined') {
+    // eslint-disable-next-line no-param-reassign
+    input = new URL('enigma_bg.wasm', import.meta.url);
+  }
+  const imports = __wbg_get_imports();
+
+  if (typeof input === 'string' || (typeof Request === 'function' && input instanceof Request) || (typeof URL === 'function' && input instanceof URL)) {
+    // eslint-disable-next-line no-param-reassign
+    input = fetch(input);
+  }
+
+  __wbg_init_memory(imports);
+
+  const { instance, module } = await __wbg_load(await input, imports);
+
+  return __wbg_finalize_init(instance, module);
+}
+
+export { initSync };
+// eslint-disable-next-line camelcase
+export default __wbg_init;

tools/rum/loader.js

@@ -7,14 +7,19 @@ import { addCalculatedProps } from './cruncher.js';
 
 export default class DataLoader {
   constructor() {
-    this.cache = new Map();
     this.API_ENDPOINT = 'https://rum.fastly-aem.page/bundles';
     this.DOMAIN = 'www.thinktanked.org';
     this.DOMAIN_KEY = '';
   }
 
-  flush() {
-    this.cache.clear();
+  async init() {
+    this.cache = await caches.open('bundles');
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  async flush() {
+    // await caches.delete('bundles');
+    // this.cache = await caches.open('bundles');
   }
 
   set domainKey(key) {
@@ -46,13 +51,24 @@ export default class DataLoader {
     return u.toString();
   }
 
+  async fetchBundles(apiRequestURL) {
+    const cacheResponse = await this.cache.match(apiRequestURL);
+    if (cacheResponse) {
+      return cacheResponse;
+    }
+    const networkResponse = await fetch(apiRequestURL);
+    const networkResponseClone = networkResponse.clone();
+    this.cache.put(apiRequestURL, networkResponseClone);
+    return networkResponse;
+  }
+
   async fetchUTCMonth(utcISOString) {
     const [date] = utcISOString.split('T');
     const dateSplits = date.split('-');
     dateSplits.pop();
     const monthPath = dateSplits.join('/');
     const apiRequestURL = this.apiURL(monthPath);
-    const resp = await fetch(apiRequestURL);
+    const resp = await this.fetchBundles(apiRequestURL);
     const json = await resp.json();
     const { rumBundles } = json;
     rumBundles.forEach((bundle) => addCalculatedProps(bundle));
@@ -63,7 +79,7 @@ export default class DataLoader {
     const [date] = utcISOString.split('T');
     const datePath = date.split('-').join('/');
     const apiRequestURL = this.apiURL(datePath);
-    const resp = await fetch(apiRequestURL);
+    const resp = await this.fetchBundles(apiRequestURL);
     const json = await resp.json();
     const { rumBundles } = json;
     rumBundles.forEach((bundle) => addCalculatedProps(bundle));
@@ -75,7 +91,7 @@ export default class DataLoader {
     const datePath = date.split('-').join('/');
     const hour = time.split(':')[0];
     const apiRequestURL = this.apiURL(datePath, hour);
-    const resp = await fetch(apiRequestURL);
+    const resp = await this.fetchBundles(apiRequestURL);
     const json = await resp.json();
     const { rumBundles } = json;
     rumBundles.forEach((bundle) => addCalculatedProps(bundle));

tools/rum/package.json

@@ -1,8 +1,17 @@
 {
-  "name": "@adobe/aem-rum-explorer",
+  "name": "enigma",
+  "collaborators": [
+    "Andrei Kalfas <[email protected]>"
+  ],
   "version": "0.1.0",
-  "scripts": {
-    "test": "node --test --experimental-test-coverage --test-reporter=lcov --test-reporter-destination=../../lcov.info --test-reporter=spec --test-reporter-destination=stdout --test-reporter=junit --test-reporter-destination=../../junit.xml"
-  },
-  "type": "module"
+  "files": [
+    "enigma_bg.wasm",
+    "enigma.js",
+    "enigma.d.ts"
+  ],
+  "module": "enigma.js",
+  "types": "enigma.d.ts",
+  "sideEffects": [
+    "./snippets/*"
+  ]
 }

tools/rum/slicer.js

@@ -24,6 +24,7 @@ const elems = {};
 const dataChunks = new DataChunks();
 
 const loader = new DataLoader();
+await loader.init();
 loader.apiEndpoint = API_ENDPOINT;
 
 const herochart = new window.slicer.Chart(dataChunks, elems);

tools/rum/utils.js

@@ -195,11 +195,14 @@ export function parseSearchParams(params, filterFn, transformFn) {
       return acc;
     }, {});
 }
+const cached = {};
 export function parseConversionSpec() {
+  if (cached.conversionSpec) return cached.conversionSpec;
   const params = new URL(window.location).searchParams;
   const transform = ([key, value]) => [key.replace('conversion.', ''), value];
   const filter = ([key]) => (key.startsWith('conversion.'));
-  return parseSearchParams(params, filter, transform);
+  cached.conversionSpec = parseSearchParams(params, filter, transform);
+  return cached.conversionSpec;
 }
 
 /**