Replace collate option to use partition: "measurement"

CHANGELOG.md

@@ -7,8 +7,8 @@ project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
-- Add `collate` option for collating results tables by measurement (when
-  multiple measurements are in use).
+- Add `partition: "measurement"` option for collating results tables by
+  measurement (when multiple measurements are in use).
 
 ## [0.7.0] 2022-07-15
 

README.md

@@ -857,7 +857,7 @@ tach http://example.com
 | `--package-version` / `-p`  | _(none)_                                | Specify an NPM package version to swap in ([details](#swap-npm-dependencies))                                                                                      |
 | `--browser` / `-b`          | `chrome`                                | Which browsers to launch in automatic mode, comma-delimited (chrome, firefox, safari, edge, ie) ([details](#browsers))                                             |
 | `--window-size`             | `1024,768`                              | "width,height" in pixels of the browser windows that will be created                                                                                               |
-| `--sample-size` / `-n`      | `50`                                    | Minimum number of times to run each benchmark ([details](#minimum-sample-size))                                                                                            |
+| `--sample-size` / `-n`      | `50`                                    | Minimum number of times to run each benchmark ([details](#minimum-sample-size))                                                                                    |
 | `--auto-sample-conditions`  | `0%`                                    | The degrees of difference to try and resolve when auto-sampling ("N%" or "Nms", comma-delimited) ([details](#auto-sample-conditions))                              |
 | `--timeout`                 | `3`                                     | The maximum number of minutes to spend auto-sampling ([details](#auto-sample))                                                                                     |
 | `--measure`                 | `callback`                              | Which time interval to measure (`callback`, `global`, `fcp`) ([details](#measurement-modes))                                                                       |
@@ -872,3 +872,4 @@ tach http://example.com
 | `--trace`                   | `false`                                 | Enable performance tracing ([details](#performance-traces))                                                                                                        |
 | `--trace-log-dir`           | `${cwd}/logs`                           | The directory to put tracing log files. Defaults to `${cwd}/logs`.                                                                                                 |
 | `--trace-cat`               | [default categories](./src/defaults.ts) | The tracing categories to record. Should be a string of comma-separated category names                                                                             |
+| `--partition`               | _(none)_                                | Use `"measurement"` to partition a single large results table into multiple tables for each [measurement](#measurement-modes).                                     |

config.schema.json

@@ -476,17 +476,17 @@
             "minItems": 1,
             "type": "array"
         },
-        "collate": {
-            "description": "Whether to collate result table by measurement (when multiple measurements\nare in use)",
-            "type": "boolean"
-        },
         "horizons": {
             "description": "Deprecated alias for autoSampleConditions.",
             "items": {
                 "type": "string"
             },
             "type": "array"
         },
+        "partition": {
+            "description": "What to partition the results table by. Use \"measurement\" when multiple\nmeasurements are in use and you want multiple smaller results tables.",
+            "type": "string"
+        },
         "resolveBareModules": {
             "description": "Whether to automatically convert ES module imports with bare module\nspecifiers to paths.",
             "type": "boolean"

src/config.ts

@@ -36,7 +36,7 @@ export interface Config {
   forceCleanNpmInstall: boolean;
   csvFileStats: string;
   csvFileRaw: string;
-  collate?: boolean;
+  partition?: 'measurement';
 }
 
 export async function makeConfig(opts: Opts): Promise<Config> {
@@ -86,8 +86,8 @@ export async function makeConfig(opts: Opts): Promise<Config> {
     if (opts['window-size'] !== undefined) {
       throw new Error('--window-size cannot be specified when using --config');
     }
-    if (opts['collate'] !== undefined) {
-      throw new Error('--collate cannot be specified when using --config');
+    if (opts['partition'] !== undefined) {
+      throw new Error('--partition cannot be specified when using --config');
     }
     const rawConfigObj = await fsExtra.readJson(opts.config);
     const validatedConfigObj = await parseConfigFile(rawConfigObj, opts.config);
@@ -176,7 +176,7 @@ export function applyDefaults(partial: Partial<Config>): Config {
         : defaults.resolveBareModules,
     root: partial.root !== undefined ? partial.root : defaults.root,
     timeout: partial.timeout !== undefined ? partial.timeout : defaults.timeout,
-    collate: partial.collate !== undefined ? partial.collate : defaults.collate,
+    partition: partial.partition,
   };
 }
 

src/configfile.ts

@@ -63,10 +63,10 @@ export interface ConfigFile {
   horizons?: string[];
 
   /**
-   * Whether to collate result table by measurement (when multiple measurements
-   * are in use)
+   * What to partition the results table by. Use "measurement" when multiple
+   * measurements are in use and you want multiple smaller results tables.
    */
-  collate?: boolean;
+  partition?: string;
 
   /**
    * Benchmarks to run.
@@ -386,6 +386,14 @@ export async function parseConfigFile(
     validated.autoSampleConditions = validated.horizons;
   }
 
+  if (validated.partition !== undefined) {
+    if (validated.partition !== 'measurement') {
+      throw new Error(
+        `The "partition" setting only accepts "measurement" as an option.`
+      );
+    }
+  }
+
   return {
     root,
     sampleSize: validated.sampleSize,
@@ -396,7 +404,7 @@ export async function parseConfigFile(
         : undefined,
     benchmarks,
     resolveBareModules: validated.resolveBareModules,
-    collate: validated.collate,
+    partition: validated.partition,
   };
 }
 

src/defaults.ts

@@ -20,7 +20,7 @@ export const mode = 'automatic';
 export const resolveBareModules = true;
 export const forceCleanNpmInstall = false;
 export const measurementExpression = 'window.tachometerResult';
-export const collate = false;
+export const partition = '';
 export const traceLogDir = path.join(process.cwd(), 'logs');
 export const traceCategories = [
   'blink',

src/flags.ts

@@ -230,11 +230,12 @@ export const optDefs: commandLineUsage.OptionDefinition[] = [
     defaultValue: defaults.traceCategories.join(','),
   },
   {
-    name: 'collate',
+    name: 'partition',
     description:
-        `Whether to collate result table by measurement (when multiple ` +
-        `measurements are in use)`,
-    type: Boolean,
+      `What to partition the results table by. Use "measurement" ` +
+      `when multiple measurements are in use and you want multiple ` +
+      `smaller results tables.`,
+    type: String,
   },
 ];
 
@@ -266,7 +267,7 @@ export interface Opts {
   trace: boolean;
   'trace-log-dir': string;
   'trace-cat': string;
-  collate: boolean;
+  partition: string;
 
   // Extra arguments not associated with a flag are put here. These are our
   // benchmark names/URLs.

src/format.ts

@@ -103,14 +103,17 @@ export function automaticResultTable(results: ResultStats[]): AutomaticResults {
   return {fixed: fixedTable, unfixed: unfixedTable};
 }
 
-export function collatedResultTables(results: ResultStatsWithDifferences[]) {
+export function partitionResultTableByMeasurement(
+  results: ResultStatsWithDifferences[]
+) {
   const collated: {[index: string]: ResultStatsWithDifferences[]} = {};
   results.forEach((result) => {
     const meas = measurementName(result.result.measurement);
     (collated[meas] || (collated[meas] = [])).push({
       ...result,
       differences: result.differences.filter(
-          (_, i) => measurementName(results[i].result.measurement) === meas)
+        (_, i) => measurementName(results[i].result.measurement) === meas
+      ),
     });
   });
   const tables: AutomaticResults[] = [];

src/runner.ts

@@ -33,7 +33,7 @@ import {
   automaticResultTable,
   spinner,
   benchmarkOneLiner,
-  collatedResultTables,
+  partitionResultTableByMeasurement,
 } from './format.js';
 import {Config} from './config.js';
 import * as github from './github.js';
@@ -421,8 +421,10 @@ export class Runner {
     console.log();
     const {fixed, unfixed} = automaticResultTable(withDifferences);
     console.log(horizontalTermResultTable(fixed));
-    if (config.collate) {
-      for (const {unfixed} of collatedResultTables(withDifferences)) {
+    if (config.partition === 'measurement') {
+      for (const {unfixed} of partitionResultTableByMeasurement(
+        withDifferences
+      )) {
         console.log(verticalTermResultTable(unfixed));
       }
     } else {

src/test/config_test.ts

@@ -45,7 +45,7 @@ suite('makeConfig', function () {
       csvFileStats: '',
       csvFileRaw: '',
       githubCheck: undefined,
-      collate: false,
+      partition: undefined,
       benchmarks: [
         {
           browser: {
@@ -91,7 +91,7 @@ suite('makeConfig', function () {
       csvFileRaw: '',
       // TODO(aomarks) Be consistent about undefined vs unset.
       githubCheck: undefined,
-      collate: false,
+      partition: undefined,
       benchmarks: [
         {
           browser: {
@@ -137,7 +137,7 @@ suite('makeConfig', function () {
       csvFileStats: '',
       csvFileRaw: '',
       githubCheck: undefined,
-      collate: false,
+      partition: undefined,
       benchmarks: [
         {
           browser: {
@@ -190,7 +190,7 @@ suite('makeConfig', function () {
       remoteAccessibleHost: '',
       // TODO(aomarks) Be consistent about undefined vs unset.
       githubCheck: undefined,
-      collate: false,
+      partition: undefined,
       benchmarks: [
         {
           browser: {
@@ -224,7 +224,7 @@ suite('makeConfig', function () {
     const expected: Config = {
       mode: 'automatic',
       csvFileStats: '',
-      collate: false,
+      partition: undefined,
       csvFileRaw: '',
       jsonFile: '',
       legacyJsonFile: '',

src/test/configfile_test.ts

@@ -48,7 +48,7 @@ suite('config', () => {
         timeout: 7,
         autoSampleConditions: ['0ms', '1ms', '2%', '+3%'],
         resolveBareModules: false,
-        collate: true,
+        partition: 'measurement',
         benchmarks: [
           {
             name: 'remote',
@@ -102,7 +102,7 @@ suite('config', () => {
           relative: [-0.02, 0.02, 0.03],
         },
         resolveBareModules: false,
-        collate: true,
+        partition: 'measurement',
         benchmarks: [
           {
             name: 'remote',
@@ -186,7 +186,7 @@ suite('config', () => {
         timeout: undefined,
         autoSampleConditions: undefined,
         resolveBareModules: undefined,
-        collate: undefined,
+        partition: undefined,
         benchmarks: [
           {
             name: 'http://example.com?foo=bar',
@@ -236,7 +236,7 @@ suite('config', () => {
         sampleSize: undefined,
         timeout: undefined,
         autoSampleConditions: undefined,
-        collate: undefined,
+        partition: undefined,
         resolveBareModules: undefined,
         benchmarks: [
           {
@@ -290,7 +290,7 @@ suite('config', () => {
         timeout: undefined,
         autoSampleConditions: undefined,
         resolveBareModules: undefined,
-        collate: undefined,
+        partition: undefined,
         benchmarks: [
           {
             name: '/mybench/index.html?foo=a',
@@ -370,7 +370,7 @@ suite('config', () => {
         timeout: undefined,
         autoSampleConditions: undefined,
         resolveBareModules: undefined,
-        collate: undefined,
+        partition: undefined,
         benchmarks: [
           {
             name: 'http://example.com',
@@ -446,7 +446,7 @@ suite('config', () => {
         timeout: undefined,
         autoSampleConditions: undefined,
         resolveBareModules: undefined,
-        collate: undefined,
+        partition: undefined,
         benchmarks: [
           {
             name: 'http://example.com?foo=bar',
@@ -496,7 +496,7 @@ suite('config', () => {
         sampleSize: undefined,
         timeout: undefined,
         autoSampleConditions: undefined,
-        collate: undefined,
+        partition: undefined,
         resolveBareModules: undefined,
         benchmarks: [
           {
@@ -583,7 +583,7 @@ suite('config', () => {
         sampleSize: undefined,
         timeout: undefined,
         autoSampleConditions: undefined,
-        collate: undefined,
+        partition: undefined,
         resolveBareModules: undefined,
         benchmarks: [
           {