Add option to partition results table by measurements

CHANGELOG.md

@@ -7,6 +7,9 @@ project adheres to [Semantic Versioning](http://semver.org/).
 
 <!-- ## Unreleased -->
 
+- Add `collate` option for collating results tables by measurement (when
+  multiple measurements are in use).
+
 ## [0.5.5] 2020-09-21
 
 - A warning is now displayed when there are multiple performance marks or

config.schema.json

@@ -428,6 +428,10 @@
             "minItems": 1,
             "type": "array"
         },
+        "collate": {
+            "description": "Whether to collate result table by measurement (when multiple measurements\nare in use)",
+            "type": "boolean"
+        },
         "horizons": {
             "description": "The degrees of difference to try and resolve when auto-sampling\n(e.g. 0ms, +1ms, -1ms, 0%, +1%, -1%, default 0%).",
             "items": {

package.json

@@ -13,6 +13,7 @@
   "scripts": {
     "prepare": "if [ -f './tsconfig.json' ]; then npm run build; fi;",
     "build": "rimraf lib/ client/lib/ && mkdir lib && npm run generate-json-schema && tsc && tsc -p client/ && npm run lint",
+    "build:watch": "tsc --watch",
     "generate-json-schema": "typescript-json-schema tsconfig.json ConfigFile --include src/configfile.ts --required --noExtraProps > config.schema.json",
     "lint": "tslint --project . --format stylish",
     "format": "clang-format --style=file -i \"--glob=./{client,}/src/**/*.ts\"",

src/config.ts

@@ -41,6 +41,7 @@ export interface Config {
   forceCleanNpmInstall: boolean;
   csvFileStats: string;
   csvFileRaw: string;
+  collate?: boolean;
 }
 
 export async function makeConfig(opts: Opts): Promise<Config> {
@@ -86,6 +87,9 @@ export async function makeConfig(opts: Opts): Promise<Config> {
     if (opts['window-size'] !== undefined) {
       throw new Error('--window-size cannot be specified when using --config');
     }
+    if (opts['collate'] !== undefined) {
+      throw new Error('--collate cannot be specified when using --config');
+    }
     const rawConfigObj = await fsExtra.readJson(opts.config);
     const validatedConfigObj = await parseConfigFile(rawConfigObj);
 
@@ -164,6 +168,7 @@ export function applyDefaults(partial: Partial<Config>): Config {
         defaults.resolveBareModules,
     root: partial.root !== undefined ? partial.root : defaults.root,
     timeout: partial.timeout !== undefined ? partial.timeout : defaults.timeout,
+    collate: partial.collate !== undefined ? partial.collate : defaults.collate,
   };
 }
 

src/configfile.ts

@@ -47,6 +47,12 @@ export interface ConfigFile {
    */
   horizons?: string[];
 
+  /**
+   * Whether to collate result table by measurement (when multiple measurements
+   * are in use)
+   */
+  collate?: boolean;
+
   /**
    * Benchmarks to run.
    * @TJS-minItems 1
@@ -300,6 +306,7 @@ export async function parseConfigFile(parsedJson: unknown):
         undefined,
     benchmarks,
     resolveBareModules: validated.resolveBareModules,
+    collate: validated.collate,
   };
 }
 

src/defaults.ts

@@ -24,6 +24,7 @@ export const mode = 'automatic';
 export const resolveBareModules = true;
 export const forceCleanNpmInstall = false;
 export const measurementExpression = 'window.tachometerResult';
+export const collate = false;
 
 export function measurement(url: LocalUrl|RemoteUrl): Measurement {
   if (url.kind === 'remote') {

src/flags.ts

@@ -201,6 +201,13 @@ export const optDefs: commandLineUsage.OptionDefinition[] = [
         ` (default "${defaults.windowWidth},${defaults.windowHeight}").`,
     type: String,
   },
+  {
+    name: 'collate',
+    description:
+        `Whether to collate result table by measurement (when multiple ` +
+        `measurements are in use)`,
+    type: Boolean,
+  },
 ];
 
 export interface Opts {
@@ -228,6 +235,7 @@ export interface Opts {
   'csv-file': string;
   'csv-file-raw': string;
   'json-file': string;
+  collate: boolean;
 
   // Extra arguments not associated with a flag are put here. These are our
   // benchmark names/URLs.

src/format.ts

@@ -17,6 +17,7 @@ import ansi = require('ansi-escape-sequences');
 
 import {Difference, ConfidenceInterval, ResultStats, ResultStatsWithDifferences} from './stats';
 import {BenchmarkSpec, BenchmarkResult} from './types';
+import {measurementName} from './measure';
 
 export const spinner = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'].map(
     (frame) => ansi.format(`[blue]{${frame}}`));
@@ -104,6 +105,23 @@ export function automaticResultTable(results: ResultStats[]): AutomaticResults {
   return {fixed: fixedTable, unfixed: unfixedTable};
 }
 
+export function collatedResultTables(results: ResultStatsWithDifferences[]) {
+  const collated: {[index: string]: ResultStatsWithDifferences[]} = {};
+  results.forEach((result) => {
+    const meas = measurementName(result.result.measurement);
+    (collated[meas] || (collated[meas] = [])).push({
+      ...result,
+      differences: result.differences.filter(
+          (_, i) => measurementName(results[i].result.measurement) === meas)
+    });
+  });
+  const tables: AutomaticResults[] = [];
+  for (const results of Object.values(collated)) {
+    tables.push(automaticResultTable(results));
+  }
+  return tables;
+}
+
 /**
  * Format a terminal text result table where each result is a row:
  *

src/runner.ts

@@ -21,7 +21,7 @@ import {measure, measurementName} from './measure';
 import {BenchmarkResult, BenchmarkSpec} from './types';
 import {formatCsvStats, formatCsvRaw} from './csv';
 import {ResultStatsWithDifferences, horizonsResolved, summaryStats, computeDifferences} from './stats';
-import {verticalTermResultTable, horizontalTermResultTable, verticalHtmlResultTable, horizontalHtmlResultTable, automaticResultTable, spinner, benchmarkOneLiner} from './format';
+import {verticalTermResultTable, horizontalTermResultTable, verticalHtmlResultTable, horizontalHtmlResultTable, automaticResultTable, spinner, benchmarkOneLiner, collatedResultTables} from './format';
 import {Config} from './config';
 import * as github from './github';
 import {Server, Session} from './server';
@@ -343,7 +343,13 @@ export class Runner {
     console.log();
     const {fixed, unfixed} = automaticResultTable(withDifferences);
     console.log(horizontalTermResultTable(fixed));
-    console.log(verticalTermResultTable(unfixed));
+    if (config.collate) {
+      for (const {unfixed} of collatedResultTables(withDifferences)) {
+        console.log(verticalTermResultTable(unfixed));
+      }
+    } else {
+      console.log(verticalTermResultTable(unfixed));
+    }
 
     if (hitTimeout === true) {
       console.log(ansi.format(

src/test/config_test.ts

@@ -50,6 +50,7 @@ suite('makeConfig', function() {
       csvFileStats: '',
       csvFileRaw: '',
       githubCheck: undefined,
+      collate: false,
       benchmarks: [
         {
           browser: {
@@ -93,6 +94,7 @@ suite('makeConfig', function() {
       csvFileRaw: '',
       // TODO(aomarks) Be consistent about undefined vs unset.
       githubCheck: undefined,
+      collate: false,
       benchmarks: [
         {
           browser: {
@@ -136,6 +138,7 @@ suite('makeConfig', function() {
       csvFileStats: '',
       csvFileRaw: '',
       githubCheck: undefined,
+      collate: false,
       benchmarks: [
         {
           browser: {
@@ -186,6 +189,7 @@ suite('makeConfig', function() {
       remoteAccessibleHost: '',
       // TODO(aomarks) Be consistent about undefined vs unset.
       githubCheck: undefined,
+      collate: false,
       benchmarks: [
         {
           browser: {

src/test/configfile_test.ts

@@ -50,6 +50,7 @@ suite('config', () => {
         timeout: 7,
         horizons: ['0ms', '1ms', '2%', '+3%'],
         resolveBareModules: false,
+        collate: true,
         benchmarks: [
           {
             name: 'remote',
@@ -103,6 +104,7 @@ suite('config', () => {
           relative: [-0.02, 0.02, 0.03],
         },
         resolveBareModules: false,
+        collate: true,
         benchmarks: [
           {
             name: 'remote',
@@ -186,6 +188,7 @@ suite('config', () => {
         timeout: undefined,
         horizons: undefined,
         resolveBareModules: undefined,
+        collate: undefined,
         benchmarks: [
           {
             name: 'http://example.com?foo=bar',
@@ -252,6 +255,7 @@ suite('config', () => {
         timeout: undefined,
         horizons: undefined,
         resolveBareModules: undefined,
+        collate: undefined,
         benchmarks: [
           {
             name: '/mybench/index.html?foo=a',
@@ -331,6 +335,7 @@ suite('config', () => {
         timeout: undefined,
         horizons: undefined,
         resolveBareModules: undefined,
+        collate: undefined,
         benchmarks: [
           {
             name: 'http://example.com',
@@ -406,6 +411,7 @@ suite('config', () => {
         timeout: undefined,
         horizons: undefined,
         resolveBareModules: undefined,
+        collate: undefined,
         benchmarks: [
           {
             name: 'http://example.com?foo=bar',

src/test/format_test.ts

@@ -15,7 +15,7 @@ import * as path from 'path';
 import stripAnsi = require('strip-ansi');
 
 import {ConfigFile} from '../configfile';
-import {automaticResultTable, verticalTermResultTable} from '../format';
+import {automaticResultTable, verticalTermResultTable, collatedResultTables} from '../format';
 import {fakeResults, testData} from './test_helpers';
 
 /**
@@ -29,6 +29,21 @@ async function fakeResultTable(configFile: ConfigFile): Promise<string> {
   return stripAnsi(verticalTermResultTable(resultTable));
 }
 
+/**
+ * Given a config file object, generates fake measurement results, and returns
+ * the collated terminal formatted result table that would be printed (minus
+ * color etc. formatting).
+ */
+async function fakeCollatedResultTable(configFile: ConfigFile):
+    Promise<string> {
+  const results = await fakeResults(configFile);
+  return collatedResultTables(results)
+      .map((result) => {
+        return stripAnsi(verticalTermResultTable(result.unfixed));
+      })
+      .join('\n');
+}
+
 suite('format', () => {
   let prevCwd: string;
   suiteSetup(() => {
@@ -169,4 +184,102 @@ suite('format', () => {
     `;
     assert.equal(actual, expected.trim() + '\n');
   });
+
+  test('multiple measurements, collate: false', async () => {
+    const config: ConfigFile = {
+      benchmarks: [
+        {
+          name: 'foo',
+          url: 'http://foo.com',
+          measurement: [
+            {name: 'render', mode: 'performance', entryName: 'render'},
+            {name: 'update', mode: 'performance', entryName: 'update'}
+          ],
+        },
+        {
+          name: 'bar',
+          url: 'http://bar.com',
+          measurement: [
+            {name: 'render', mode: 'performance', entryName: 'render'},
+            {name: 'update', mode: 'performance', entryName: 'update'}
+          ],
+        },
+      ],
+    };
+
+    const actual = await fakeResultTable(config);
+    const expected = `
+┌──────────────┬──────────┬───────────────────┬───────────────────┬───────────────────┬───────────────────┬───────────────────┐
+│ Benchmark    │ Bytes    │          Avg time │   vs foo [render] │   vs foo [update] │   vs bar [render] │   vs bar [update] │
+├──────────────┼──────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤
+│ foo [render] │ 1.00 KiB │  8.56ms - 11.44ms │                   │            unsure │            faster │            faster │
+│              │          │                   │          -        │       -20% - +20% │         42% - 58% │         42% - 58% │
+│              │          │                   │                   │ -2.03ms - +2.03ms │  7.97ms - 12.03ms │  7.97ms - 12.03ms │
+├──────────────┼──────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤
+│ foo [update] │ 1.00 KiB │  8.56ms - 11.44ms │            unsure │                   │            faster │            faster │
+│              │          │                   │       -20% - +20% │          -        │         42% - 58% │         42% - 58% │
+│              │          │                   │ -2.03ms - +2.03ms │                   │  7.97ms - 12.03ms │  7.97ms - 12.03ms │
+├──────────────┼──────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤
+│ bar [render] │ 2.00 KiB │ 18.56ms - 21.44ms │            slower │            slower │                   │            unsure │
+│              │          │                   │        68% - 132% │        68% - 132% │          -        │       -10% - +10% │
+│              │          │                   │  7.97ms - 12.03ms │  7.97ms - 12.03ms │                   │ -2.03ms - +2.03ms │
+├──────────────┼──────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┼───────────────────┤
+│ bar [update] │ 2.00 KiB │ 18.56ms - 21.44ms │            slower │            slower │            unsure │                   │
+│              │          │                   │        68% - 132% │        68% - 132% │       -10% - +10% │          -        │
+│              │          │                   │  7.97ms - 12.03ms │  7.97ms - 12.03ms │ -2.03ms - +2.03ms │                   │
+└──────────────┴──────────┴───────────────────┴───────────────────┴───────────────────┴───────────────────┴───────────────────┘
+    `;
+    assert.equal(actual, expected.trim() + '\n');
+  });
+
+  test('multiple measurements, collate: false', async () => {
+    const config: ConfigFile = {
+      benchmarks: [
+        {
+          name: 'foo',
+          url: 'http://foo.com',
+          measurement: [
+            {name: 'render', mode: 'performance', entryName: 'render'},
+            {name: 'update', mode: 'performance', entryName: 'update'}
+          ],
+        },
+        {
+          name: 'bar',
+          url: 'http://bar.com',
+          measurement: [
+            {name: 'render', mode: 'performance', entryName: 'render'},
+            {name: 'update', mode: 'performance', entryName: 'update'}
+          ],
+        },
+      ],
+    };
+
+    const actual = await fakeCollatedResultTable(config);
+    const expected = `
+┌──────────────┬──────────┬───────────────────┬──────────────────┬──────────────────┐
+│ Benchmark    │ Bytes    │          Avg time │  vs foo [render] │  vs bar [render] │
+├──────────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤
+│ foo [render] │ 1.00 KiB │  8.56ms - 11.44ms │                  │           faster │
+│              │          │                   │         -        │        42% - 58% │
+│              │          │                   │                  │ 7.97ms - 12.03ms │
+├──────────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤
+│ bar [render] │ 2.00 KiB │ 18.56ms - 21.44ms │           slower │                  │
+│              │          │                   │       68% - 132% │         -        │
+│              │          │                   │ 7.97ms - 12.03ms │                  │
+└──────────────┴──────────┴───────────────────┴──────────────────┴──────────────────┘
+
+┌──────────────┬──────────┬───────────────────┬──────────────────┬──────────────────┐
+│ Benchmark    │ Bytes    │          Avg time │  vs foo [update] │  vs bar [update] │
+├──────────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤
+│ foo [update] │ 1.00 KiB │  8.56ms - 11.44ms │                  │           faster │
+│              │          │                   │         -        │        42% - 58% │
+│              │          │                   │                  │ 7.97ms - 12.03ms │
+├──────────────┼──────────┼───────────────────┼──────────────────┼──────────────────┤
+│ bar [update] │ 2.00 KiB │ 18.56ms - 21.44ms │           slower │                  │
+│              │          │                   │       68% - 132% │         -        │
+│              │          │                   │ 7.97ms - 12.03ms │                  │
+└──────────────┴──────────┴───────────────────┴──────────────────┴──────────────────┘
+    `;
+    assert.equal(actual, expected.trim() + '\n');
+  });
 });