Benchmark Utilities / UW Interactive Data Lab

UW Interactive Data Lab

Data visualization and analysis at the University of Washington, Seattle.

Workspace

Published

Edited

Jun 1, 2021

Importers

6 stars

// Time the execution of the provided function, invoked with the given arguments.

// The function may be async / return Promises.

async function timeit(f, ...args) {

const start = performance.now();

await f(...args);

return (performance.now() - start);

}

// Run a suite of benchmark experiments.

// Yields results as they become available for progressive updates.

// Experiment 'run' functions may be async and will be await'd.

// If the runFlag is falsy, the benchmarks will not be run.

// Use this to control when benchmarks are actually evaluated.

async function* benchmarkStats(

experiments,

runFlag = 1,

numTrials = 25,

numWarmups = 5

) {

// benchmark statistics

const stats = [];

// marshall current results

function results(step) {

const result = [];

for (let i = 0; i < experiments.length; i++) {

const times = stats.map(stat => stat[i]);

const sd = d3.deviation(times);

result.push({

...experiments[i],

mean: d3.mean(times),

min: d3.min(times),

max: d3.max(times),

sd, // std dev

sem: sd / Math.sqrt(numTrials), // std error of the mean

times

});

}

result.numTrials = numTrials;

result.currentTrial = step + 1;

result.running = step + 1 < numTrials;

return result;

}

// initially yield empty result, exit if runFlag is not truthy

yield results(-1);

if (!runFlag) return;

// run a single trial of each experiment

const runTrial = async () => {

const trial = [];

for (let i = 0; i < experiments.length; ++i) {

const e = experiments[i];

const results = await timeit(e.run);

trial.push(results);

}

return trial;

}

// run warm-up trials

for (let i = 0; i < numWarmups; i++) {

await runTrial();

const result = [...experiments];

result.running = true;

result.warmup = true;

result.numWarmups = numWarmups;

result.currentTrial = i;

yield result;

}

// run measured benchmark trials

for (let i = 0; i < numTrials; i++) {

// add short pause so the browser can do things it needs to do

let stat = await Promises.delay(10, runTrial());

stats.push(stat);

yield results(i);

}

// Generate a button that increments a counter (default 0)

// This button can be used to run/re-run benchmarks

function runButton(label, value = 0) {

const button = html`<button name="button">${label}</button>`;

button.value = value;

button.onclick = event => {

button.dispatchEvent(new CustomEvent("input"));

button.value = (+button.value) + 1;

};

return button;

}

// Generate an HTML table to show benchmark results

function benchmarkTable(stats) {

const longestTime = d3.max(stats.map(s => s.max));

const best = d3.min(stats, d => d.mean);

const commas = d3.format(',d');

const fmt = d3.format(',.2f');

return md`

### ${

stats.running

? stats.warmup

? `Running Warmup ${stats.currentTrial} of ${stats.numWarmups}`

: `Running Trial ${stats.currentTrial} of ${stats.numTrials}`

: `${stats.numTrials} Trials`

}

</div>

<thead>

<tr>

<th>Method</th>

<th style="text-align:right;">Range</th>

<th style="text-align:right;">Histogram</th>

</tr>

</thead>

<tbody>

${stats.map(

e =>

html`<tr>

<td style="text-align:right; font-variant-numeric: tabular-nums">${

e.mean === best ? "—" : `× ${fmt(e.mean / best)}`

}</td>

<td style="text-align:right; font-variant-numeric: tabular-nums">${fmt(

e.mean

)} ms</td>

<td style="text-align:right; font-variant-numeric: tabular-nums">${html`${hist(

e.times,

{

domain: [0, longestTime],

width: 120,

height: 20,

thresholds: 25

}

)}`}</td>

</tr>`

)}

</tbody>

</table>

}

Purpose-built for displays of data

Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.

Learn more