Faculty at CMU (@cmudig) and researcher at @apple.
Published
Edited
Mar 23, 2021
Importers
6 stars
function table(stats, numWarmups, numTrials, numRepeats) {
const longestTime = d3.max(stats.map(s => s.max));
const best = d3.min(stats, d => d.median);
return md`
### ${
stats.running
? stats.warmup
? `Running Warmup ${stats.currentTrial} of ${numWarmups}`
: `Running Trial ${stats.currentTrial} of ${numTrials}`
: `${numTrials} Trials`
} ${
numRepeats > 1
? `<small>(with ${numRepeats} repeats per trial)</small>`
: ''
}
<div style="max-width: 700px"><div style="background: #eee;"><div style="transition: all; background:black;width:${(stats.currentTrial /
numTrials) *
100}%;height:3px;"></div>
</div>
<table style="max-width: 100%">
<thead>
<tr>
<th>Method</th>
<th style="text-align:right">Mean</th>
<th style="text-align:right">%Median</th>
${numRepeats > 1 ? '' : `<th style="text-align:right">Stdev</th>`}
<th style="text-align:right">Range</th>
<th style="text-align:right">Histogram</th>
</tr>
</thead>
<tbody>
${stats.map(
e =>
html`<tr>
<td>${e.name}</td>
<td style="text-align:right; font-variant-numeric: tabular-nums">${fmt(
e.mean
)} ${numRepeats > 1 ? '' : `± ${fmt(e.sem)}`} ms</td>
<td style="text-align:right; font-variant-numeric: tabular-nums">${
e.median === best ? "—" : `+${commas(100 * (e.median / best - 1))}%`
}</td>
${
numRepeats > 1
? ''
: `<td style="text-align:right; font-variant-numeric: tabular-nums">${fmt(
e.sd
)}</td>`
}
<td style="text-align:right; font-variant-numeric: tabular-nums">${fmt(
e.min
)} - ${fmt(e.max)} ms</td>
<td style="text-align:right; font-variant-numeric: tabular-nums">${html`${hist(
e.times,
{
domain: [0, longestTime],
width: 120,
height: 20,
thresholds: 25
}
)}`}</td>
</tr>`
)}
</tbody>
</table>
${
numRepeats > 1
? md`<small>**Note:** Because the number of repeats per trial is larger than 1, some statistics cannot be computed.</small>`
: ''
}`;
}
async function* computeStats(
reRun,
experiments,
numTrials,
numWarmups,
numRepeats
) {
const stats = [];
function results(step) {
const result = [];
for (let i = 0; i < experiments.length; i++) {
const times = stats.map(stat => stat[i]);
const sd = d3.deviation(times);
result.push({
...experiments[i],
mean: d3.mean(times),
median: d3.median(times),
min: d3.min(times),
max: d3.max(times),
// standard deviation
sd,
// standard error of the mean (https://en.wikipedia.org/wiki/Standard_error)
sem: sd / Math.sqrt(numTrials),
times
});
}
result.currentTrial = step + 1;
result.running = step + 1 < numTrials;
return result;
}
yield results(-1);
const maybeReRun = reRun;
// await new Promise(requestIdleCallback);
const runTrial = () =>
experiments.map(e => {
if (e.before) e.before();
const results = timeit(e.run, numRepeats);
if (e.after) e.after();
return results;
});
for (let i = 0; i < numWarmups; i++) {
runTrial();
const result = [...experiments];
result.running = true;
result.warmup = true;
result.currentTrial = i;
yield result;
}
for (let i = 0; i < numTrials; i++) {
// add short pause so the browser can do things it needs to do
let stat = await Promises.delay(10, runTrial());
stats.push(stat);
yield results(i);
}
}
function timeit(f, numRepeats, ...args) {
const start = performance.now();
for (let i = 0; i < numRepeats; i++) {
f(...args);
}
return (performance.now() - start) / numRepeats;
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
