Published
Edited
Jul 9, 2019
// edit this value
visits_per_version = 13000;
// edit this value
baseline_conversion_rate = 3; // percent
// edit this value
expected_increase = 20; // percent
// see the code for calculate_p_value below
calculate_p_value(
visits_per_version,
baseline_conversion_rate,
expected_increase
);
is_significant(
calculate_p_value(
visits_per_version,
baseline_conversion_rate,
expected_increase
)
)? md`**Yes**, that p-value is significant`: md`**No**, that p-value is not significant`;
// inspired by https://vwo.com/ab-split-test-significance-calculator/
function calculate_p_value(visits_per_version, baseline_conversion_rate, expected_increase) {
var visits_versiona = visits_per_version;
var visits_versionb = visits_per_version; // symmetrical test
var baseline_conversion_percentage = baseline_conversion_rate * 0.01;
var expected_increase_percentage = expected_increase * 0.01;
var baseline_conversions_abs = baseline_conversion_percentage * visits_versiona;
var versionb_conversions_abs = baseline_conversions_abs * (1 + (expected_increase_percentage));
var versionb_conversion_percentage = versionb_conversions_abs / visits_versionb;
var std_error = Math.sqrt(
(baseline_conversion_percentage * (1 - baseline_conversion_percentage) / visits_versiona)
+ (versionb_conversion_percentage * (1 - versionb_conversion_percentage) / visits_versionb)
);
var z_value = (versionb_conversion_percentage - baseline_conversion_percentage) / std_error;
var p_value = get_p_value_from_z_value(z_value);
if (p_value > 0.5)
p_value = 1 - p_value;
p_value = Math.round(p_value * 1000) / 1000;
return p_value;
}
function is_significant(p_value) {
return p_value < 0.05;
}
// source: https://vwo.com/ab-split-test-significance-calculator/
//
// This takes a z-value, which is a measure related to how much variance
// is in the data given the visits_per_version in relation to the
// baseline_conversion_rate and the expected_increase.
//
// The p-value is a measure of percentage of the area under a curve,
// which is why there are these d1 to d6 constants. They help
// approximate the calculation of that area under the curve.
// It's approximated calculus, essentially.
// See https://www.analyzemath.com/calculus/Integrals/area_under_curve.html
function get_p_value_from_z_value(z_value) {
var d1 = 0.0498673470,
d2 = 0.0211410061,
d3 = 0.0032776263,
d4 = 0.0000380036,
d5 = 0.0000488906,
d6 = 0.0000053830;
var a = Math.abs(z_value);
var t = 1.0 + a * (d1 + a * (d2 + a * (d3 + a * (d4 + a * (d5 + a * d6)))));
t *= t;
t *= t;
t *= t;
t *= t;
t = 1.0 / (t + t);
if (z_value >= 0)
t = 1 - t;
return t;
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
