31:T499,md`Our colleagues, product managers Alice and Bob, independently build two new features for the app, A and B. Now, let's say *we* have a magic crystal ball and therefore know that feature A in fact increases engagement rate by ${A_delta_rate_percent}%. That is, if you roll out feature A to users, engagement rate will be ${base_rate_percent}%+${A_delta_rate_percent}%=${base_rate_percent + A_delta_rate_percent}%. But Alice doesn't have our crystal ball and doesn't know this -- she'll have to run an A/B experiment to measure this effect. Looking into the crystall ball again, we see that Bob's feature B increases engagement rate by ${B_delta_rate_percent}%. Same as with Alice, Bob doesn't know this yet, but if you roll Bob's feature out, _our_ crystal ball tells us that the true engagement rate becomes ${base_rate_percent}%+${B_delta_rate_percent}%=${base_rate_percent + B_delta_rate_percent}%. And let's say A and B effects are additive; that is, if someone is exposed to both A and B, engagement rate becomes a sum of both ${base_rate_percent}%+${A_delta_rate_percent}%+${B_delta_rate_percent}%=${base_rate_percent + A_delta_rate_percent + B_delta_rate_percent}%. `32:T7f8,exp_data = { console.log("hello") const T = 't' const C = 'c' const U = 'u' const A_exposure = Array.from({length: total_users}, (v, k) => { return (Math.random() < (A_exp.alloc / 100)) ? (Math.random() < .5 ? T : C) : U }) const B_exposure = Array.from({length: total_users}, (v, k) => { return (Math.random() < (B_exp.alloc / 100)) ? (Math.random() < .5 ? T : C) : U }) const rates = Array.from({length: total_users}, (v, k) => { return base_rate_percent + (A_exposure[k] == T ? A_delta_rate_percent : 0) + (B_exposure[k] == T ? B_delta_rate_percent : 0) }) const outcomes = Array.from({length: total_users}, (v, k) => { let days = 0; for (let i = 0; i < A_exp.duration; i++) { if (Math.random() < (rates[k] / 100)) days += 1; } return days; }) function make_summary(exp1, exp2) { let summary = { treat_only: 0, treat_both: 0, control_only: 0, control_both: 0, unalloc: 0, control_succ: 0, treat_succ: 0, }; for (let i = 0; i < total_users; i++) { if ((exp1[i] == T) && (exp2[i] != T)) summary.treat_only += 1; if ((exp1[i] == T) && (exp2[i] == T)) summary.treat_both += 1; if ((exp1[i] == C) && (exp2[i] != T)) summary.control_only += 1; if ((exp1[i] == C) && (exp2[i] == T)) summary.control_both += 1; if (exp1[i] == U) summary.unalloc += 1; if (exp1[i] == T) summary.treat_succ += outcomes[i]; if (exp1[i] == C) summary.control_succ += outcomes[i]; } summary.treat = summary.treat_only + summary.treat_both summary.control = summary.control_only + summary.control_both return summary } return { A_exposure: A_exposure, B_exposure: B_exposure, rates: rates, outcomes: outcomes, A_summary: make_summary(A_exposure, B_exposure), B_summary: make_summary(B_exposure, A_exposure), } }33:T5d3,binomTest = function(trialsNum, successesNum, hypothesisTrialsNum, hypothesisSuccessesNum, confidenceRate) { // adopted from https://github.com/thii/binomjs var confidence = new Confidence(); confidence.addVariant({ id: 'A', name: 'Hypothesis Data', conversionCount: hypothesisSuccessesNum, eventCount: hypothesisTrialsNum }) confidence.addVariant({ id: 'B', name: 'Test Data', conversionCount: successesNum, eventCount: trialsNum }) var confidenceTestResult = confidence.getResult(); var averageSuccessRate, minSuccessRate, maxSuccessRate; var pValue = getPValue(confidence._zScore); var testResult; var error; if (confidenceTestResult.hasWinner) { minSuccessRate = confidenceTestResult.confidenceInterval.min / 100; maxSuccessRate = confidenceTestResult.confidenceInterval.max / 100; averageSuccessRate = confidenceTestResult.winnerID == 'B' ? successesNum / trialsNum : hypothesisSuccessesNum / hypothesisTrialsNum; } else { error = confidenceTestResult.readable; } if (( (1 - confidenceRate) / 2 >= pValue ) && ( confidenceTestResult.winnerID == 'B' )) { testResult = true; } else { testResult = false; } return { averageSuccessRate: averageSuccessRate, minSuccessRate: minSuccessRate, maxSuccessRate: maxSuccessRate, pValue: pValue, testResult: testResult, error: error } }; 34:T89b,function grid_chart(zones) { const width = 600; const height = 100; const num_rows = 9; const num_cols = 30; const square_h = 10; const square_w = 10; const total_count = zones.reduce((acc, v) => acc + v.count, 0) zones.reduce((acc, v) => { v.range = [acc / total_count, (acc + v.count) / total_count]; return acc + v.count }, 0) console.log(zones, total_count) const overlap = (r1, r2) => { return Math.max(0, Math.min(r1[1], r2[1]) - Math.max(r1[0], r2[0])) } const svg = d3.select(DOM.svg(width, height)) .style("font", "10px sans-serif") .style("width", width * 1.1) .style("height", height * 1.1); const d = Array.from({length: num_rows * num_cols}, (v, k) => { const low = k / (num_rows * num_cols) const high = (k+1) / (num_rows * num_cols) const max_overlap_zone = zones.reduce((acc, zone, i) => { const o = overlap([low, high], zone.range) if (o > acc.overlap) { return {zone: zone, overlap: o}; } else { return acc; } }, {zone: zones[0], overlap: 0}).zone; return { "row": Math.floor(k/num_rows), "col": k%num_rows, "idx": k, "color": max_overlap_zone.color } }); var squares = svg.selectAll(".square").data(d); squares.enter().append('rect') .attr('x', (cell) => cell.row * square_h) .attr('y', (cell) => cell.col * square_w) .attr("width", square_w) .attr('height', square_h) .attr('fill', (cell) => cell.color) .attr("style", "stroke: #E6E6E6; stroke-width:1px;") var legend = svg.append("g") .attr("class","legend") .attr("transform","translate("+ (square_w * num_cols + 20)+ ",10)") .style("font-size","12px") zones.forEach((z, i) => { var row = legend.append("g") row.append("text") .text(z.name + " (" + Math.round(z.count) + " users)") .attr("y", (i*16 + 8)).attr("x", 15) row.append("rect") .attr('x', 0) .attr('y', (i*16)) .attr("width", square_w) .attr('height', square_h) .attr('fill', z.color) .attr("style", "stroke: #E6E6E6; stroke-width:1px;") }) return svg.node() }35:[] 2f:{"title":"A mobile app and two new features","cells":[{"id":0,"value":"md`\n# A mobile app and two new features\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":24,"value":"md`\nImagine we work on a commercial mobile app or a website. We want people to use as much as possible, so our main KPI is user engagement, and we measure it as a probability of opening the app at least once on a given day. Today, the app engagement rate is ${base_rate_percent}%. Or in other words, people use the app ${base_rate_percent*30/100} days a month on average.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":6,"value":"viewof base_rate_percent = slider(\"Base engagement rate, %\", 10, 0, 100, 1, \"auto\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":54,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":95,"value":"md`\n# Experiment setup\n\nAlice and Bob want to A/B test their features to see if they help increase the engagement rate. Ideally, their experiments will help them to get a good estimate of the increased engagement rate that we got from the crystal ball, so they can make the right decision when it comes to rolling out this feature to all users or not.\n\nThe app has ${total_users} total users. Alice doesn't know anything about Bob's experiment and vice versa. Both Alice and Bob will independently allocate ${A_exp.alloc}% of users for the experiment, and split those users into treatment and control groups equally. They'll run their experiments for ${A_exp.duration} days _simultaneously_. \n\nMath tells us to expect to have around ${Math.round(A_exp.alloc*total_users/100/2)} (${A_exp.alloc}%*${total_users}/2) users to be enrolled in treatment and control groups each, for each of Bob's and Alices's experiments. In reality the number will be slightly different, since we're enrolling each user randomly.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":154,"value":"viewof total_users = slider(\"Total users: \", 12000, 100, 100000, 100, \"auto\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":124,"value":"viewof A_exp = multislider([\n {id: \"alloc\", label: \"Allocation, %: \", val: 100, min: 1, max: 100, step: 1, width: \"auto\"},\n {id: \"duration\", label: \"Duration, days: \", val: 14, min: 5, max: 90, step: 1, width: \"auto\"}\n])","pinned":false,"hidden":false,"mode":"ojs"},{"id":298,"value":"md`\n# Run the experiments\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":378,"value":"md`Let's run the simulation:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":346,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":263,"value":"md`Now, let's see how this looks from Alice's point of view. There are treatment and control groups, as usual (red and white). However, unbeknownst to Alice, some of those users will also be exposed to Bob's experiment treatment (dark shades below):`","pinned":false,"hidden":false,"mode":"ojs"},{"id":174,"value":"chart_a = {\n\n const zones = [\n {\n name: \"A treatment\",\n count: exp_data.A_summary.treat_only,\n color: \"#e33\"\n },\n {\n name: \"A treatment, also exposed to B\",\n count: exp_data.A_summary.treat_both,\n color: \"#a22\"\n },\n {\n name: \"A control\",\n count: exp_data.A_summary.control_only,\n color: \"#3e3\"\n },\n {\n name:\"A control, also exposed to B\",\n count: exp_data.A_summary.control_both,\n color: \"#2a2\"\n },\n {\n name: \"Unallocated for Alice's experiment\",\n count: exp_data.A_summary.unalloc,\n color: \"gray\"\n }\n ]\n return grid_chart(zones);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":368,"value":"md`And we'll see a very similar picture from Bob's point of view. Again, dark shades are users exposed to Alice's experiment, but Bob doesn't know that:`","pinned":false,"hidden":false,"mode":"ojs"},{"id":372,"value":"chart_b = {\n\n const zones = [\n {\n name: \"B treatment, not exposed to A\",\n count: exp_data.B_summary.treat_only,\n color: \"#e33\"\n },\n {\n name: \"B treatment, exposed to A\",\n count: exp_data.B_summary.treat_both,\n color: \"#a22\"\n },\n {\n name: \"B control, not exposed to A\",\n count: exp_data.B_summary.control_only,\n color: \"#3e3\"\n },\n {\n name:\"B control, also exposed to A\",\n count: exp_data.B_summary.control_both,\n color: \"#2a2\"\n },\n {\n name: \"Unallocated for Bob's experiment\",\n count: exp_data.B_summary.unalloc,\n color: \"gray\"\n }\n ]\n return grid_chart(zones);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":388,"value":"md`\n# Results - z-test\n\nIf we look at the simulated results for Alice's experiment, users in control group had engagement rate of ${(exp_data.A_summary.control_succ / (exp_data.A_summary.control * A_exp.duration) * 100).toFixed(2)}% vs treatment group ${(exp_data.A_summary.treat_succ / (exp_data.A_summary.treat * A_exp.duration) * 100).toFixed(2)}%. Remember that for Alice's feature, \"true\" engagement rate from crystal ball would be exactly ${base_rate_percent + A_delta_rate_percent}%, but that's only if Bob didn't run his experiment, which is not the case. If our data scientist performed a statistical test (z-test) on the data we collected, they would say that ${fmt_test_result(A_test_result)}\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":57,"value":"viewof A_delta_rate_percent = slider(\"A crystal ball lift, %\", 2, -5, 5, .1, \"30px\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":461,"value":"md`For Bob's experiment, users in control group had engagement rate of ${(exp_data.B_summary.control_succ / (exp_data.B_summary.control * B_exp.duration) * 100).toFixed(2)}% vs treatment group ${(exp_data.B_summary.treat_succ / (exp_data.B_summary.treat * B_exp.duration) * 100).toFixed(2)}% . If a data scientist performed a z-test, they would say that ${fmt_test_result(B_test_result)}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":88,"value":"viewof B_delta_rate_percent = slider(\"B crystal ball lift, %\", 0.5, -5, 5, .1, \"30px\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":482,"value":"md`\n# Results - Bayesian\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":507,"value":"md`\nWe can also analyze the results from Bayesian point of view. The model we use assumes that \"app open\" actions follow Bernoulli distribution, and uses a non-informative beta prior on engagement rates.\n\nHere's what posterior distribution of rates would look like for each experiment:\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":484,"value":"{\n let x = stdlib.linspace( 0.07, 0.2, 200 )\n let c = x.map((v) => stdlib.base.dists.beta.pdf( v, 1 + exp_data.A_summary.control_succ, 1 + exp_data.A_summary.control * A_exp.duration - exp_data.A_summary.control_succ ))\n let t = x.map((v) => stdlib.base.dists.beta.pdf( v, 1 + exp_data.A_summary.treat_succ, 1 + exp_data.A_summary.treat * A_exp.duration - exp_data.A_summary.treat_succ ))\n let plot = stdlib.plot( [x, x], [c, t] )\n plot.title = 'Posterior PDF of engagement rate, Alice'\n plot.height = 250\n plot.xLabel = 'Engagement Rate'\n plot.yLabel = ''\n let r = plot.render('html')\n return html`${r}`\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":518,"value":"{\n let x = stdlib.linspace( 0.07, 0.2, 200 )\n let c = x.map((v) => stdlib.base.dists.beta.pdf( v, 1 + exp_data.B_summary.control_succ, 1 + exp_data.B_summary.control * B_exp.duration - exp_data.B_summary.control_succ ))\n let t = x.map((v) => stdlib.base.dists.beta.pdf( v, 1 + exp_data.B_summary.treat_succ, 1 + exp_data.B_summary.treat * B_exp.duration - exp_data.B_summary.treat_succ ))\n let plot = stdlib.plot( [x, x], [c, t] )\n plot.title = 'Posterior PDF of engagement rate, Bob'\n plot.height = 250\n plot.xLabel = 'Engagement Rate'\n plot.yLabel = ''\n let r = plot.render('html')\n return html`${r}`\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":567,"value":"md`\n# Conclusion\n\nRunning Alice's and Bob's experiments concurrently doesn't affect their respective results all that much in practice; they're very likely to come to the right conclusion in the end, especially if the allocation is low enough (under 50% each). You can play with the sliders above to see that.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":436,"value":"md`\n.\n\n.\n\n.\n\n.\n\n.\n## Implementation goes below\n\n\n\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":418,"value":"conf_level = 0.95","pinned":false,"hidden":false,"mode":"ojs"},{"id":409,"value":"A_test_result = {\n //var binom = new Binom();\n return binomTest(exp_data.A_summary.treat * A_exp.duration,\n exp_data.A_summary.treat_succ,\n exp_data.A_summary.control * A_exp.duration,\n exp_data.A_summary.control_succ,\n conf_level)\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":416,"value":"B_test_result = {\n\n return binomTest(exp_data.B_summary.treat * B_exp.duration,\n exp_data.B_summary.treat_succ,\n exp_data.B_summary.control * B_exp.duration,\n exp_data.B_summary.control_succ,\n conf_level)\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":454,"value":"function fmt_test_result(test_result) {\n let res = (test_result.error !== undefined ? test_result.error : \"the result is statistically significant, and:\").toLowerCase();\n let win = (test_result.minSuccessRate ? `${test_result.testResult?\"treatment\":\"control\"} group is the winner`: \"\");\n let conf = (test_result.minSuccessRate ? (\"winning rate is between \" + (test_result.minSuccessRate * 100).toFixed(2) + \"% and \" + (test_result.maxSuccessRate * 100).toFixed(2)) + \"% with \" + conf_level + \"% confidence\" : \"\");\n \n function make_md_list(rows) {\n return rows.map((v) => v ? \"\\n * \" + v : \"\")\n }\n return res + make_md_list([win, conf]).join(\"\")\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":164,"value":"B_exp = A_exp","pinned":false,"hidden":false,"mode":"ojs"},{"id":127,"value":"import {form} from \"@mbostock/form-input\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":170,"value":"d3 = require(\"d3@^5.9\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":480,"value":"stdlib = require( \"https://unpkg.com/@stdlib/stdlib@0.0.60/dist/stdlib-flat.min.js\" )","pinned":false,"hidden":false,"mode":"ojs"},{"id":405,"value":"// Binom = await require('https://bundle.run/binomjs@0.1.1')\nBinom = await require('https://unpkg.com/binomjs@0.1.1/binom.js')","pinned":false,"hidden":false,"mode":"ojs"},{"id":644,"value":" Confidence = await require('confidencejs');","pinned":true,"hidden":false,"mode":"ojs"},{"id":687,"value":"function getPValue(z) {\n // adopted from https://github.com/thii/binomjs\n var Z_MAX = 6;\n var w, x, y;\n \n if (z === 0.0) {\n x = 0.0;\n } else {\n y = 0.5 * Math.abs(z);\n if (y > (Z_MAX * 0.5)) {\n x = 1.0;\n } else if (y < 1.0) {\n w = y * y;\n x = ((((((((0.000124818987 * w - 0.001075204047) * w + 0.005198775019) * w - 0.019198292004) * w + 0.059054035642) * w - 0.151968751364) * w + 0.319152932694) * w - 0.531923007300) * w + 0.797884560593) * y * 2.0;\n } else {\n y -= 2.0;\n x = (((((((((((((-0.000045255659 * y + 0.000152529290) * y - 0.000019538132) * y - 0.000676904986) * y + 0.001390604284) * y - 0.000794620820) * y - 0.002034254874) * y + 0.006549791214) * y - 0.010557625006) * y + 0.011630447319) * y - 0.009279453341) * y + 0.005353579108) * y - 0.002141268741) * y + 0.000535310849) * y + 0.999936657524;\n }\n }\n\n return (1 - x) / 2;\n };","pinned":false,"hidden":false,"mode":"ojs"},{"id":684,"value":"$33","pinned":false,"hidden":false,"mode":"ojs"},{"id":260,"value":"$34","pinned":false,"hidden":false,"mode":"ojs"},{"id":307,"value":"function slider(label, val, min, max, step, width) {\n const div = html`\n ${label}\n \n `;\n const range = div.querySelector(\"[type=range]\");\n const number = div.querySelector(\"[type=number]\");\n div.value = range.value = number.value = val;\n range.oninput = () => number.value = div.value = range.valueAsNumber;\n number.oninput = () => range.value = div.value = number.valueAsNumber;\n return div;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":328,"value":"function multislider(items) {\n const divs = items.map(item => {\n return `

${item.label}\n

`\n })\n const div = html`${divs.concat(\"\")}`;\n\n const sync = (name, default_val) => {\n const range = div.querySelector('#' + name + \" > [type=range]\");\n const number = div.querySelector('#' + name + \" > [type=number]\");\n div.value[name] = range.value = number.value = default_val;\n range.oninput = () => number.value = div.value[name] = range.valueAsNumber;\n number.oninput = () => range.value = div.value[name] = number.valueAsNumber;\n }\n div.value = {}\n \n items.forEach(item => sync(item.id, item.val))\n return div;\n}","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W35","stdlib":"1","titling":"auto"}