44:["$","title",null,{"children":"Regression to the Mean | Bart Sakwerda | Observable"}] 4c:T2973,{ // started from //:http://blockbuilder.org/uredkar/e87f0cf2925cd5a78ba919bf7a279e1d // setup const height = 500; const margin = { top: 33, right: 5, bottom: 35, left: 50 }, innerWidth = Math.min(width, height) - margin.left - margin.right, innerHeight = height - margin.top - margin.bottom; // numer of points in scatterplot const numberOfPoints = 500; // initialize global to hold state info for vvarious elements const elemInfo = { // for mean-line and point in rectangles meanInRange: undefined } // add listeners document.querySelector("#selector") .addEventListener("input", selectHandler); let data = bvn(numberOfPoints, rho); document.querySelector(".chart").innerHTML = ""; document.querySelector("#selector").value = "noshow"; const svg = d3.select(".chart").append("svg") .attr("width", innerWidth + margin.left + margin.right) .attr("height", innerHeight + margin.top + margin.bottom) .append("g") .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); const x = d3.scaleLinear() .range([0, innerWidth]); const y = d3.scaleLinear() .range([innerHeight, 0]); const xAxis = d3.axisBottom() .scale(x); const yAxis = d3.axisLeft() .scale(y); // axis title variables const yaxistext = "y" const xaxistext = "x" // text label for the x axis svg.append("text") .attr("class", "axistext") .attr("transform", "translate(" + (innerWidth / 2 + margin.right) + " ," + (innerHeight + margin.top) + ")") .style("text-anchor", "middle") .text(xaxistext); // text label for the y axis svg.append("text") .attr("class", "axistext") .attr("transform", "rotate(-90)") .attr("y", 0 - margin.left) .attr("x", 0 - (innerHeight / 2)) .attr("dy", "1em") .style("text-anchor", "middle") .text(yaxistext) // add basic axes, all points, calculate scales y.domain(d3.extent(data, function (d) { return d.y })); x.domain(d3.extent(data, function (d) { return d.x })); svg.append("g") .attr("class", "x axis") .attr("transform", "translate(0," + innerHeight + ")") .call(xAxis) svg.append("g") .attr("class", "y axis") .call(yAxis); svg.selectAll(".point") .data(data) .enter().append("circle") .attr("class", "point") .attr("r", 2) .attr("cy", function (d) { return y(d.y); }) .attr("cx", function (d) { return x(d.x); }) ; // r, slope, regression ans sd lines let xSeries = data.map(function (e) { return e.x; }); let ySeries = data.map(function (e) { return e.y; }); let {slope, intercept, rSquare, xBar, yBar, sdRatio} = leastSquares(xSeries, ySeries); // regression line equation document.getElementsByClassName("equation")[0] .innerHTML = "y = " + round(slope, 2) + "x + " + round(intercept, 2); // r document.getElementsByClassName("equation")[1] .innerHTML = "r = " + round(Math.sqrt(rSquare), 2); // Add trendline const ptAx = d3.min(xSeries); const ptAy = slope * d3.min(xSeries) + intercept; const ptBx = d3.max(xSeries); const ptBy = intercept + slope * d3.max(xSeries);; svg.append("line") .attr("class", "regression") .attr("x1", x(ptAx)) .attr("y1", y(ptAy)) .attr("x2", x(ptBx)) .attr("y2", y(ptBy)) ; const ptsdAX = d3.min(xSeries); const ptsdAY = yBar + sdRatio * (ptsdAX - xBar); const ptsdBX = d3.max(xSeries); const ptsdBY = yBar + sdRatio * (ptsdBX - xBar); svg.append("line") .attr("class", "sdline") .attr("x1", x(ptsdAX)) .attr("y1", y(ptsdAY)) .attr("x2", x(ptsdBX)) .attr("y2", y(ptsdBY)) ; svg.append("g") .attr("class", "kmeans") .attr("transform", `translate(${x(xBar) - 20}, 0)`) ; const kmeans = d3.select("g.kmeans"); kmeans.append("rect") .attr("x", 0) .attr("y", 0) .attr("width", 40) .attr("height", innerHeight) .attr("fill", "palegreen") ; kmeans.append("line") .attr("class", "meanlevel") .attr("x1", 0) .attr("x2", 40) .attr("y1", y(yBar)) .attr("y2", y(yBar)) ; kmeans.append("circle") .attr("class", "meanpoint") .attr("r", 4) .attr("fill", "darkgreen") .attr("cx", 20) .attr("cy", y(yBar)) ; const dragHandler = d3.drag().on("drag", function () { if (d3.event.x >= 0 && d3.event.x <= innerWidth - 40) { d3.select(this) .attr("transform", `translate(${d3.event.x}, 0)`); const mean = meanInRange(data, x.invert(d3.event.x), x.invert(d3.event.x + 40)); elemInfo.meanInRange = mean; const line = d3.select(".meanlevel"); const meanPoint = d3.select(".meanpoint"); if (isNaN(mean)) { line.style("visibility", "hidden"); meanPoint.style("visibility", "hidden"); } else { line.attr("y1", y(mean)).attr("y2", y(mean)); meanPoint.attr("cy", y(mean)); line.style("visibility", "visible"); meanPoint.style("visibility", "visible"); } } }); dragHandler(kmeans); // make line graph of kmeans const bandWidth = 0.2; const step = (d3.max(xSeries) - d3.min(xSeries)) / 200; const xvals = d3.range(d3.min(xSeries), d3.max(xSeries), step); let pathPoints = []; for (let i = 0; i < xvals.length; i++) { let mean = meanInRange(data, xvals[i] - bandWidth, xvals[i] + bandWidth); if (!isNaN(mean)) { pathPoints.push({x: xvals[i], y: mean}); } } const line = d3.line() .x(function(d) { return x(d.x); }) .y(function(d) { return y(d.y); }); svg.append("path") .datum(pathPoints) .attr("class", "meansPlot") .attr("fill", "none") .attr("stroke", "darkgray") .attr("stroke-linejoin", "round") .attr("stroke-linecap", "round") .attr("stroke-width", 2.0) .attr("d", line) ; /// FUNCTIONS.................................................. function gaussian(a, b) { return { u: Math.sqrt(-2 * Math.log(a)) * Math.cos(2 * Math.PI * b), v: Math.sqrt(-2 * Math.log(a)) * Math.sin(2 * Math.PI * b) }; } function bvn(n = 200, rho = 0.5, mu1 = 0, sigma1 = 1, mu2 = 0, sigma2 = 1) { let data = []; for (let i = 0; i < n; i++) { //generate two independent normals (box-muller) let u1 = Math.random(); let u2 = Math.random(); let { u, v } = gaussian(u1, u2); let x = mu1 + sigma1 * u; let sdY = Math.sqrt((1 - Math.pow(rho, 2)) * Math.pow(sigma2, 2)); let y = mu2 + (sigma2 / sigma1) * rho * (u - mu1) + sdY * v; data.push({ x: x, y: y }); } return data; } function selectHandler(e) { let selection = this.value; if (selection === "noshow") { d3.select(".kmeans").style("visibility", "hidden"); d3.select(".kmeans rect") .attr("cursor", "") .attr("pointer-events", ""); d3.select(".meanlevel").style("visibility", "hidden"); d3.select(".meanpoint").style("visibility", "hidden"); d3.select(".meansPlot").style("visibility", "hidden"); } if (selection === "rect") { d3.select(".kmeans").style("visibility", "visible"); d3.select(".kmeans rect") .attr("cursor", "move") .attr("pointer-events", "all"); d3.select(".meanlevel").style("visibility", !isNaN(elemInfo.meanInRange) ? "visible" : "hidden"); d3.select(".meanpoint").style("visibility", !isNaN(elemInfo.meanInRange) ? "visible" : "hidden"); d3.select(".meansPlot").style("visibility", "hidden"); } if (selection === "all") { d3.select(".kmeans").style("visibility", "hidden"); d3.select(".kmeans rect") .attr("cursor", "") .attr("pointer-events", ""); d3.select(".meanlevel").style("visibility", "hidden"); d3.select(".meanpoint").style("visibility", "hidden"); d3.select(".meansPlot").style("visibility", "visible"); } } function meanInRange(data, a, b) { let sum = 0; let count = 0; for (let i = 0; i < data.length; i++) { let x = data[i].x; if (x >= a && x <= b) { sum += data[i].y count++ } } return (sum / count); } // round decimals function round(value, decimals) { return Number(Math.round(value + 'e' + decimals) + 'e-' + decimals); } // calculate linear regression function leastSquares(xSeries, ySeries) { const reduceSumFunc = function (prev, cur) { return prev + cur; } const xBar = xSeries.reduce(reduceSumFunc) * 1.0 / xSeries.length; const yBar = ySeries.reduce(reduceSumFunc) * 1.0 / ySeries.length; const ssXX = xSeries.map(function (d) { return Math.pow(d - xBar, 2); }) .reduce(reduceSumFunc); const ssYY = ySeries.map(function (d) { return Math.pow(d - yBar, 2); }) .reduce(reduceSumFunc); const sdRatio = Math.sqrt(ssYY / ssXX); const ssXY = xSeries.map(function (d, i) { return (d - xBar) * (ySeries[i] - yBar); }) .reduce(reduceSumFunc); const slope = ssXY / ssXX; const intercept = yBar - (xBar * slope); const rSquare = Math.pow(ssXY, 2) / (ssXX * ssYY); return {slope, intercept, rSquare, xBar, yBar, sdRatio}; } }4d:[] 48:{"id":"d65d5a404326865a","slug":"regression-to-the-mean","title":"Regression to the Mean","thumbnail":"1c96686bacd38ebea428b89abc5600c33f0da1db50b68944e2bcb4e83bc8e40c","access_level":"public","create_time":"$D2018-10-07T16:28:30.561Z","update_time":"$D2018-10-07T16:28:30.561Z","view_time":null,"first_public_time":"$D2019-10-03T06:11:37.426Z","has_unpublished_edits":null,"fork":{"id":"ab0a22bfd59871cc","slug":"regression-to-the-mean","title":"Regression to the Mean","owner":{"id":"53c0831ffe2cb008","login":"homerhanumat","name":"Homer White"}},"stars":0,"starred":false,"owner":{"type":"team","id":"0ea2fc7a29cae5f9","login":"barts83eu","name":"Bart Sakwerda","avatar_url":"https://avatars-next.observableusercontent.com/398e424acb0a28b1fbe1adb62c12d9ddcabdf6757e07f51fae07da710222f5d1","tier":"starter_2024","bio":"@cebu-data @dataopsph\n\n","following":false},"version":153,"publish_version":153,"role":null,"authored":false,"authors":[{"type":"individual","id":"8b1d4111b5424764","login":"barts83eu","name":"Bart Sakwerda","avatar_url":"https://avatars-next.observableusercontent.com/398e424acb0a28b1fbe1adb62c12d9ddcabdf6757e07f51fae07da710222f5d1","tier":null,"bio":"@cebu-data @dataopsph\n\n","following":false}],"members":[{"type":"individual","id":"8b1d4111b5424764","login":"barts83eu","name":"Bart Sakwerda","avatar_url":"https://avatars-next.observableusercontent.com/398e424acb0a28b1fbe1adb62c12d9ddcabdf6757e07f51fae07da710222f5d1","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Regression to the Mean","cells":[{"id":0,"value":"md`# Regression to the Mean\n\nThis is an instructional app to illustrate regression to the mean.\n\n## Background\n\nThere are two lines typically associatd with a scatterplot. The *standard deviation* line\ncontains the point of averages of the *x* and *y*-values, and its slope is the ratio of \nthe standard deviation of the *y*-values to the standard deviation of the *x*-values, thus:`\n\n\n\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":81,"value":"tex.block`y - \\bar{y} = \\frac{s_y}{s_x}(x - \\bar{x}).`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":83,"value":"md`The regression lines also goes through the point of averages, but its slope is less in absolute value, being equal to the slope of the SD-line multiplied by the correlation coefficient *r*, thus:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":88,"value":"tex.block`y - \\bar{y} = r\\frac{s_y}{s_x}(x - \\bar{x}).`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":78,"value":"md`Most beginning students regard the SD-line as a superior summary of the scatterplot: the regression line appears to them to be a bit too shallow.\n\nNevertheless the regression line is much better for predicting *y*-values from a known *x*-value. The point of the app is to realize this.\n\n## The App`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"html``\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":28,"value":"{\n const appContent = html`

\n \n

`;\n //appContent.dispatchEvent(new CustomEvent(\"update\"));\n return appContent;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":141,"value":"viewof rho = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":145,"value":"rho","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":49,"value":"md`The plot above shows 500 points randomly generated from a\nbivariate normal distrobution with a correaltion that the user can\nadjust. The red dotted line is the SD-line; the blue line is the regression line.\n\nThere is an option to compute means of *y*-values of the points in a draggable rectangle, or to view a plot of near-neighbor means of *y*-values. Either way, the regression line emerges as the superior predictor of the mean *y*-value for a given *x*-value.\n\nUse the slider above to adjust the target correlation of the scatteplot.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":61,"value":"md`\n\n## The Lesson\n\nIf one uses the SD-line for prediction then a person who is *a* standard deviations above the mean with respect to her *x*-value would be predicted to be *a* standard deviations above the mean with respect to her *y*-value. But the app shows that such a prediction is liable to well off the mark.\n\nOn the other hand when one relies on the regression line for prediction then an individual who is *a* standard deviations above the mean with respect to her *x*-value is predicted to have a *y*-value that is only ${tex`r \\times a`} standard deviations above the mean of all *y*-values: above average to be sure, but not so strikingly above average as her x-value was.\n\nThis is the phenomenon known as *regression to the mean*.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":14,"value":"d3 = require(\"https://d3js.org/d3.v4.min.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"$4c","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W4d","stdlib":"1"}} 24:null