2a:null 2b:["$","title",null,{"children":"Law of Large Numbers | Mattias Villani | Observable"}] 31:T619,plt = Plot.plot({ caption:html` Sampling distribution for the sample mean
Probability that the sample mean deviates more than ε from μ `, color: { legend: true }, x: { label: "sample mean, xBar", axis: true, domain: [userinputs[2] - 3*userinputs[3], userinputs[2] + 3*userinputs[3]] }, y: { axis: false }, marks: [ Plot.ruleY([0]), Plot.ruleX([userinputs[2]], {label: "hej"}), Plot.line(datapdf, {x: "x", y: "pdf", stroke : "steelblue", strokeWidth: 2}), Plot.areaY(datapdf, {filter: d => d.x >= userinputs[2] + userinputs[1], x: "x", y: "pdf", fill: "steelblue", opacity: 0.2}), Plot.areaY(datapdf, {filter: d => d.x <= userinputs[2] - userinputs[1], x: "x", y: "pdf", fill: "steelblue", opacity: 0.2}), Plot.arrow([{x1: userinputs[2]*1.002, y1: 0, x2: userinputs[2] + userinputs[1], y2: 0}], {x1: "x1", y1: "y1", x2: "x2", y2: "y2", stroke : "purple", strokeWidth: 2}), Plot.text([{x: userinputs[2] + userinputs[1]/2, y: 0, z: "ϵ"}], {x: "x", y: "y", text: "z", fill: "purple", dy : -8}), Plot.arrow([{x1: userinputs[2]*0.998, y1: 0, x2: userinputs[2] - userinputs[1], y2: 0}], {x1: "x1", y1: "y1", x2: "x2", y2: "y2", stroke : "purple", strokeWidth: 2}), Plot.text([{x: userinputs[2] - userinputs[1]/2, y: 0, z: "-ϵ"}], {x: "x", y: "y", text: "z", fill: "purple", dy : -8}), ] })32:[] 2e:{"id":"85f734df8eaff66d","slug":"law-large-numbers","title":"Law of Large Numbers","thumbnail":"babf2440da0e2c7382b2b5f73f782ad4085fc001a79e90a9d724684c9b208558","access_level":"public","create_time":"$D2023-07-23T16:33:45.449Z","update_time":"$D2025-03-11T21:38:40.049Z","view_time":null,"first_public_time":"$D2023-07-23T16:33:45.449Z","fork":{"id":"c484bb2064980aba","slug":"stora-talens-lag-allman","title":"Stora talens lag","owner":{"id":"7592a7ab121f74a0","login":"mattiasvillani","name":"Mattias Villani"}},"stars":0,"starred":false,"owner":{"type":"team","id":"7592a7ab121f74a0","login":"mattiasvillani","name":"Mattias Villani","avatar_url":"https://avatars-next.observableusercontent.com/a440aaf39a3abefb86cbcc758fd7a5ab42555ae68d6825b52ec006f385a94d1d","tier":"pro_2024"},"version":1258,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"56499be8cf43f0ef","login":"mattiasvillani","name":"Mattias Villani","avatar_url":"https://avatars-next.observableusercontent.com/a440aaf39a3abefb86cbcc758fd7a5ab42555ae68d6825b52ec006f385a94d1d","tier":null}],"members":[{"type":"individual","id":"56499be8cf43f0ef","login":"mattiasvillani","name":"Mattias Villani","avatar_url":"https://avatars-next.observableusercontent.com/a440aaf39a3abefb86cbcc758fd7a5ab42555ae68d6825b52ec006f385a94d1d","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":true,"trashed":false,"subscribed":null,"forks":3,"collection":{"id":"3247547e8812c274","slug":"statistics","title":"Basic Statistics","owner":{"id":"7592a7ab121f74a0","login":"mattiasvillani","name":"Mattias Villani"}},"type":"notebook","body":{"title":"Law of Large Numbers","cells":[{"id":620,"value":"# Law of Large Numbers","pinned":false,"hidden":false,"mode":"md"},{"id":1170,"value":"toc()","pinned":false,"hidden":false,"mode":"ojs"},{"id":1083,"value":"### Law of Large Numbers for binary variables\n\nIf you [toss a coin](https://g.co/kgs/3deymt) a large number of times, you will note that the [relative frequency](https://en.wikipedia.org/wiki/Empirical_probability) of Heads tends to stabilize around the probability of Heads, i.e. ${tex`p=0.5`} if the coin is fair. This is a consequence of a general law for averages of many independent variables, the **Law of Large Numbers**.\n\nIn the interactive graph below, you can experiment with the number of coin tosses ${tex`n`} and the probability of getting Heads in each toss ${tex`p`}. Try this:\n\n- increase the number of coin tosses ${tex`n`} and observe how the relative frequency of *Heads* stabilizes around ${tex`0.5`}\n- change the probability *p* of getting *Heads* and observe how the relative frequency stabilizes around that probability ${tex`p`}.","pinned":false,"hidden":false,"mode":"md"},{"id":1096,"value":"viewof samplesize_coin = Inputs.range([1, 1000], {value: 10, label: tex`n`, step: 1})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1101,"value":"viewof p = Inputs.range([0.001, 0.999], {value: 0.5, label: tex`p`, step: 0.001})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1099,"value":"pltconv = {\n return Plot.plot({\n y: { label: \"sample mean\", domain: [0,1]},\n x: { label: \"number of coin tosses\"},\n marks: [\n Plot.ruleX([0]),\n Plot.lineY([{x: 0, y: p},{x: samplesize_coin, y: p}], {x: \"x\", y: \"y\", stroke: \"orange\"}),\n Plot.lineY(running_means_coin, {x: \"x\", y: \"y\", stroke: \"steelblue\"})\n ]\n})\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1121,"value":"We can record the results of the ${tex`i`}th coin toss in a binary random variable ${tex`X_i`} as follows:\n\n${tex`\nX_i = \n\\begin{cases}\n 1 \\text{ if i:th toss is Heads} \\\\\n 0 \\text{ if i:th toss is Tails}\n\\end{cases}\n`}\n\nThe random variables ${tex`X_1,X_2,...,X_n`} are then independent and [Bernoulli](bernoulli-distribution) distributed with success probability ${tex`p`}. The relative frequency from the coin tosses is simply the mean:

\n${tex`\n\\bar X_n = \\frac{1}{n} \\sum_{i=1}^n X_i = \\frac{\\text{number of successes}}{\\text{number of trials}}\n`}\n

\nwhere we use ${tex`n`} as the subscript on ${tex`\\bar X_n`} to clearly indicate that the mean depends on the number of observations ${tex`n`}. We can now very informally begin to express the Law of Large Numbers:\n\n>

The mean of independent Bernoulli-distributed variables with probability ${tex`p`} will stabilize around ${tex`p`} as ${tex`n`} increases.","pinned":false,"hidden":false,"mode":"md"},{"id":1128,"value":"For a binary variable like Bernoulli, the expected value ${tex`\\operatorname{E}(X)`} is equal to the probability of success ${tex`p`}, i.e., ${tex`\\operatorname{E}(X)=p`}. We can therefore express the Law of Large Numbers slightly different:\n\n>

**Informal version of the Law of Large Numbers for Bernoulli variables**
\n> The sample mean of independent Bernoulli distributed variables with probability ${tex`p`} will stabilize around the expected value of the variables
${tex`\\operatorname{E}(X)=p`} as ${tex`n`} increases.","pinned":false,"hidden":false,"mode":"md"},{"id":583,"value":"### Law of Large Numbers of general random variables\nThe Law of Large Numbers applies not only to Bernoulli distributed variables, but to *all* independent random variables with finite variance, for example:\n- If ${tex`X_1,X_2,...,X_n`} are independent observations from a [normal distribution](normal-gaussian-distribution) ${tex`X_i \\overset{\\mathrm{iid}}{\\sim} \\operatorname{N}(\\mu,\\sigma^2)`}. The mean of these variables will stabilize around the expected value ${tex`\\mu=\\operatorname{E}(X)`} as the sample size ${tex`n`} increases.\n- If ${tex`X_1,X_2,...,X_n`} are independent observations from an [exponential distribution](exponential-distribution) with parameter ${tex`\\lambda`}, the Law of Large Numbers says that the mean will stabilize around the expected value in this exponential distribution: ${tex`\\mathrm{E}(X)=\\frac{1}{\\lambda}`}.
\n\n> The sample mean of a large number of observations will always be very close to the expected value in the population. This statement will be made precise below.","pinned":false,"hidden":false,"mode":"md"},{"id":867,"value":"Let's examine how the sample mean ${tex`\\bar X_n`} of random variables from a normal population\n\n${tex`X_1,X_2,\\ldots,X_n \\overset{\\mathrm{iid}}{\\sim} \\operatorname{N}(\\mu,\\sigma^2)`}\n\nstabilizes around the expected value ${tex`\\mu`}. \n\nIn the figure below, you can increase the sample size ${tex`n`} and see how the sample mean ${tex`\\bar X_n`} stabilizes around ${tex`\\mu`}.","pinned":false,"hidden":false,"mode":"md"},{"id":921,"value":"viewof samplesize = Inputs.range([1, 10000], {value: 10, label: tex`n`, step: 1})","pinned":false,"hidden":false,"mode":"ojs"},{"id":881,"value":"viewof param = Inputs.form([\n Inputs.range([-5, 5], {value: 1, label: tex`\\mu`, step: 0.01}),\n Inputs.range([0.01, 5], {value: 1, label: tex`\\sigma`, step: 0.01})\n])","pinned":false,"hidden":false,"mode":"ojs"},{"id":914,"value":"pltkonv = {\n return Plot.plot({\n y: { label: \"sample mean\"},\n x: { label: \"number of observations\"},\n marks: [\n Plot.ruleX([0]),\n Plot.lineY([{x: 0, y: param[0]},{x: samplesize, y: param[0]}], {x: \"x\", y: \"y\", stroke: \"orange\"}),\n Plot.lineY(running_means, {x: \"x\", y: \"y\", stroke: \"steelblue\"})\n ]\n})\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":585,"value":"### Mathematical description as convergence in probability 😱","pinned":false,"hidden":false,"mode":"md"},{"id":635,"value":"The mathematically correct description of the Law of Large Numbers requires more background in mathematical analysis than what is required for a basic course in statistics. For those who know a little extra mathematics, a bit more info is provided here. The final interactive graph in the notebook can be useful even if you can't follow the mathematics.","pinned":false,"hidden":false,"mode":"md"},{"id":1190,"value":"Here is the formal description of the Law of Large Numbers:","pinned":false,"hidden":false,"mode":"md"},{"id":587,"value":">

**Law of Large Numbers**
\nIf ${tex`X_1,X_2,...,X_n`} are independent identically distributed random variables from a distribution with expected value ${tex`\\mu`} and finite variance, then it holds that\n\n>

${tex`\n \\bar X_n \\overset{P}{\\rightarrow} \\mu \\text{ as }n\\rightarrow \\infty,\n`}

\n>

which is read as ${tex`\\bar X_n`} *converges in probability* to the expected value ${tex`\\mu`} when ${tex`n`} grows really large (approaches infinity ${tex`\\infty`}). \n\nThe result actually holds also without the assumption of finite variance, and there are other versions for dependent variables too, but this is the version seen in most basic statistics textbooks.","pinned":false,"hidden":false,"mode":"md"},{"id":612,"value":"The mathematical term *converges in probability* was previously written more informally as *stabilizes around*. The exact mathematical definition is","pinned":false,"hidden":false,"mode":"md"},{"id":589,"value":">

**Convergence in Probability**
\n${tex`\n \\bar X_n \\overset{P}{\\rightarrow} \\mu`} means that for all positive numbers ${tex`\\epsilon`}, we have that\n\n>

${tex`\n \\lim_{n\\rightarrow \\infty} P(\\vert \\bar X_n -\\mu \\vert >\\epsilon) = 0\n`}

","pinned":false,"hidden":false,"mode":"md"},{"id":1195,"value":"The meaning of this is that the probability that ${tex`\\bar X_n`} deviates from ${tex`\\mu`} by more than ${tex`\\epsilon`} becomes increasingly small as ${tex`n`} increases. Hence, the distribution of ${tex`\\bar X_n`} becomes more and more concentrated around ${tex`\\mu`} as ${tex`n`} grows larger. This holds true regardless of how stringent our requirements are for ${tex`\\bar X_n`} to be close to ${tex`\\mu`}, that is no matter how small ${tex`\\epsilon`} is (this is the 'for all positive ${tex`\\epsilon`}' part).","pinned":false,"hidden":false,"mode":"md"},{"id":775,"value":"textNormalPop = md`\nTo get a sense of what the Law of Large Numbers implies, we can examine the distribution of ${tex`\\bar X_n`} when data comes from the normal population \n\n${tex`X_1,X_2,\\ldots,X_n \\overset{\\mathrm{iid}}{\\sim} \\operatorname{N}(\\mu,\\sigma^2)`} \n\nIn this case, when observations are normally distributed, it can be shown that the sampling distribution for the mean is exactly\n\n${tex`\\bar X_n \\sim \\operatorname{N}(\\mu,\\sigma^2/n)`}\n\nfor any *n*, even very small ones.\n\nIn the figure below, you can \n- increase ${tex`n`} to observe how the sampling distribution for ${tex`\\bar X_n`} becomes increasingly concentrated around the population mean ${tex`\\mu`} and that the probability ${tex`P(\\vert \\bar X_n -\\mu \\vert >\\epsilon)`} (light blue area) goes to zero.\n- make ${tex`\\epsilon`} even smaller so that the light blue areas appear again, and then make ${tex`n`} even larger and observe again how the light blue areas eventually disappears.\n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":691,"value":"viewof userinputs = Inputs.form([\n Inputs.range([1,3000], {value: 1, step: 1, label: tex`n`}),\n Inputs.range([0.001, 0.25], {value: 0.25, step: 0.001, label: tex`\\epsilon`}),\n Inputs.range([1, 5], {value: 3, step: 0.001, label: tex`\\mu`}),\n Inputs.range([0.1, 5], {value: 0.2, step: 0.1, label: tex`\\sigma`})\n])","pinned":false,"hidden":false,"mode":"ojs"},{"id":722,"value":"prob_within_eps = tex.block`P(\\vert \\bar X_n -\\mu \\vert >${userinputs[1]}) = ${(jstat.normal.cdf(userinputs[2] - userinputs[1], userinputs[2], userinputs[3]/Math.sqrt(userinputs[0])) + (1-jstat.normal.cdf(userinputs[2] + userinputs[1], userinputs[2], userinputs[3]/Math.sqrt(userinputs[0])))).toPrecision(4)}`","pinned":false,"hidden":false,"mode":"ojs"},{"id":672,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":566,"value":"
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n\n---\n#### 🙈 Appendix 🙈\n\nHere is just some code and details that you probably don't want to read.","pinned":false,"hidden":false,"mode":"md"},{"id":1163,"value":"#### Imports","pinned":false,"hidden":false,"mode":"md"},{"id":657,"value":"jstat = require('jstat')","pinned":false,"hidden":false,"mode":"ojs"},{"id":1165,"value":"function toc(selector = \"h2,h3\", heading = \"Table of Contents\") {\n return Generators.observe(notify => {\n let headings = [];\n\n function observed() {\n const h = Array.from(document.querySelectorAll(selector));\n if (h.length !== headings.length || h.some((h, i) => headings[i] !== h)) {\n notify(html`${heading}

${DOM.text(h.textContent)}`,\n {onclick: e => (e.preventDefault(), h.scrollIntoView())}\n );\n })}`);\n }\n }\n\n const observer = new MutationObserver(observed);\n observer.observe(document.body, {childList: true, subtree: true});\n observed();\n return () => observer.disconnect();\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1168,"value":"#### Data","pinned":false,"hidden":false,"mode":"md"},{"id":1113,"value":"xall_coin = Array.from({length: 10000}, d3.randomBernoulli(p))","pinned":false,"hidden":false,"mode":"ojs"},{"id":885,"value":"xall = d3.range(10000).map( x => jstat.normal.sample(param[0],param[1]))","pinned":false,"hidden":false,"mode":"ojs"},{"id":1108,"value":"running_means_coin = xall.slice(0, samplesize_coin).map(function(x, i) {\n return { x: i + 1, y: d3.mean(xall_coin.slice(0, i + 1)) };\n });","pinned":false,"hidden":false,"mode":"ojs"},{"id":890,"value":"running_means = xall.slice(0, samplesize).map(function(x, i) {\n return { x: i + 1, y: d3.mean(xall.slice(0, i + 1)) };\n });","pinned":false,"hidden":false,"mode":"ojs"},{"id":662,"value":"datapdf = {\n const x = d3.range(userinputs[2]-3*userinputs[3], userinputs[2]+3*userinputs[3], 0.001);\n const normpdf = x.map(x => ({x: x, pdf: jstat.normal.pdf( x, userinputs[2], userinputs[3]/Math.sqrt(userinputs[0]))}));\n return normpdf\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":6,"value":"d3 = require('d3-selection@1', 'd3-array@2', 'd3-random@2', 'd3-format@1')","pinned":false,"hidden":false,"mode":"ojs"},{"id":591,"value":"viewof md_colour = Inputs.color({label: \"Highlight color cells\", value: \"#F5F5DC\"})","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}} 2f:"$2e:body"