2a:null 2b:["$","title",null,{"children":"Likelihood and Information | Mattias Villani | Observable"}] 31:T517,pdfdata = { const ngrid = 500 const lowerprob = 0.001 const upperprob = 0.999 if (dist_type == 'bernoulli'){ const thetaHat = d3.mean(sample) const SEthetaHat = 1 / info[0] const theta = params[0]; const maxloglik = loglik(thetaHat) const thetaHatgrid = d3.range(0.001,0.999,0.001) var pdf = thetaHatgrid.map(d => ({par: d, loglik: loglik(d), lik: Math.exp(loglik(d) - maxloglik)})) return pdf } if (dist_type == 'exponential'){ const thetaHat = d3.mean(sample) const SEthetaHat = Math.sqrt(thetaHat**2 / n) const theta = params[0]; const maxloglik = loglik(thetaHat) const thetaHatgrid = d3.range(d3.max([0.01, thetaHat - 10*SEthetaHat]), thetaHat + 10*SEthetaHat, 10*SEthetaHat/ngrid) var pdf = thetaHatgrid.map(d => ({par: d, loglik: loglik(d), lik: Math.exp(loglik(d) - maxloglik)})) return pdf } if (dist_type == 'poisson'){ const thetaHat = d3.mean(sample) const SEthetaHat = Math.sqrt(thetaHat / n) const theta = params[0]; const maxloglik = loglik(thetaHat) const thetaHatgrid = d3.range(d3.max([0.01, thetaHat - 5*SEthetaHat]), thetaHat + 5*SEthetaHat, 10*SEthetaHat/ngrid) var pdf = thetaHatgrid.map(d => ({par: d, loglik: loglik(d), lik: Math.exp(loglik(d) - maxloglik)})) return pdf } }32:[] 2e:{"id":"a5a2590f79bc9a3b","slug":"likelihood-and-information","title":"Likelihood and Information","thumbnail":"f08fce7dca401d0ebf12fb6590b55703ad9707d7f3bbacbb2d9650b55cd410d0","access_level":"public","create_time":"$D2025-04-28T20:46:04.757Z","update_time":"$D2026-05-10T21:35:17.068Z","view_time":null,"first_public_time":"$D2025-04-28T22:34:05.336Z","fork":{"id":"20856782195d9fb4","slug":"chebyshevs-inequality","title":"Chebyshev's inequality","owner":{"id":"7592a7ab121f74a0","login":"mattiasvillani","name":"Mattias Villani"}},"stars":0,"starred":false,"owner":{"type":"team","id":"7592a7ab121f74a0","login":"mattiasvillani","name":"Mattias Villani","avatar_url":"https://avatars-next.observableusercontent.com/a440aaf39a3abefb86cbcc758fd7a5ab42555ae68d6825b52ec006f385a94d1d","tier":"pro_2024"},"version":1814,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"56499be8cf43f0ef","login":"mattiasvillani","name":"Mattias Villani","avatar_url":"https://avatars-next.observableusercontent.com/a440aaf39a3abefb86cbcc758fd7a5ab42555ae68d6825b52ec006f385a94d1d","tier":null}],"members":[{"type":"individual","id":"56499be8cf43f0ef","login":"mattiasvillani","name":"Mattias Villani","avatar_url":"https://avatars-next.observableusercontent.com/a440aaf39a3abefb86cbcc758fd7a5ab42555ae68d6825b52ec006f385a94d1d","tier":null,"role":"editor"},{"type":"team","id":"7592a7ab121f74a0","login":"mattiasvillani","name":"Mattias Villani","avatar_url":"https://avatars-next.observableusercontent.com/a440aaf39a3abefb86cbcc758fd7a5ab42555ae68d6825b52ec006f385a94d1d","tier":"pro_2024","role":"viewer"}],"comments":[],"custom_thumbnail":false,"trashed":false,"subscribed":null,"forks":1,"collection":{"id":"186f48bbffc13b97","slug":"stm","title":"STM","owner":{"id":"7592a7ab121f74a0","login":"mattiasvillani","name":"Mattias Villani"}},"type":"notebook","body":{"title":"Likelihood and Information","cells":[{"id":0,"value":"# Likelihood and Information\n","pinned":false,"hidden":false,"mode":"md"},{"id":1445,"value":"> This notebook explores the concepts of **observed information** and **expected information** (also called **Fisher information**) that quantifies the amount of information in the likelihood from a sample of size ${tex`n`}.","pinned":false,"hidden":false,"mode":"md"},{"id":1643,"value":"The **observed information** in a sample of ${tex`n`} observations is the negative second derivative of the log-likelihood function ${tex`\\ell(\\theta)`} evaluated at the maximum likelihood estimator ${tex`\\hat\\theta`} of the parameter ${tex`\\theta`}:\n${tex.block`J_n(\\hat\\theta)= - \\ell ''(\\hat\\theta)`}\n\nThe **Fisher** information is the expected information, where the expectation is with respect to the sampling distribution of the data\n${tex.block`I_n(\\theta)= \\mathbb{E}\\big(J_n(\\theta)\\big)`}\n\nThe sampling distribution of the maximum likelihood estimator ${tex`\\hat\\theta`} can be approximated in large samples by\n${tex.block`\\hat\\theta \\overset{\\mathrm{approx}}{\\sim} N(\\theta,I_n^{-1}(\\theta))`}\n\nor, since ${tex`I_n(\\theta) \\approx J_n(\\hat\\theta)`} in large samples,\n\n${tex.block`\\hat\\theta \\overset{\\mathrm{approx}}{\\sim} N(\\theta,J_n^{-1}(\\hat\\theta))`}","pinned":false,"hidden":false,"mode":"md"},{"id":729,"value":"## Interactive exploration","pinned":false,"hidden":false,"mode":"md"},{"id":287,"value":"viewof dist_type = Inputs.select(['bernoulli','poisson', 'exponential'], {value: \"poisson\", label: \"distribution:\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":792,"value":"viewof params = {\n var form;\n if (dist_type == 'bernoulli'){\n form = Inputs.form([\n Inputs.range([0, 1], {value: 0.3, step: 0.01, label: tex`\\text{probability }\\theta`})\n ])\n }\n if (dist_type == 'exponential'){\n form = Inputs.form([\n Inputs.range([0, 3], {value: 1, step: 0.1, label: tex`\\text{scale }\\beta`})\n ])\n }\n if (dist_type == 'poisson'){\n form = Inputs.form([\n Inputs.range([0, 3], {value: 1, step: 0.1, label: tex`\\text{mean }\\lambda`})\n ])\n }\n return form;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1364,"value":"viewof n = Inputs.range([1, n_max], {label: tex`\\text{sample size, }n`, value: 5, step: 1})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1486,"value":"viewof simulate = Inputs.button(\"Simulate data!\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":1582,"value":"**Observed information** ${tex`J=`} ${info[0].toFixed(2)} and **Fisher information** ${tex`I=`} ${info[1].toFixed(2)}
\nThe approximate standard error based on the observed information is ${Math.sqrt(1 / info[0]).toFixed(3)}
\nThe approximate standard error based on the Fisher information is ${Math.sqrt(1 / info[1]).toFixed(3)}","pinned":false,"hidden":false,"mode":"md"},{"id":1634,"value":"viewof centerAtTrue = Inputs.toggle({label: \"center sampling dist on true param\", value: true})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1692,"value":"plt = html`\n \n \n \n

\n ${pltlike}\n

\n \n

\n ${pltsamp}\n

\n\n \n`","pinned":false,"hidden":false,"mode":"ojs"},{"id":1013,"value":"
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n
\n\n---\n## 🙈 Appendix 🙈\n\nJust some code and implementation details that you probably don't want to read.","pinned":false,"hidden":false,"mode":"md"},{"id":765,"value":"#### Imports","pinned":false,"hidden":false,"mode":"md"},{"id":770,"value":"jStat = require('jstat')","pinned":false,"hidden":false,"mode":"ojs"},{"id":62,"value":"d3 = require(\"https://d3js.org/d3.v5.min.js\")\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":1608,"value":"import {colors} from \"@mattiasvillani/statistics-tools\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":1078,"value":"#### Style","pinned":false,"hidden":false,"mode":"md"},{"id":874,"value":"html``","pinned":false,"hidden":false,"mode":"ojs"},{"id":1029,"value":"viewof md_colour = Inputs.color({label: \"Highlight color cells\", value: \"#F5F5DC\"})","pinned":false,"hidden":false,"mode":"ojs"},{"id":760,"value":"#### Data","pinned":false,"hidden":false,"mode":"md"},{"id":1500,"value":"n_max = 100","pinned":false,"hidden":false,"mode":"ojs"},{"id":1489,"value":"data = {\n simulate\n if (dist_type == 'bernoulli'){\n const theta = params[0];\n return d3.range(n_max).map(d => jStat.uniform.sample(0,1) <= theta)\n }\n if (dist_type == 'exponential'){\n const beta_ = params[0];\n return d3.range(n_max).map(d => jStat.exponential.sample(1/beta_))\n }\n if (dist_type == 'poisson'){\n const lambda = params[0];\n return d3.range(n_max).map(d => jStat.poisson.sample(lambda))\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1508,"value":"sample = data.slice(0, n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1495,"value":"loglik = {\n if (dist_type == 'bernoulli'){\n const theta = params[0];\n const s = d3.sum(data)\n const f = sample.length - s\n return function(par){return d3.sum(sample.map(d => d*Math.log(par) + (1-d)*Math.log(1-par)))}\n }\n if (dist_type == 'exponential'){\n const beta = params[0];\n return function(par){return d3.sum(sample.map(d => Math.log(jStat.exponential.pdf(d, 1/par))))}\n }\n if (dist_type == 'poisson'){\n const lambda = params[0];\n return function(lambda){return d3.sum(sample.map(d => Math.log(jStat.poisson.pdf(d, lambda))))}\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":27,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":1578,"value":"info = {\n if (dist_type == 'bernoulli'){\n const obsinfo = n / (thetaHat*(1-thetaHat))\n const fisherinfo = n / (params[0]*(1-params[0]))\n return [obsinfo, fisherinfo, d3.mean(sample)]\n }\n if (dist_type == 'exponential'){\n const obsinfo = n / (d3.mean(sample)**2)\n const fisherinfo = n / (params[0]**2)\n return [obsinfo, fisherinfo, d3.mean(sample)]\n }\n if (dist_type == 'poisson'){\n const obsinfo = n/d3.mean(sample)\n const fisherinfo = n/params[0]\n return [obsinfo, fisherinfo, d3.mean(sample)]\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1591,"value":"sampdistdata = {\n const ngrid = 500 \n const SEthetaHatObs = Math.sqrt(1 / info[0])\n const SEthetaHatExp = Math.sqrt(1 / info[1])\n const theta = params[0];\n const thetaHat = centerAtTrue ? theta : info[2] \n //const thetagrid = d3.range(d3.max([0.01, theta - 15*SEthetaHatExp]), theta + 15*SEthetaHatExp, 30*SEthetaHatExp/ngrid)\n const thetagrid = d3.range(domainSampMLE[0], domainSampMLE[1], (domainSampMLE[1]-domainSampMLE[0])/1000)\n var pdf = thetagrid.map(d => ({par: d, \n sampdistExact: jStat.normal.pdf(d, theta, SEthetaHatExact), \n sampdistObs: jStat.normal.pdf(d, thetaHat, SEthetaHatObs), \n sampdistExp: jStat.normal.pdf(d, theta, SEthetaHatExp)\n }\n )\n ) \n return pdf\n \n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1790,"value":"SEthetaHatExact = {\n if (dist_type == 'bernoulli'){\n return Math.sqrt(params[0]*(1-params[0])/n)\n }\n if (dist_type == 'poisson'){\n return Math.sqrt(params[0]/n)\n }\n if (dist_type == 'exponential'){\n return Math.sqrt(params[0]**2/n)\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1721,"value":"thetaHat = {\n if (dist_type == 'bernoulli'){ \n return d3.mean(sample)\n }\n if (dist_type == 'exponential'){ \n return d3.mean(sample)\n }\n if (dist_type == 'poisson'){\n return d3.mean(sample)\n }\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1728,"value":"SEthetaHatExp = Math.sqrt(1 / info[1])","pinned":false,"hidden":false,"mode":"ojs"},{"id":1750,"value":"domainLike = {\n if (dist_type == 'bernoulli'){\n return [0, 1]\n }\n if (dist_type == 'poisson'){\n return [0, 5]\n }\n if (dist_type == 'exponential'){\n return [0, 5]\n }\n return [d3.max([0.01, thetaHat - 10*SEthetaHatExp]), thetaHat + 10*SEthetaHatExp]\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1760,"value":"domainSampMLE = {\n if (dist_type == 'bernoulli'){\n return [0.001, 0.999]\n }\n return [0, params[0] + 3*Math.sqrt(n/info[1])]\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":572,"value":"#### Code","pinned":false,"hidden":false,"mode":"md"},{"id":1345,"value":"function studenttpdf(x, mu, sigma, df){\n return jStat.studentt.pdf((x-mu)/sigma, df)/sigma\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":1348,"value":"function studenttcdf(x, mu, sigma, df){\n return jStat.studentt.cdf((x-mu)/sigma, df)\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":79,"value":"\nheight = 250\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":86,"value":"width = 800","pinned":false,"hidden":false,"mode":"ojs"},{"id":72,"value":"margin = ({top: 20, right: 20, bottom: 30, left: 40})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1193,"value":"#### Detached plots","pinned":false,"hidden":false,"mode":"md"},{"id":1336,"value":"pltlike = Plot.plot({\n title: \"The likelihood function\",\n color: {\n legend: true,\n domain: [\"Normalized Likelihood\"],\n range: [colors[3]]\n },\n x: {\n label: \"parameter\",\n domain: domainLike\n },\n y: {\n label: \"normalized likelihood\",\n //axis: false,\n //domain: [0,jstat.chisquare.pdf(Math.max(0,params[0]-2), params[0])]\n },\n marks: [\n Plot.line(pdfdata, {x: \"par\", y: \"lik\", stroke : colors[3], strokeWidth: 4}),\n Plot.ruleY([0])\n ]\n })","pinned":false,"hidden":false,"mode":"ojs"},{"id":1601,"value":"pltsamp = Plot.plot({\n title: \"Sampling distribution of ML estimator\",\n color: {\n legend: true,\n //domain: [\"Exact std err\", \"Std err from observed info\", \"Std err from Fisher info\"],\n // range: [colors[1], colors[0], colors[2]]\n domain: [\"Std err from observed info\", \"Std err from Fisher info\"],\n range: [colors[0], colors[2]]\n \n },\n y: {\n label: \"Sampling density\"\n },\n x: {\n label: \"Maximum likelihood estimator\",\n domain: domainSampMLE\n },\n marks: [\n Plot.ruleY([0]),\n Plot.lineY(sampdistdata, {x: \"par\", y: \"sampdistObs\", strokeWidth: 3, stroke: colors[0]}),\n Plot.lineY(sampdistdata, {x: \"par\", y: \"sampdistExp\", strokeWidth: 3, stroke: colors[2]}),\n //Plot.lineY(sampdistdata, {x: \"par\", y: \"sampdistExact\", strokeWidth: 3, stroke: colors[1], strokeDasharray: \"5,5\"})\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W32","stdlib":"1","titling":"auto"}} 2f:"$2e:body"