29:["$","title",null,{"children":"Chatterjee correlation | Toph Tucker | Observable"}] 2e:[] 2c:{"id":"a1ce419a2a2c5d6d","slug":"chatterjee-correlation","title":"Chatterjee correlation","thumbnail":"ec1fca4531609a1ca34fa753f717c17ce31914b30290f180c9919a98ce48cb72","access_level":"public","create_time":"$D2022-01-15T00:16:16.882Z","update_time":"$D2022-07-06T13:48:47.861Z","view_time":null,"first_public_time":"$D2022-07-06T13:52:49.745Z","has_unpublished_edits":null,"fork":{"id":"59a5a1642eb82b95","slug":"chatterjee-correlation","title":"Chatterjee correlation","owner":{"id":"da2a32154638c457","login":"fil","name":"Fil"}},"stars":13,"starred":false,"owner":{"type":"team","id":"97290e936c339d8e","login":"tophtucker","name":"Toph Tucker","avatar_url":"https://avatars-next.observableusercontent.com/3706c31a8f9c59bb716c022c04b9c073144cd1368b78ac85f49139be6c93c34e","tier":"pro_2026"},"version":631,"publish_version":631,"role":null,"authored":false,"authors":[{"type":"individual","id":"0bb2842b01e0c652","login":"tophtucker","name":"Toph Tucker","avatar_url":"https://avatars-next.observableusercontent.com/3706c31a8f9c59bb716c022c04b9c073144cd1368b78ac85f49139be6c93c34e","tier":null},{"type":"individual","id":"dbf69c13a14d2f62","login":"visnup","name":"Visnu Pitiyanuvath","avatar_url":"https://avatars-next.observableusercontent.com/b847c4c49993b1df7599d355f47dc0ad9c2dcf43439ce9e8f3410e6368b3aac9","tier":null}],"members":[{"type":"individual","id":"0bb2842b01e0c652","login":"tophtucker","name":"Toph Tucker","avatar_url":"https://avatars-next.observableusercontent.com/3706c31a8f9c59bb716c022c04b9c073144cd1368b78ac85f49139be6c93c34e","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":3,"collection":{"id":"ee48d2d87f751f74","slug":"sketches","title":"Sketches","owner":{"id":"97290e936c339d8e","login":"tophtucker","name":"Toph Tucker"}},"type":"notebook","body":{"titling":"auto","title":"Chatterjee correlation","cells":[{"id":0,"value":"# Chatterjee correlation\n\nA new coefficient of correlation, Sourav Chatterjee, https://arxiv.org/pdf/1909.10140.pdf (April 2020)\n\nConsider a dataset of points with coordinates (x,y). How close are we to having *y* as a pure and nicely behaved function of *x*? This is what the Chatterjee correlation measures.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":407,"value":"","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":630,"value":"---\n## Made with Plot","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":598,"value":"Plot.plot({\n marks: [\n Plot.ruleX([0]),\n Plot.ruleY([0]),\n Plot.image(data, {\n x: \"pearson\",\n y: \"chatterjee\",\n src: (d) =>\n `data:image/svg+xml,${\n d3\n .select(plot(d.data, plotSize, false))\n .attr(\"xmlns\", d3.namespaces.svg)\n .node().outerHTML\n }`,\n width: plotSize,\n height: plotSize,\n opacity: 0.8\n })\n ],\n x: { nice: true },\n y: { nice: true },\n width,\n height: width\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":522,"value":"---\n## Helpers","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":233,"value":"funcs = [\n [\"sine\", (x) => Math.sin(9 * x)],\n [\"linear\", (x) => x],\n [\"circle\", (x, i) => (i % 2 ? -1 : 1) * Math.sqrt(1 - (2 * x - 1) ** 2)],\n [\"parabola\", (x) => (2 * (x - 0.5)) ** 2]\n // add your own!\n]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":21,"value":"plot = (data, size = 300, axis = true) => Plot.plot({\n marks: [\n Plot.frame({fill: \"rgba(250,250,250,0.8)\", stroke: \"black\"}),\n Plot.dot(data, {\n r: 1.5,\n opacity: 0.9\n })\n ],\n inset: size / 15,\n x: { axis },\n y: { axis },\n nice: true,\n width: size,\n height: size\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":534,"value":"function* noise(data, rnd) {\n for (const [x, y] of data) yield [x + rnd(), y + rnd()];\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":532,"value":"function* sample(fn, N = 40) {\n for (let i = 0; i < N + 1; i++) yield [i / N, fn(i / N, i)];\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":358,"value":"data = d3.cross(funcs, [0, 0.06, 0.12], (func, sigma) => {\n const data = [...noise(sample(func[1]), d3.randomNormal(0, sigma))];\n return {\n func,\n noise: sigma,\n data,\n pearson: pearson(data),\n chatterjee: chatterjee(data)\n };\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":368,"value":"htl.html`

${data.map(d => plot(d.data))}

`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":395,"value":"// pearson\nx = d3.scaleLinear([-0.4, 1], [margin.left, width - margin.right])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":403,"value":"// chatterjee\ny = d3.scaleLinear([-0.6, 1], [height - margin.bottom, margin.top])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":404,"value":"height = width","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":398,"value":"margin = ({top: 60, right: 60, bottom: 60, left: 60})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":478,"value":"plotSize = 80","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":320,"value":"---\n## Chatterjee correlation","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":47,"value":"Note: the chatterjee coefficient is bounded by ${tex`[-\\frac12 + O(\\frac1n), \\frac{n-2}{n+1}]` }.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":27,"value":"chatterjee = (data, { x = (d) => d[0], y = (d) => d[1] } = {}) => {\n // 0. Extract values\n const X = d3.map(data, x);\n const Y = d3.map(data, y);\n const N = X.length;\n\n // 1. rearrange along X\n const I = d3.sort(d3.range(N), (i) => X[i]);\n\n // 2. rank Y\n const rank = d3.rank(Y);\n\n // 3. compute ξ (simple formula with no ties)\n return (\n 1 -\n (3 / (N * N - 1)) *\n d3.sum(d3.pairs(I), ([a, b]) => Math.abs(rank[a] - rank[b]))\n );\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":168,"value":"---\n## Pearson correlation","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":387,"value":"(width > 500 ? tex.block: tex)`${width > 500 ? `corr(X, Y) =` : ''}\\frac\n {\\sum_{i=1}^{N} \\left( X_i - \\mu\\left(X\\right) \\right)\\left( Y_i - \\mu\\left(Y\\right) \\right)}\n {\\sqrt{\\sum_{i=1}^{N} \\left( X_i - \\mu\\left(X\\right) \\right)^2} \n \\sqrt{\\sum_{i=1}^{N} \\left( Y_i - \\mu\\left(Y\\right) \\right)^2}}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":333,"value":"pearson = (data, { x = (d) => d[0], y = (d) => d[1] } = {}) => {\n // 0. Extract values\n const X = d3.map(data, x);\n const Y = d3.map(data, y);\n const N = X.length;\n\n // 1. Calculate covariance\n const cov =\n (1 / (X.length - 1)) *\n d3.sum(X.map((_, i) => (X[i] - d3.mean(X)) * (Y[i] - d3.mean(Y))));\n\n // 2. Divide covariance by product of standard deviations\n return cov / (d3.deviation(X) * d3.deviation(Y));\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W2e","stdlib":"1"}} 2d:"$2c:body" 28:E{"digest":"NEXT_REDIRECT;replace;/@tophtucker/chatterjee-correlation;307;"}