2f:null 30:["$","title",null,{"children":"Smoothing with simple moving average vs. binomial filter | Toph Tucker | Observable"}] 35:T509,md`In response, [Steven Wittens writes](https://twitter.com/unconed/status/1251228671345598465): >"Make each point the average of the last 7" .... this is a box filter. If it has odd length, it won't dampen the nyquist frequency. This creates graphs that seem like they contain more signal than they do. >Use binomial filters instead, i.e. \`[1, 2, 1]\` or \`[1, 6, 15, 20, 15, 6, 1]\` >Example: If I have the signal \`[1 -1 1 -1 1 -1]\` and convolve with \`[1 1 1]\`, I get \`[0 1 -1 1 -1 0]\` (noise in the middle). With \`[1 2 1]\` I get \`[1, 0, 0, 0, 0, 0, -1]\` (no noise in the middle). I was delighted that someone could look at what I was calling an SMA and bring some proper signal-processing theory to bear on it. The last and first time I’d seen any mention of the Nyquist frequency was probably his own excellent talk [The Pixel Factory](http://acko.net/files/gltalks/pixelfactory/online.html#26), slides 26 – 31. I don’t _really_ understood convolution, but I think it’s like: slide a filter over a signal and multiply the corresponding entries. So with the binomial filter \`[1, 2, 1]\` and the alternating signal \`[1, -1, 1, -1, 1, -1]\`, map each entry to the sum of one times the previous entry, two times the current entry, and one times the next entry:`36:T662,chart = series => { const stacked = layout === "Stacked"; const h = 20; const top = 15; const padding = 5; const height = stacked ? top + series.length * (h + padding) : top + h + padding; const len = series[0].data.length; const x = d3 .scaleLinear() .domain([0, len]) .range([0, len * 10]); const y = d3 .scaleLinear() .domain(d3.extent(series[0].data)) .range([h, 0]); const line = d3 .line() .x((_, i) => x(i)) .y(d => y(d)) .defined(d => d !== undefined); const colorFn = d3 .scaleOrdinal() .domain([0, 1, 2]) .range(["black", "red", "blue"]); const color = stacked ? () => "black" : colorFn; const preview = "[" + series[0].data.slice(0, 6).join(",") + (series[0].data.length > 6 ? "…" : "") + "]"; return svg``; }37:[] 33:{"id":"c78ace25d81a3dba","slug":"smoothing-with-simple-moving-average-vs-binomial-filter","title":"Smoothing with simple moving average vs. binomial filter","thumbnail":"7d2e89ae92bf1533f6635cd8292abc0220effb43f89e7c2740b9ec07950d6aa5","access_level":"public","create_time":"$D2020-04-17T20:35:37.376Z","update_time":"$D2020-04-21T03:02:34.025Z","view_time":null,"first_public_time":"$D2020-04-21T02:50:54.488Z","has_unpublished_edits":null,"fork":false,"stars":6,"starred":false,"owner":{"type":"team","id":"97290e936c339d8e","login":"tophtucker","name":"Toph Tucker","avatar_url":"https://avatars-next.observableusercontent.com/3706c31a8f9c59bb716c022c04b9c073144cd1368b78ac85f49139be6c93c34e","tier":"pro_2026"},"version":811,"publish_version":811,"role":null,"authored":false,"authors":[{"type":"individual","id":"0bb2842b01e0c652","login":"tophtucker","name":"Toph Tucker","avatar_url":"https://avatars-next.observableusercontent.com/3706c31a8f9c59bb716c022c04b9c073144cd1368b78ac85f49139be6c93c34e","tier":null}],"members":[{"type":"individual","id":"0bb2842b01e0c652","login":"tophtucker","name":"Toph Tucker","avatar_url":"https://avatars-next.observableusercontent.com/3706c31a8f9c59bb716c022c04b9c073144cd1368b78ac85f49139be6c93c34e","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":{"id":"e94c1734e845649b","slug":"coronavirus","title":"Coronavirus","owner":{"id":"97290e936c339d8e","login":"tophtucker","name":"Toph Tucker"}},"type":"notebook","body":{"title":"Smoothing with simple moving average vs. binomial filter","cells":[{"id":30,"value":"md`# Smoothing with simple moving average vs. binomial filter\n\n[My animation](https://observablehq.com/@tophtucker/building-up-to-john-burn-murdochs-coronavirus-tracker) imitating [John Burn-Murdoch’s coronavirus tracker](https://www.ft.com/coronavirus-latest) smooths the data with what I would call a seven-day simple moving average; it maps each point to the average of the last \\`n\\`, here seven. (When there are fewer than seven points to average, this implementation returns \\`undefined\\`.)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":23,"value":"sma = n => arr => arr.map((d, i, arr) => d3.mean(arr.slice(i - n + 1, i + 1)))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":588,"value":"chart([{ data: ex }, { data: sma(7)(ex), code: \"sma(7)\" }])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":0,"value":"$35","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":638,"value":"// no normalization!\n_convolve = filter => signal => {\n const arr = [];\n // offset centers the filter window on the current index of the signal\n const offset = Math.floor(filter.length / 2);\n // loop over each entry in the signal\n for (let i = 0; i < signal.length; i++) {\n let sum = 0;\n for (let j = 0; j < filter.length; j++) {\n const offsetI = i + j - offset;\n // skip out-of-bounds entries\n if (offsetI >= 0 && offsetI < signal.length) {\n // sum the products of the filter entries and signal entries\n sum += signal[i + j - offset] * filter[j];\n }\n }\n arr.push(sum);\n }\n return arr;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":255,"value":"md`The results match Steven’s examples, anyway:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":52,"value":"_convolve([1, 1, 1])([1, -1, 1, -1, 1, -1])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":99,"value":"_convolve([1, 2, 1])([1, -1, 1, -1, 1, -1])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":702,"value":"md`But as it stands it’ll amplify signals; it’s summing the weighted entries but not dividing by the sum of the weights:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":703,"value":"_convolve([1, 2, 1])([1, 1, 1])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":728,"value":"md`So I think we have to normalize, but I’m kinda just spitballing:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":142,"value":"// with normalization!\nconvolve = filter => signal => {\n const arr = [];\n const offset = Math.floor(filter.length / 2);\n for (let i = 0; i < signal.length; i++) {\n let numerator = 0,\n denominator = 0;\n for (let j = 0; j < filter.length; j++) {\n const offsetI = i + j - offset;\n if (offsetI >= 0 && offsetI < signal.length) {\n numerator += signal[i + j - offset] * filter[j];\n denominator += filter[j];\n }\n }\n arr.push(numerator / denominator);\n }\n return arr;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":733,"value":"md`This no longer quite matches Steven’s examples:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":735,"value":"convolve([1, 1, 1])([1, -1, 1, -1, 1, -1])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":737,"value":"convolve([1, 2, 1])([1, -1, 1, -1, 1, -1])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":721,"value":"md`But it otherwise behaves as I’d expect and want.\n\nNote how, as Steven warned, odd-numbered SMA periods (or box filters) stay jagged:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":208,"value":"{\n const arr = alt(14);\n const str = String(arr);\n return chart([\n { data: arr },\n { data: sma(2)(arr), code: \"sma(2)\" },\n { data: sma(3)(arr), code: \"sma(3)\" },\n { data: sma(4)(arr), code: \"sma(4)\" },\n { data: sma(5)(arr), code: \"sma(5)\" },\n { data: convolve([1, 1, 1])(arr), code: \"convolve([1,1,1])\" },\n { data: convolve([1, 2, 1])(arr), code: \"convolve([1,2,1])\" },\n {\n data: convolve(binomialCoef(6))(arr),\n code: `convolve([${binomialCoef(6)}])`\n }\n ]);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":176,"value":"md`Steven’s two examples, \\`[1, 2, 1]\\` and \\`[1, 6, 15, 20, 15, 6, 1]\\`, are rows 2 and 6 of Pascal’s triangle; we can compute row \\`n\\` like this:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":105,"value":"// calculates one row of pascal's triangle, i.e. the binomial coefficients, n choose 0 thru n choose n\n// https://en.wikipedia.org/wiki/Pascal's_triangle#Calculating_a_row_or_diagonal_by_itself\nbinomialCoef = n => {\n const arr = [1];\n while (arr.length <= n) {\n const k = arr.length;\n arr.push((arr[k - 1] * (n + 1 - k)) / k);\n }\n return arr;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":438,"value":"md`See how binomial filters compare to box filters of the same length:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":421,"value":"viewof length = slider({\n min: 0,\n max: 10,\n step: 1,\n value: 2,\n format: v => `[${binomialCoef(v).join(\", \")}]`\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":424,"value":"viewof layout = radio({ options: [\"Stacked\", \"Overlaid\"], value: \"Stacked\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":360,"value":"html`

\n ${data.map(\n ({ key, series }) =>\n html`

${key}

\n ${chart(series)}\n

`\n )}\n

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":793,"value":"md`It’s kinda ethnographically striking that the basic top-of-Google literature on simple moving averages ([Wikipedia](https://en.wikipedia.org/wiki/Moving_average), [Investopedia](https://www.investopedia.com/terms/s/sma.asp)) doesn’t mention box filters or convolution. There are [few pages that mention both](https://www.google.com/search?q=%22box+filter%22+%22simple+moving+average%22). The Wikipedia page for [finite impulse response](https://en.wikipedia.org/wiki/Finite_impulse_response) gets into it.\n\nTo-do: animate filter sliding across signal…`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":173,"value":"md`---\n\n## Appendix\n\nAlternate, equivalent, more “functional” (?) implementations of \\`convolve\\` and \\`binomialCoef\\`:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":47,"value":"convolveAlt = filter => signal =>\n signal.map((_, i) => {\n const [numerator, denominator] = filter.reduce(\n ([numerator, denominator], f, j) => {\n const offsetI = i - Math.floor(filter.length / 2) + j;\n return offsetI >= 0 && offsetI < signal.length\n ? [numerator + f * signal[offsetI], denominator + f]\n : [numerator, denominator];\n },\n [0, 0]\n );\n return numerator / denominator;\n })","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":131,"value":"binomialCoefAlt = n =>\n Array(n)\n .fill(true)\n .reduce(\n arr => (\n arr.push((arr[arr.length - 1] * (n + 1 - arr.length)) / arr.length), arr\n ),\n [1]\n )","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":199,"value":"$36","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":332,"value":"data = Array.from(newCases.entries())\n .map(([key, map]) => ({\n key,\n data: Array.from(map.values()).slice(-30)\n }))\n .map(({ key, data }) => ({\n key,\n series: [\n { data },\n {\n code: `convolve([${binomialCoef(length)}])`,\n data: convolve(binomialCoef(length))(data)\n },\n {\n code: `convolve([${Array(length + 1).fill(1)}])`,\n data: convolve(Array(length + 1).fill(1))(data)\n }\n ]\n }))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":584,"value":"newCases = getCovidDataGrouped({ statistic: \"positiveIncrease\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":749,"value":"ex = Array.from(newCases.get(\"MA\").values()).slice(-30)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":656,"value":"alt = n => {\n const arr = [];\n for (let i = 0; i < n; i++) {\n arr.push(Math.sign(-(i % 2) + .5));\n }\n return arr;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":325,"value":"import { getCovidDataGrouped } from \"@tophtucker/covid-tracking-project-data\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":419,"value":"import { slider, radio } from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":25,"value":"d3 = require(\"d3@5\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W37","stdlib":"1","titling":"auto"}} 34:"$33:body"