2a:null 2b:["$","title",null,{"children":"Unaggregating Cloud Watch Metrics | Tom Larkworthy | Observable"}] 31:T5e2,

Boto3 call to collect the Metric Data

${md` ~~~python import boto3 import json PERIOD = 60 # seconds cw = boto3.client('cloudwatch', region_name='eu-central-1') metric = { "Namespace": "Taktile", "MetricName": "latency", "Dimensions": [ {"Name":"path","Value":"..."}, {"Name":"method","Value":"POST"}, {"Name":"status","Value":"200"}, ] } resp = cw.get_metric_data( MetricDataQueries=[ { "Id": "min", "MetricStat": { "Metric": metric, "Period": PERIOD, "Stat": "Minimum", }, "ReturnData": True, }, { "Id": "max", "MetricStat": { "Metric": metric, "Period": PERIOD, "Stat": "Maximum", }, "ReturnData": True, }, { "Id": "sum", "MetricStat": { "Metric": metric, "Period": PERIOD, "Stat": "Sum", }, "ReturnData": True, }, { "Id": "count", "MetricStat": { "Metric": metric, "Period": PERIOD, "Stat": "SampleCount", }, "ReturnData": True, } ], StartTime="2025-12-03T20:43Z", EndTime="2025-12-04T20:43Z", ) print(json.dumps(resp, indent=2, default=str)) ~~~`}

32:T5be,generateSyntheticFromStats = ( rows, { periodMs = 60000, rng = Math.random } = {} ) => { const all = []; for (const d of rows ?? []) { if (!d || !d.time) continue; const bin = []; const count = Math.max(1, Math.round(d.count ?? 1)); const t0 = d.time.getTime(); const t1 = t0 + periodMs; const randTime = () => new Date(t0 + rng() * (t1 - t0)); const push = (v) => { if (v == null || !Number.isFinite(v)) return; bin.push({ time: randTime(), value: v, bucketTime: d.time }); }; const min = d.min ?? d.avg ?? d.max ?? 0; const max = d.max ?? d.avg ?? d.min ?? min; const sum = d.sum != null && Number.isFinite(d.sum) ? d.sum : d.avg != null && Number.isFinite(d.avg) ? d.avg * count : ((min + max) / 2) * count; if (count === 1) { push(min); } else if (count === 2) { push(min); push(max); } else if (count === 3) { const mean = sum / count; push(min); push(mean); push(max); } else if (count === 4) { const innerSum = sum - min - max; const mid = innerSum / 2; push(min); push(mid); push(mid); push(max); } else { let remaining = count; push(min); push(max); remaining -= 2; for (let i = 0; i < remaining; i++) { push(sampleFairFromStats([d.min, d.max, d.sum, d.count])); } } all.push(...bin); } return all; }33:[{"id":"796451bd6a37e846c1dfdc05ae8f6183403366b64feec9039d51ef545696dc9a877f290330e74a5513882e6ac79991942eb5c9ad26f0ba577a82589bd99273f2","href":"https://static.observableusercontent.com/files/796451bd6a37e846c1dfdc05ae8f6183403366b64feec9039d51ef545696dc9a877f290330e74a5513882e6ac79991942eb5c9ad26f0ba577a82589bd99273f2","name":"logs-insights-results (27)@1.csv","type":"text/csv","lastModified":1764937893308.178,"size":72095},{"id":"b4aecb7f7da3614f7c43042e343a14064a2591b65b236c9cdf0632f4953482101613f656d8f3a2e23436e55c17b47b8e370b4cb376756a4b1cebb38cf47707dc","href":"https://static.observableusercontent.com/files/b4aecb7f7da3614f7c43042e343a14064a2591b65b236c9cdf0632f4953482101613f656d8f3a2e23436e55c17b47b8e370b4cb376756a4b1cebb38cf47707dc","name":"data@4.json","type":"application/json","lastModified":1764928372782.219,"size":158550},{"id":"6718c84577234c69aca9260072ece1c6e12b7db6771ff4e4bff1794face216993740abf0cf643732fa41493cc4e16a768401487225a75c9af9dfb352dce74332","href":"https://static.observableusercontent.com/files/6718c84577234c69aca9260072ece1c6e12b7db6771ff4e4bff1794face216993740abf0cf643732fa41493cc4e16a768401487225a75c9af9dfb352dce74332","name":"image@2.png","type":"image/png","lastModified":1764921090588.25,"size":52646}] 2e:{"id":"e24a01d86b590220","slug":"unaggregating-cloudwatch-metrics","title":"Unaggregating Cloud Watch Metrics","thumbnail":"9fc1ba7ae3ab121321e03813b4899402bf9a4f9209b277d82d5a01fafd036667","access_level":"public","create_time":"$D2025-12-04T20:20:43.234Z","update_time":"$D2025-12-07T09:31:10.356Z","view_time":null,"first_public_time":"$D2025-12-06T16:36:06.551Z","fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"7db5ed2b0697d645","login":"tomlarkworthy","name":"Tom Larkworthy","avatar_url":"https://avatars-next.observableusercontent.com/47327a8bc1966f2186dcb3ebf4b7ee6e4e7ab9a5c2a07405aff57200ea778f71","tier":"starter_2024"},"version":781,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"5215f6ec4a999d40","login":"tomlarkworthy","name":"Tom Larkworthy","avatar_url":"https://avatars-next.observableusercontent.com/47327a8bc1966f2186dcb3ebf4b7ee6e4e7ab9a5c2a07405aff57200ea778f71","tier":null}],"members":[{"type":"individual","id":"5215f6ec4a999d40","login":"tomlarkworthy","name":"Tom Larkworthy","avatar_url":"https://avatars-next.observableusercontent.com/47327a8bc1966f2186dcb3ebf4b7ee6e4e7ab9a5c2a07405aff57200ea778f71","tier":null,"role":"editor"},{"type":"team","id":"7db5ed2b0697d645","login":"tomlarkworthy","name":"Tom Larkworthy","avatar_url":"https://avatars-next.observableusercontent.com/47327a8bc1966f2186dcb3ebf4b7ee6e4e7ab9a5c2a07405aff57200ea778f71","tier":"starter_2024","role":"editor"}],"comments":[],"custom_thumbnail":true,"trashed":false,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"title":"Unaggregating Cloud Watch Metrics","cells":[{"id":0,"value":"# Unaggregating Cloud Watch Metrics\n\n\n[Visnu Pitiyanuvath](https://observablehq.com/@visnup) of Observable advocates _\"never aggregate your datapoints\"_. The outliers _are_ the signal! His talk made me realize that we, the developer community, could do better, *especially* the Cloud providers. ","pinned":false,"hidden":false,"mode":"md"},{"id":611,"value":"

The \"DataViz Dashboards for Developers\" talk that inspired me

\n\n

","pinned":false,"hidden":false,"mode":"html"},{"id":614,"value":"\n\nEver since that talk my heart sinks when I assemble a Cloud Watch dashboard. Yes, they work and are useful, but why are they just _lines_?... after some though I realized that, well, maybe they do not need to be? \n\nMaybe there is a way to take _Visnu_'s lessons and apply them to my day-to-day work. Can we invert Cloud Watch metrics back to something close to the original unaggregated datapoints? Turns out, YES WE CAN!","pinned":false,"hidden":false,"mode":"md"},{"id":385,"value":"## My Uninspiring Cloud Watch Dashboard","pinned":false,"hidden":false,"mode":"md"},{"id":232,"value":" $\"image@2.png\"$ ","pinned":false,"hidden":false,"mode":"html"},{"id":383,"value":"## The same data after unaggregating it","pinned":false,"hidden":false,"mode":"md"},{"id":182,"value":"synthetic_example = Plot.plot({\n width: 640,\n height: 300,\n marginLeft: 50,\n marks: [\n Plot.dot(resampled, {\n x: \"time\",\n y: \"value\",\n r: 3,\n fill: foreground,\n opacity: 0.1\n })\n // Plot.dot(tidy_data, {\n // x: \"time\",\n // y: \"avg\",\n // r: 3,\n // fill: \"red\"\n // })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":470,"value":"viewof animate = Inputs.toggle({\n label: \"resample\",\n value: false\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":448,"value":"Thats the same data source that displays the Cloud Watch metric above. LOOK AT THE DETAILS. I can see that just before 12pm, traffic stopped, and then there was pent up demand unleashed at 12pm. You _cannot_ see traffic volume in the Cloud Watch chart of latency! I can also see that the biggest outlier is over 20 seconds! Again I can't usually see that in Cloud Watch. THERE IS SO MUCH MAGNIFISCENT DETAIL REVEALED.\n\nIt's the same metric! That detail was there, you just have to visualize it.","pinned":false,"hidden":false,"mode":"md"},{"id":457,"value":"## How it works\n\nA Cloud Watch metric precisely tracks the _max_, _min_, _count_ and _sum_ into temporal bins ([AWS docs](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/Statistics-definitions.html)). The main idea is to use those statitics to generate something close to the real dataset.\n\nIf the sample count in a bin is 1, 2 or 3, you know all the values *exactly*, that's essentially unaggregated points. If you have more than 4 points in a bin, you still know 2 of those points exactly (the min and max), plus you know how many more points there are. We use sampling to regenerate the remaining points, using the statistics to estimate of the skewness. That's what produce the above graph.\n\nIs it accurate though?","pinned":false,"hidden":false,"mode":"md"},{"id":657,"value":"## Ground truth","pinned":false,"hidden":false,"mode":"md"},{"id":381,"value":"\n\nI have the real datapoints (below) gathered from logs. There are some differences between the unaggregated data and the ground truth data. The ground truth shows a stronger floor latency distribution. But still, the unaggregated graph did a pretty amazing job at representing the real distribution. Its shocking to me that this level of detail was already present in my existing Cloud Watch metrics.","pinned":false,"hidden":false,"mode":"md"},{"id":365,"value":"ground_truth = Plot.plot({\n width: 640,\n height: 300,\n marginLeft: 50,\n x: {\n type: \"time\"\n },\n marks: [\n Plot.dot(truth, {\n x: \"time\",\n y: \"value\",\n r: 3,\n fill: foreground,\n opacity: 0.1\n })\n ]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":682,"value":"## ⚠️ High throughput Metrics\n\nFor metric streams under 1 request per second, Cloud Watch metrics streams do not lose much detail because their 4 dimensional statistics are nearly enough to represent the raw dataset in 5 second bins. At higher rates though the synthetically sampled data dominates and obscures.","pinned":false,"hidden":false,"mode":"md"},{"id":548,"value":"---","pinned":false,"hidden":false,"mode":"md"},{"id":566,"value":"Original blog on [Observablehq](https://observablehq.com/@tomlarkworthy/unaggregating-cloudwatch-metrics). Exported to a hermetic local-first form with [Lopecode](https://github.com/tomlarkworthy/lopecode)","pinned":false,"hidden":false,"mode":"md"},{"id":575,"value":"exporter()","pinned":false,"hidden":false,"mode":"ojs"},{"id":539,"value":"### Appendix: Data","pinned":false,"hidden":false,"mode":"md"},{"id":337,"value":"$31","pinned":false,"hidden":false,"mode":"md"},{"id":21,"value":"data = FileAttachment(\"data@4.json\").json()","pinned":false,"hidden":false,"mode":"ojs"},{"id":23,"value":"tidy_data = {\n const byTime = new Map();\n const idToField = {\n min: \"min\",\n max: \"max\",\n sum: \"sum\",\n count: \"count\"\n };\n for (const { Id, Timestamps = [], Values = [] } of data.MetricDataResults) {\n const field = idToField[Id];\n if (!field) continue;\n for (let i = 0; i < Timestamps.length; i++) {\n const t = new Date(Timestamps[i]);\n const key = t.getTime();\n let entry = byTime.get(key);\n if (!entry) {\n entry = { time: t };\n byTime.set(key, entry);\n }\n entry[field] = Values[i];\n }\n }\n return Array.from(byTime.values()).sort((a, b) => a.time - b.time);\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":251,"value":"resampled = {\n while (animate) {\n yield new Promise((resolve) => {\n setTimeout(() => {\n resolve(generateSyntheticFromStats(tidy_data, { periodMs: 60_000 }));\n }, 200);\n });\n }\n yield generateSyntheticFromStats(tidy_data, { periodMs: 60_000 });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":361,"value":"truth = (await FileAttachment(\"logs-insights-results (27)@1.csv\").csv()).map(\n (d) => ({\n time: new Date(d[\"@timestamp\"]).getTime(),\n value: Number.parseInt(d.integration_latency)\n })\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":546,"value":"### Appendix: Code","pinned":false,"hidden":false,"mode":"md"},{"id":179,"value":"$32","pinned":true,"hidden":false,"mode":"ojs"},{"id":218,"value":"sampleFairFromStats = ([min, max, sum, count], rng = Math.random) => {\n const a = min;\n const b = max;\n if (\n a == null ||\n b == null ||\n !Number.isFinite(a) ||\n !Number.isFinite(b) ||\n a === b\n )\n return a ?? b ?? 0;\n const n = Math.max(1, count ?? 1);\n const mean = sum != null && Number.isFinite(sum) ? sum / n : (a + b) / 2;\n if (!Number.isFinite(mean) || mean <= a || mean >= b)\n return a + rng() * (b - a);\n const Fm = (mean - a) / (b - a);\n const u = rng();\n if (u < Fm) {\n return a + Math.sqrt(u * (b - a) * (mean - a));\n } else {\n return b - Math.sqrt((1 - u) * (b - a) * (b - mean));\n }\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":702,"value":"unzip = async (attachment) =>\n await new Response(\n (await attachment.stream()).pipeThrough(new DecompressionStream(\"gzip\"))\n ).blob()","pinned":true,"hidden":false,"mode":"ojs"},{"id":590,"value":"foreground = {\n const styles = getComputedStyle(document.body);\n return styles.getPropertyValue(\"--theme-foreground\").trim() || \"black\";\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":604,"value":"background = {\n const styles = getComputedStyle(document.body);\n return styles.getPropertyValue(\"--theme-background\").trim() || \"black\";\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":564,"value":"### Appendix: developer tools","pinned":false,"hidden":false,"mode":"md"},{"id":577,"value":"import { exporter } from \"@tomlarkworthy/exporter-2\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":16,"value":"robocoop","pinned":false,"hidden":false,"mode":"ojs"},{"id":9,"value":"import { robocoop } from \"@tomlarkworthy/robocoop-2\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":408,"value":"lite_youtube_css","pinned":false,"hidden":false,"mode":"ojs"},{"id":406,"value":"import { lite_youtube_css } from \"@tomlarkworthy/lite-youtube-embed\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":775,"value":"import { md } from \"@tomlarkworthy/editable-md\"","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W33","stdlib":"1","titling":"auto"}} 2f:"$2e:body"