2f:T61e,plot = (interval, level, counts, quantize, domain, ghost, caption, legend) => Plot.plot({ marks: [ // ghost Plot.rectY( quantizedBinRanges(level), Plot.normalizeY("max", { x1: (d) => d[0], // adjust for the 'inclusive' bin count. // todo: figure out how to not have the hanging extra guy at the end... x2: (d, i, a) => d[1] + 1, y: (d, i) => ghost[i], fill: () => "Full dataset", inset: 0.5 }) ), // counts Plot.rectY( quantizedBinRanges(level), Plot.normalizeY("max", { x1: (d) => d[0], // adjust for the 'inclusive' bin count. // todo: figure out how to not have the hanging extra guy at the end... x2: (d, i, a) => d[1] + 1, y: (d, i) => counts[i], fill: () => "Filtered dataset", // fill: (d) => interval[0] < d[1] && d[0] <= interval[1], inset: 0.5 }) ), // filter selection Plot.rectY([interval], { x1: (d) => d[0], x2: (d) => d[1] + 1, y0: null, y1: null, fill: "#748FDE", fillOpacity: 0.15, stroke: "#748FDE", strokeOpacity: 0.5, insetTop: -7 }), Plot.ruleY([0]) ], height, color: { legend, domain: ["Filtered dataset", "Full dataset"], range: ["#000", "#DDD8D4"] }, x: { transform: quantize.invert }, y: { percent: true, tickFormat: "s", label: "% of dataset" }, caption })30:T6fe,// - note: yao found many things confusing; i have notes from july 15 in a bike outline. ('full dataset' should always be visible, maybe side-by-side bars; the gray in particular is hard to see, particularly when covered up; distinction between 'input' and 'output' dimensions (causally), unclear how filtering a dimension interacts with the black bars in the same dimension) // - investigate and debug v strange behavior when one or more of the ranges are zero. // x show the 'ghost' of the full dataset behind the filtered query. // x show the brushed region on the chart. // x debug incorrect filtering behavior. // x can do date using wm range filtering. // x figure out our approach wrt. the maximum symbol and querying for it inclusively // x consider partially highlighting bars corresponding to the selection range // - note: to remove the raw data and work entirely in the quantized domain, we had to record the original scale domains (eg. `distanceDomain`). // todo: 'local' checkbox that is checked if localhost server is accessible // - question: why is there a count of 1 result when querying all zero ranges (which should be fine since they are inclusive), but the only chart that contains a (nonzero) bar is the one for arrival delay, in the final bin? if there is a result, shouldn't each chart contain a bar? // - bug: on mobile, zora observed "RuntimeError: Out of bounds table access (evaluating 'wasm.waveletmatrix32_count_symbol_ranges (retptr, this.wbg_ptr, ptr0, len0, range_lo, range_hi, dims)')" // - why do the distance queries take the longest? i think z=distance, y=delay, x=hour. // - experiment with the alternate approach of defining each "marginal" query as a set of contiguous z-ranges, split from the original bounding box using bigmin/litmax or similar.31:[{"id":"960d0da04ab99d3de4c842ca5728994151095bdc7e621a8ffce2adf45b60158850ff3ea2f93f8777d820353aa531b8e89a5aea3c8c2d831da90e58a5ec1ce37d","href":"https://static.observableusercontent.com/files/960d0da04ab99d3de4c842ca5728994151095bdc7e621a8ffce2adf45b60158850ff3ea2f93f8777d820353aa531b8e89a5aea3c8c2d831da90e58a5ec1ce37d","name":"wm (1).buf","type":"application/octet-stream","lastModified":1688782639059.825,"size":23907512}] 2d:{"title":"Cross-filtering 10 million flights","cells":[{"id":0,"value":"# Cross-filtering 10 million flights\n\nExperimenting with a new approach to the interactive cross-filtering of [10 million flight records](https://observablehq.com/@uwdata/mosaic-cross-filter-flights-10m).","pinned":false,"hidden":false,"mode":"md"},{"id":2436,"value":"Filtered to **${comma(selectedCount)}** / **${comma(wm.len())}** flights.","pinned":false,"hidden":false,"mode":"md"},{"id":2085,"value":"viewof res = Inputs.radio(\n new Map([\n [\"Low\", 4],\n [\"Medium\", 5],\n [\"High\", 6]\n ]),\n {\n value: 5,\n label: \"Resolution\"\n }\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":1455,"value":"plot(\n delay,\n delayLevel,\n delayCounts,\n quantizeDelay,\n delayDomain,\n fullDelayCounts,\n \"Arrival delay (minutes)\",\n true\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":391,"value":"viewof delay = interval(defaultInterval, {\n step: 1,\n value: defaultInterval,\n label: \"Arrival delay\",\n format: (d) =>\n `${fmt(quantizeDelay.invert(d[0]))} ... ${fmt(\n quantizeDelay.invert(d[1])\n )} minutes`\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1460,"value":"plot(\n hour,\n hourLevel,\n hourCounts,\n quantizeHour,\n hourDomain,\n fullHourCounts,\n \"Hour of the day\",\n true\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":385,"value":"viewof hour = interval(defaultInterval, {\n step: 1,\n value: defaultInterval,\n label: \"Hour of day\",\n format: (d) =>\n `${fmt(quantizeHour.invert(d[0]))} ... ${fmt(\n quantizeHour.invert(d[1])\n )} hours`\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":1462,"value":"plot(\n distance,\n distanceLevel,\n distanceCounts,\n quantizeDistance,\n distanceDomain,\n fullDistanceCounts,\n \"Flight distance (miles)\",\n true\n)","pinned":false,"hidden":false,"mode":"ojs"},{"id":388,"value":"viewof distance = interval(defaultInterval, {\n step: 1,\n value: defaultInterval,\n label: \"Distance\",\n format: (d) =>\n `${fmt(quantizeDistance.invert(d[0]))} ... ${fmt(\n quantizeDistance.invert(d[1])\n )} miles`\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":2049,"value":"We can also filter by the row index in the original dataset. This could be useful if, for example, the rows are sorted by time.","pinned":false,"hidden":false,"mode":"md"},{"id":1272,"value":"// We can also filter in the natural ordering of the original dataset.\n// This can be useful as e.g. a time dimension.\nviewof indexRange = interval([0, wm.len() - 1], {\n step: 1,\n value: [0, wm.len() - 1],\n label: \"Row index\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":2454,"value":"
\n
\n
\n
\n\n----\n## Appendix\n\n","pinned":false,"hidden":false,"mode":"md"},{"id":2591,"value":"// 23,907,432: 24mb","pinned":true,"hidden":false,"mode":"ojs"},{"id":1335,"value":"$2f","pinned":true,"hidden":false,"mode":"ojs"},{"id":979,"value":"computeHourCounts = (distance, delay) => {\n const hourRanges = quantizedBinRanges(hourLevel).map((hour) =>\n encodeRange(distance, delay, hour)\n );\n return counts(hourRanges, \"hour\");\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2432,"value":"comma = d3.format(\",.2s\") // format for filtered indicator","pinned":true,"hidden":false,"mode":"ojs"},{"id":2293,"value":"overlaps = (a, b) => a[0] < b[1] && b[0] < a[1] // range a overlaps range b","pinned":true,"hidden":false,"mode":"ojs"},{"id":462,"value":"$30","pinned":true,"hidden":false,"mode":"ojs"},{"id":1683,"value":"fmt = d3.format(\",.2r\") // format for interval slider labels","pinned":true,"hidden":false,"mode":"ojs"},{"id":805,"value":"quantizationLevel = 6 // number of bits per dimension","pinned":true,"hidden":false,"mode":"ojs"},{"id":707,"value":"maxSymbolPerDim = 2 ** quantizationLevel - 1 // maximum value in a particular dimension","pinned":true,"hidden":false,"mode":"ojs"},{"id":1903,"value":"distanceDomain = [0, 4962] // recorded from the original data","pinned":true,"hidden":false,"mode":"ojs"},{"id":1904,"value":"delayDomain = [-60, 180] // recorded from the original data","pinned":true,"hidden":false,"mode":"ojs"},{"id":1906,"value":"hourDomain = [0, 24] // recorded from the original data","pinned":true,"hidden":false,"mode":"ojs"},{"id":814,"value":"dims = 3 // number of dimensions each symbol represents","pinned":true,"hidden":false,"mode":"ojs"},{"id":813,"value":"maxSymbol = 2 ** (quantizationLevel * dims) - 1 // the maximum allowable symbol is dictated by the quantization level and number of dimensions: e.g. 3 bits per level for 3d, and a quantization level of 8 means (3 * 8)-bit = 24 symbols.","pinned":true,"hidden":false,"mode":"ojs"},{"id":155,"value":"quantizeDistance = d3\n .scaleLinear()\n .domain(distanceDomain)\n .rangeRound([0, maxSymbolPerDim])","pinned":true,"hidden":true,"mode":"ojs"},{"id":286,"value":"quantizeDelay = d3\n .scaleLinear()\n .domain(delayDomain)\n .rangeRound([0, maxSymbolPerDim])","pinned":true,"hidden":true,"mode":"ojs"},{"id":296,"value":"quantizeHour = d3\n .scaleLinear()\n .domain(hourDomain)\n .rangeRound([0, maxSymbolPerDim])","pinned":true,"hidden":true,"mode":"ojs"},{"id":1713,"value":"// decode the wavelet matrix containing the quantized 'data cube'\nwm = FileAttachment(\"wm (1).buf\")\n .arrayBuffer()\n .then((data) => rs.WaveletMatrix32.decode(new Uint8Array(data)))","pinned":true,"hidden":false,"mode":"ojs"},{"id":2661,"value":"wm.get(0)","pinned":true,"hidden":false,"mode":"ojs"},{"id":2668,"value":"wm.max_symbol()","pinned":true,"hidden":false,"mode":"ojs"},{"id":2674,"value":"// lib = import(await FileAttachment(\"made-of-bits@3.js\").url())","pinned":true,"hidden":false,"mode":"ojs"},{"id":2710,"value":"// import { lib } from \"2aa3808823902569\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":2713,"value":"import { lib } from \"2aa3808823902569\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":2687,"value":"{\n const n = 25;\n\n console.time(\"wm counts\");\n for (let i = 0; i < n; i++) wm.counts();\n console.timeEnd(\"wm counts\");\n\n console.time(\"js counts\");\n for (let i = 0; i < n; i++) js.counts();\n console.timeEnd(\"js counts\");\n\n console.log(\"\");\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2676,"value":"js = new lib.WaveletMatrix(out, wm.max_symbol())","pinned":true,"hidden":false,"mode":"ojs"},{"id":2664,"value":"out = {\n console.time(\"get\");\n let len = wm.len();\n const out = new Uint32Array(len);\n for (let i = 0; i < len; i++) {\n out[i] = wm.get(i);\n }\n console.timeEnd(\"get\");\n return out;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2678,"value":"{\n const a = wm.counts();\n const b = js.counts();\n for (let i = 0; i < b.length; i++) {\n const cond =\n a.symbol[i] === b[i].symbol &&\n a.start[i] === b[i].start &&\n a.end[i] === b[i].end;\n if (!cond) throw new Error(`non-equal results at index ${i}`);\n }\n console.log(\"\");\n return { a, b };\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2659,"value":"wm.num_levels()","pinned":true,"hidden":false,"mode":"ojs"},{"id":884,"value":"// return bin ranges for a particula dimension that have been quantized to `level` bits. If `level` is the quantization level of the wavelet matrix then we'll return ranges corresponding to a step of 1 in the data cube along this dimension, eg. [[0, 1], [1, 2], ..., [62, 63]]\nquantizedBinRanges = (level) =>\n d3\n .pairs(\n d3.range(\n 0,\n 2 ** quantizationLevel + 1e-6,\n 2 ** (quantizationLevel - level)\n )\n )\n .map(([lo, hi]) => [lo, hi - 1])","pinned":true,"hidden":false,"mode":"ojs"},{"id":946,"value":"encodeRange = (distance, delay, hour) => [\n encode3(distance[0], delay[0], hour[0]),\n encode3(distance[1], delay[1], hour[1]) + 1 // exclusive endpoint but inclusive in each dimension\n]","pinned":true,"hidden":false,"mode":"ojs"},{"id":341,"value":"selectedRange = encodeRange(distance, delay, hour)","pinned":true,"hidden":false,"mode":"ojs"},{"id":1320,"value":"quantizedBinRanges(1)","pinned":true,"hidden":false,"mode":"ojs"},{"id":1346,"value":"delayLevel = res","pinned":true,"hidden":false,"mode":"ojs"},{"id":1356,"value":"distanceLevel = res","pinned":true,"hidden":false,"mode":"ojs"},{"id":1359,"value":"hourLevel = res","pinned":true,"hidden":false,"mode":"ojs"},{"id":2212,"value":"delayCounts = computeDelayCounts(distance, hour)","pinned":true,"hidden":false,"mode":"ojs"},{"id":2216,"value":"fullDelayCounts = computeDelayCounts(defaultInterval, defaultInterval)","pinned":true,"hidden":false,"mode":"ojs"},{"id":457,"value":"computeDelayCounts = (distance, hour) => {\n const delayRanges = quantizedBinRanges(delayLevel).map((delay) =>\n encodeRange(distance, delay, hour)\n );\n return counts(delayRanges, \"delay\");\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2190,"value":"distanceCounts = computeDistanceCounts(delay, hour)","pinned":true,"hidden":false,"mode":"ojs"},{"id":2193,"value":"fullDistanceCounts = computeDistanceCounts(defaultInterval, defaultInterval)","pinned":true,"hidden":false,"mode":"ojs"},{"id":609,"value":"computeDistanceCounts = (delay, hour) => {\n const distanceRanges = quantizedBinRanges(distanceLevel).map((distance) =>\n encodeRange(distance, delay, hour)\n );\n return counts(distanceRanges, \"distance\");\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2204,"value":"hourCounts = computeHourCounts(distance, delay)","pinned":true,"hidden":false,"mode":"ojs"},{"id":2210,"value":"fullHourCounts = computeHourCounts(defaultInterval, defaultInterval)","pinned":true,"hidden":false,"mode":"ojs"},{"id":325,"value":"selectedCount = counts(selectedRange, \"selected\") //[0]","pinned":true,"hidden":false,"mode":"ojs"},{"id":2633,"value":"viewof reload","pinned":true,"hidden":false,"mode":"ojs"},{"id":516,"value":"counts = (ranges, name) => {\n let [startSymbol, endSymbol] = d3.zip(...ranges);\n console.time(name + \": counts\");\n let ret = wm.count({\n startSymbol,\n endSymbol,\n start: indexRange[0],\n end: indexRange[1] + 1,\n masks\n });\n console.timeEnd(name + \": counts\");\n return ret;\n}","pinned":true,"hidden":false,"mode":"ojs"},{"id":2639,"value":"masks = wm.morton_masks_for_dims(3)","pinned":true,"hidden":false,"mode":"ojs"},{"id":2598,"value":"d3.zip(...d3.zip([1, 2, 3], [4, 5, 6]))","pinned":true,"hidden":false,"mode":"ojs"},{"id":910,"value":"defaultInterval = [0, maxSymbolPerDim]","pinned":true,"hidden":false,"mode":"ojs"},{"id":728,"value":"height = 175","pinned":true,"hidden":false,"mode":"ojs"},{"id":381,"value":"import { interval } from \"@mootari/range-slider\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":302,"value":"import { encode3, decode3x, decode3y, decode3z } from \"393eae81fa26b317\"","pinned":true,"hidden":false,"mode":"ojs"},{"id":2479,"value":"import { rs, viewof reload } from \"d4d70888d178542b\"","pinned":true,"hidden":false,"mode":"ojs"}],"files":"$W31","stdlib":"1","titling":"auto"}