29:null 2a:["$","title",null,{"children":"Data Wrangling in Javascript | Alex Cao | Observable"}] 2f:[{"id":"ea5ddea7b3448d7bb7361256f1ca6fd5cced5c2776ccdf12a9d625ca97f23a632e010e5e11acc1398a2c2c101a98c074c481acedb1c2db6d64c89d7510cbe5cc","href":"https://static.observableusercontent.com/files/ea5ddea7b3448d7bb7361256f1ca6fd5cced5c2776ccdf12a9d625ca97f23a632e010e5e11acc1398a2c2c101a98c074c481acedb1c2db6d64c89d7510cbe5cc","name":"headway_by_date_hour@1.txt","type":"text/plain","lastModified":1607482817810.582,"size":25854},{"id":"279d52133964bee2546cc5964d0630526b4d8dba88ad7179f3713d7cde9617053b724925b6b4ec4e0f9d22bb410a4dc57727409ae0d496cef8a1035adddc31c9","href":"https://static.observableusercontent.com/files/279d52133964bee2546cc5964d0630526b4d8dba88ad7179f3713d7cde9617053b724925b6b4ec4e0f9d22bb410a4dc57727409ae0d496cef8a1035adddc31c9","name":"wait_time_min@1.csv","type":"text/csv","lastModified":1653588331033.029,"size":126146}] 2d:{"id":"647f1c84b9ad977c","slug":"data-wrangling-in-javascript","title":"Data Wrangling in Javascript","thumbnail":"bd2a0a15894b226c3f811db1e593ec569d202d3179191766b21c1aa4e3f33d4e","access_level":"public","create_time":"$D2020-06-09T18:56:57.405Z","update_time":"$D2023-07-28T20:14:16.915Z","view_time":null,"first_public_time":"$D2021-01-28T21:31:30.446Z","has_unpublished_edits":null,"fork":false,"stars":7,"starred":false,"owner":{"type":"team","id":"2e5d4d2fdce202fc","login":"caocscar","name":"Alex Cao","avatar_url":"https://avatars-next.observableusercontent.com/b0a8f76660298ded21f964c4a15751d294befb2473172e51ed66b8c43405f98e","tier":"starter_2024"},"version":952,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"7e33a195e62bba19","login":"caocscar","name":"Alex Cao","avatar_url":"https://avatars-next.observableusercontent.com/b0a8f76660298ded21f964c4a15751d294befb2473172e51ed66b8c43405f98e","tier":null}],"members":[{"type":"individual","id":"7e33a195e62bba19","login":"caocscar","name":"Alex Cao","avatar_url":"https://avatars-next.observableusercontent.com/b0a8f76660298ded21f964c4a15751d294befb2473172e51ed66b8c43405f98e","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":3,"collection":null,"type":"notebook","body":{"title":"Data Wrangling in Javascript","cells":[{"id":155,"value":"md`# Data Wrangling in Javascript`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":238,"value":"md`My repo of some examples of common data wrangling techniques`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":328,"value":"toc()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":0,"value":"data = [\n {'date': '10-19','sex': 'M','count': 1456},\n {'date': '10-19','sex': 'M','count': 150},\n {'date': '10-19','sex': 'M','count': 544},\n {'date': '10-29','sex': 'M','count': 2351},\n {'date': '10-29','sex': 'F','count': 649},\n {'date': '10-29','sex': 'F','count': 636},\n {'date': '10-30','sex': 'F','count': 747},\n {'date': '10-30','sex': 'B','count': 678},\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":714,"value":"wait_time_min = FileAttachment(\"wait_time_min@1.csv\").csv({typed: true})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":244,"value":"headway = FileAttachment(\"headway_by_date_hour@1.txt\").csv({typed:true});","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":340,"value":"fam = ['alex', 'nancy', 'lilly', 'forest', 'neko']","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":373,"value":"segments = [\n {segment: 1, auto_ct: 7725, healthy_ct: 15216, pct: 0.508},\n {segment: 2, auto_ct: 86535, healthy_ct: 106618, pct: 0.812},\n {segment: 3, auto_ct: 4933, healthy_ct: 7099, pct: 0.695},\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":398,"value":"engage = [\n {takeover: 'planned', type: 'boy', count: 1},\n {takeover: 'planned', type: 'girl', count: 2},\n {takeover: 'unplanned', type: 'boy', count: 3},\n {takeover: 'unplanned', type: 'girl', count: 4},\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":482,"value":"missing = Array(0)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":935,"value":"{\n let abc = Array.from(d3.flatRollup(data, v => {\n return {\n 'date': v[0].date,\n 'sex': v[0].sex,\n 'ct': v.length,\n }\n }, d => d.date, d => d.sex), ([_, __, value]) => {_, value});\n return abc\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":589,"value":"{\n let abc = d3.flatRollup(data, v => v.length, d => d.date, d => d.sex);\n return abc.map(([date, sex, count]) => ({date, sex, count}))\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":603,"value":"cohort = d3.groups(vehicleHealth, d => d.date)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":669,"value":"cohort.map(d => {\n let row = {};\n row.date = d[0];\n row.value = d[1];\n return row;\n})\n// equivalent to the below\n// cohort.map(d => ({date: d[0], value: d[1]}));","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":719,"value":"# Binning Data","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":723,"value":"bins = d3.bin().thresholds([-10, -5, 0])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":726,"value":"buckets = bins(wait_time_min.map(d => d.actual_wait_time_min));","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":730,"value":"binStats = buckets.map(d => {return {min: d3.min(d), max: d3.max(d), ct: d.length, pct: d.length / wait_time_min.length}})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":367,"value":"md`## Convert to d3.stack format`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":377,"value":"// use long to wide algorithm; accomodates missing cell values\nstack = {\n const category1 = ['planned', 'unplanned'];\n const category2 = ['boy', 'girl'];\n const column = \"takeover\"\n let group = d3.rollup(engage, v => d3.sum(v, d => d.count), d => d[column], d => d.type);\n let stackInput = [];\n for (let cat1 of category1) {\n let row = {};\n row[column] = cat1;\n if (group.has(cat1)) {\n const innermap = group.get(cat1);\n category2.forEach(d => { row[d] = innermap.has(d) ? innermap.get(d) : 0 });\n } else {\n category2.forEach(d => { row[d] = 0 });\n }\n stackInput.push(row);\n }\n return d3.stack()\n .keys(category2)(stackInput)\n// .map(arr => arr.map(([y0, y1]) => [y0, y1, arr.index]))\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":336,"value":"md`## Create Combinations (itertools)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":344,"value":"combos = {\n const famSorted = fam.sort();\n let data = [];\n for (let i=0; i d3.sum(v, d => d.count), d => d.date, d => d.sex), ([key,value]) => {\n let row = {};\n row['date'] = key;\n const categories = new Set(data.map(d => d.sex));\n categories.forEach(d => { row[d] = value.has(d) ? value.get(d) : 0 });\n return row;\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":252,"value":"md`Example 2`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":257,"value":"head = Array.from(d3.rollup(headway, v => d3.sum(v, d => d.count), d => d.period, d => d.bin), ([key,value]) => {\n let row = {};\n row['key'] = key;\n const categories = new Set(headway.map(d => d.bin));\n categories.forEach(d => { row[d] = value.has(d) ? value.get(d) : 0 });\n return row;\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":268,"value":"headoverall = {\n let row = {};\n row['key'] = 'overall';\n for (let col of Object.keys(head[0]).slice(1)) {\n row[col] = d3.sum(head, d => d[col])\n }\n return head.concat(row)\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":223,"value":"md`## Assign new data to existing object`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":194,"value":"// option 1\nwidedata = Object.assign(wide, {columns: Object.keys(wide[0])});","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":436,"value":"// option 2\nwidedata2 = {\n wide['columns'] = Object.keys(wide[0])\n return wide\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":85,"value":"md`## Wide to Long Wrangling`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":183,"value":"longdata = {\n let data = [];\n for (let d of widedata) {\n for (let col of widedata.columns.slice(1)) {\n if (d[col] > 0) {\n let row = {};\n row[widedata.columns[0]] = d[widedata.columns[0]];\n row['gender'] = col;\n row['count'] = d[col];\n data.push(row);\n }\n }\n };\n return data;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":273,"value":"md`## Initialize Empty Map`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":276,"value":"emptyMap = {\n let data = new Map;\n for (let day of weekdayService) {\n data.set(day, new Map);\n for (let hr of hours) {\n data.get(day).set(hr, 0);\n }\n }\n return data\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":283,"value":"md`## Initialize Empty Array of Objects`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":291,"value":"emptyArray = {\n let data = [];\n for (let day of weekdayService) {\n for (let hr of hours) {\n data.push({weekday: day, hour: hr, pickup: 0});\n }\n }\n return data\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":447,"value":"md`## Alternate values in Array`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":449,"value":"alternation = {\n let array1 = [1, 2, 3, 4, 5];\n let array2 = ['a', 'b', 'c', 'd'];\n return array1.map((d, i) => i < array2.length ? [d, array2[i]] : d).flat();\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":522,"value":"fleet = {\n let fleetStatus = new Map;\n d3.range(3).forEach(n => fleetStatus.set(n, 0));\n let fleet = d3.rollup(vehicleHealth, v => v.length, d => d.health);\n for (let [key, value] of fleet) {\n fleetStatus.set(key, value);\n }\n return fleetStatus;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":538,"value":"vehicleHealth = [\n {may_id: \"marty\", health: 1, date: new Date(2022,1,15)},\n {may_id: \"mchale\", health: 0, date: new Date(2022,1,14)},\n {may_id: \"mojo\", health: 1, date: new Date(2022,1,14)},\n {may_id: \"molly\", health: 1, date: new Date(2022,1,13)},\n {may_id: \"morocca\", health: 1, date: new Date(2022,1,15)},\n]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":771,"value":"## Output 2+ Metrics with 2+ Groupby Groups","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":576,"value":"{\n let aggData = d3.flatRollup(data, v => [d3.sum(v, d => d.count), d3.count(v, d => d.count)], d => d.date, d => d.sex)\n aggData = aggData.map(d => d.flat())\n return aggData.map(([lat, lng, requests, completed]) => ({lat, lng, requests, completed}));\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":775,"value":"## Sorting with NaN","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":777,"value":"// https://stackoverflow.com/questions/29017476/alternative-to-d3-ascending-d3-descending-for-sorting-undefined-values-to-b\nfunction ascendingWithNaN(a, b) {\n return isNaN(b) ? (isNaN(a) ? 0 : 1) // put NaN at the end\n : a > b ? 1\n : a < b ? -1\n : a <= b ? 0\n : NaN;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":798,"value":"function descendingWithNaN(a, b) {\n return isNaN(b) ? (isNaN(a) ? 0 : -1) // put NaN at the end\n : a > b ? -1\n : a < b ? 1\n : a <= b ? 0\n : NaN;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":780,"value":"xyz = [15, 77, 354, 17, NaN, 35, 17, NaN, 8, 28]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":782,"value":"xyz.sort(ascendingWithNaN)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":894,"value":"## Time Filtering","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":911,"value":"endDate = new Date(2022, 10, 9)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":896,"value":"movingWindow = d3.timeDay.every(2).range(d3.timeDay.offset(endDate, -14), endDate)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":904,"value":"d3.timeDay.offset(endDate, -7)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":899,"value":"abc = movingWindow.filter(d => d >= d3.timeDay.offset(endDate, -7))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":922,"value":"fgh = movingWindow.map(d => pastWeekFilter(d))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":322,"value":"md`# Constants`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":556,"value":"","pinned":true,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":303,"value":"weekdayService = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri']; ","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":306,"value":"hours = d3.range(6,20);","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":166,"value":"md`# Libraries`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":13,"value":"d3 = require(\"d3@7\", \"d3-array@^3\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":325,"value":"import {toc} from \"@harrystevens/toc\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W2f","stdlib":"1","titling":"auto"}} 2e:"$2d:body"