2b:null 2c:["$","title",null,{"children":"Template for exploring a csv file | ee2dev | Observable"}] 31:T63a,function rowConversions(row, i, colNames) { // This function is invoked for each row, being passed an object representing the current row (d), the index (i) // starting at zero for the first non-header row, and the array of column names. If the returned value is null or // undefined, the row is skipped and will be omitted from the array returned by dsv.parse; otherwise, the returned // value defines the corresponding row object --> see https://github.com/d3/d3-dsv#dsv_parse // --1-- return row as is // return row; // --2-- //row["ISO_3166_1"] = +row["ISO_3166_1"]; row["population_2018"] = +row["population_2018"].replaceAll(",",""); row["population_2019"] = +row["population_2019"].replaceAll(",",""); row["population_2020"] = +row["population_2020"].replaceAll(",",""); row["country_alt"] = row["country_alt"].replaceAll(",",";"); row["population_change_2019"] = +row["population_change_2019"]; row["population_change_2020"] = +row["population_change_2020"].replaceAll(" %","") / 100; row["population_change_2020_net"] = +row["population_change_2020_net"].replaceAll(",",""); row["density"] = +row["density"].replaceAll(",",""); row["area"] = +row["area"].replaceAll(",",""); row["migrants_net"] = +substituteNA(row["migrants_net"].replaceAll(",","")); row["fert_rate"] = +substituteNA(row["fert_rate"]); row["age_median"] = +substituteNA(row["age_median"]); row["pop_urban_perc"] = +substituteNA(row["pop_urban_perc"].replaceAll(" %","")) / 100; row["world_share"] = +row["world_share"].replaceAll(" %","") / 100; return row; }32:T5d3,data = { // convenience definition let obj = {}; obj.original = inputData_original; obj.parsed = inputData; obj.processed = processedData; /* obj.attributes2 = {}; for (const property in attributesTypes) { obj.attributes2[property] = attributesTypes[property]; } */ obj.attributes = {}; obj.attributes.all = Object.keys(processedData[0]); const nominalAttributes = Object.keys(processedData[0]).filter(d => type(d) === "nominal"); let defaultKeys = Object.keys(processedData[0]).filter(key => (type([key]) !== "nominal") || new Set(processedData.map(ele => ele[key])).size <= 20); let defaultColor; if (defaultKeys.filter((key, i) => type(key) === "nominal" && i > 1).length >= 1) { defaultColor = defaultKeys.filter((key, i) => type(key) === "nominal" && i > 1)[0]; } else if (defaultKeys.filter((key, i) => type(key) === "nominal").length >= 1) { defaultColor = defaultKeys.filter((key, i) => type(key) === "nominal")[0]; } else { defaultColor = nominalAttributes[0]; } if (defaultKeys.length < 4) { defaultKeys = Object.keys(processedData[0]); } obj.defaultKeys = defaultKeys; obj.defaultX = defaultKeys[0]; obj.defaultY = defaultKeys[1]; obj.defaultColor = defaultColor; obj.defaultSize = defaultKeys.filter(d => type(d) === "quantitative"); obj.nominalAttributes = nominalAttributes; obj.quantitativeAttributes = Object.keys(processedData[0]).filter(d => type(d) === "quantitative"); return obj; }33:T698,defaultAttributes = { // set default attributes for each visualization let vis = []; let defaults = {}; let attributeSet = new Set(Object.keys(attributes)); // vis 1: Scatterplot with circles, size, color defaults.x = {}; defaults.x.options = Object.keys(attributes); defaults.x.value = findDefaultAttribute(attributeSet, ["temporal", "quantitative", "ordinal","nominal"]); defaults.y = {}; defaults.y.options = Object.keys(attributes); defaults.y.value = findDefaultAttribute(attributeSet, ["quantitative", "ordinal","nominal"]); defaults.color = {}; defaults.color.options = Object.keys(attributes).filter(d => attributes[d] === "nominal" || attributes[d] === "ordinal"); defaults.color.value = findDefaultAttribute(attributeSet, ["ordinal","nominal"]); defaults.size = {}; defaults.size.options = Object.keys(attributes).filter(d => attributes[d] === "quantitative"); defaults.size.value = findDefaultAttribute(attributeSet, ["quantitative"]); vis.push(defaults); attributeSet = new Set(Object.keys(attributes)); defaults = {}; // vis 2: Histogram with linked scatterplot defaults.bar = {}; defaults.bar.options = Object.keys(attributes); defaults.bar.value = findDefaultAttribute(attributeSet, ["temporal", "quantitative", "ordinal","nominal"]); defaults.x = {}; defaults.x.options = Object.keys(attributes); defaults.x.value = findDefaultAttribute(attributeSet, ["temporal", "quantitative", "ordinal","nominal"]); defaults.y = {}; defaults.y.options = Object.keys(attributes); defaults.y.value = findDefaultAttribute(attributeSet, ["quantitative", "ordinal","nominal"]); vis.push(defaults); return vis; }34:[{"id":"4e1ea0463f6c6c98887de3e2d07ffecf83476ff111f736875f8d96384d48214bcc13409cf57e9f69da3faafdd9a9561071c587d27e42deb45a50011aaaa3e824","href":"https://static.observableusercontent.com/files/4e1ea0463f6c6c98887de3e2d07ffecf83476ff111f736875f8d96384d48214bcc13409cf57e9f69da3faafdd9a9561071c587d27e42deb45a50011aaaa3e824","name":"countries7.csv","type":"text/csv","lastModified":1618130179851.071,"size":59919}] 2f:{"id":"0e45156bcf47f6be","slug":"template-for-exploring-a-csv-file","title":"Template for exploring a csv file","thumbnail":"cb0e9c533e5b8f3a46afc981d1c5c9ca53e39987b2576023c6448053b7cdc8a7","access_level":"public","create_time":"$D2021-04-11T08:36:19.851Z","update_time":"$D2021-04-11T16:22:50.144Z","view_time":null,"first_public_time":"$D2021-04-11T09:18:22.858Z","fork":{"id":"65a26cacb2539eac","slug":"exploring-a-csv-file-with-vega-lite","title":"Exploring a csv file with vega-lite","owner":{"id":"e6e1e2a0da500bb6","login":"ee2dev","name":""}},"stars":3,"starred":false,"owner":{"type":"team","id":"e6e1e2a0da500bb6","login":"ee2dev","name":"","avatar_url":"https://avatars-next.observableusercontent.com/c41b514d64cff7415ffc87106c620e16cefe80b6d853fb3a05e19a000c3612c0","tier":"starter_2024"},"version":1231,"publish_version":1231,"role":null,"authored":false,"authors":[{"type":"individual","id":"9ba5e0b68d3e7972","login":"ee2dev","name":"","avatar_url":"https://avatars-next.observableusercontent.com/c41b514d64cff7415ffc87106c620e16cefe80b6d853fb3a05e19a000c3612c0","tier":null}],"members":[{"type":"individual","id":"9ba5e0b68d3e7972","login":"ee2dev","name":"","avatar_url":"https://avatars-next.observableusercontent.com/c41b514d64cff7415ffc87106c620e16cefe80b6d853fb3a05e19a000c3612c0","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"subscribed":null,"forks":3,"collection":{"id":"941be320460f1d0a","slug":"vega-lite","title":"vega-lite","owner":{"id":"e6e1e2a0da500bb6","login":"ee2dev","name":""}},"type":"notebook","body":{"title":"Template for exploring a csv file","cells":[{"id":0,"value":"md`# Template for exploring a csv file\n\nThis is a template for exploring a csv file:\n- data should be in [tidy format](https://observablehq.com/@uwdata/tidy-data-in-javascript)\n- dates should be in *\"YYYY-MM-DD\"* format\n- default type conversions for *dates, nominal, quantitative*\n- all default conversions and the type *ordinal* can be overridden manually` ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":597,"value":"heading(\" 👉 Select data source\", 1, 9);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":35,"value":"viewof file = fileInput({\n initialValue: FileAttachment(\"countries7.csv\").blob(),\n accept: \".csv\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":604,"value":"heading(\" 👉 Choose delimiter\", 2, 9);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":43,"value":"viewof delimiter = Select([\", (comma)\", \"; (semicolon)\", \"| (pipe)\"])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":49,"value":"heading(\" 🧐 Check if custom parsing is required\", 3, 9);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":173,"value":"inputData_original = {\n const psv = d3.dsvFormat(delimiter.charAt(0));\n let myFile = await Files.text(file); ;\n return psv.parse(myFile);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":293,"value":"heading(\" 🤓 Specify custom parsing (if required)\", 4, 9);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":52,"value":"inputData = {\n const psv = d3.dsvFormat(delimiter.charAt(0));\n let myFile = await Files.text(file); \n \n // Need to process file before parsing ?\n // myFile = removeLines(myFile, 1); // remove x lines from the beginning\n // myFile = substituteNA(myFile) // delete dots in column names\n\n // need conversion? update function rowConversions()\n return defaultData ? psv.parse(myFile, rowConversions) : psv.parse(myFile, d3.autoType); \n // return psv.parse(myFile);\n // return psv.parse(myFile, d3.autoType);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":57,"value":"$31","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":260,"value":"heading(\" 🧐 Check if the data looks right?\", 5, 9)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":72,"value":"Table(inputData)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":630,"value":"heading(\" 🤓 Further process your data (if applicable)\", 6, 9)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":14,"value":"processedData = {\n if (defaultData) {\n return inputData.map(d => { \n delete d[\"ISO_3166_1\"]; \n delete d[\"population_2018\"]; \n delete d[\"population_2019\"]; \n delete d[\"population_change_2019\"]; \n delete d[\"population_change_2020_net\"];\n delete d[\"flag\"];\n return d; \n });\n }\n else {\n // return getRandomSubarray(inputData, 100); // take a sample of 100 records from original data\n return inputData;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":636,"value":"heading(\" 🧐 Verify data types (note that ordinal is never set by default)\", 7, 9)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":546,"value":"viewof attributesTypes = getAttributeTypes() ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":685,"value":"heading(\" 💾 Optional: save your data (if data has been processed)\", 8, 9)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":12,"value":"DOM.download(serialize(processedData), \"output.csv\", \"Download CSV\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":641,"value":"heading(\" 📊📈✨ Explore your data \", 9, 9)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1199,"value":"{\n vl.vega.timeFormatLocale(locale); // show dates in German \n vl.vega.formatLocale(locale); // show numbers with German groupings/ separators\n \n const plot = vl.markCircle({tooltip: true}) \n .data(processedData)\n .encode(\n vl.x().field(\"density\").type(type(\"density\")), \n vl.y().field(\"area\").type(type(\"area\")).scale({ type: \"sqrt\"}), \n );\n \n return plot.render();\n // return html`
${JSON.stringify(plot.toObject(), 0, 2)}
`; // format JSON data\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":298,"value":"md`## Appendix`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":301,"value":"md`### Parsing and processing functions`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":162,"value":"function substituteNA(file) {\n file = file.replaceAll(\"N.A.\",\"\");\n return file;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":310,"value":"function substituteNA2(value) {\n if (value === \"\") {\n return NaN;\n } else if (value === \"N.A.\") {\n return NaN;\n } else {\n return value;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1223,"value":"function getRandomSubarray(arr, size) {\n let shuffled = arr.slice(0), i = arr.length, min = i - size, temp, index;\n while (i-- > min) {\n index = Math.floor((i + 1) * Math.random());\n temp = shuffled[index];\n shuffled[index] = shuffled[i];\n shuffled[i] = temp;\n }\n return shuffled.slice(min);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":730,"value":"md`### Data`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":335,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":721,"value":"attributes = {\n let obj = {};\n let arrayIndexes = new Set(Array.from(attributesTypes.keys(), i => \"\" + i));\n let props = Object.keys(attributesTypes).filter(k => !arrayIndexes.has(k));\n for (let property of props) {\n obj[property] = attributesTypes[property];\n }\n return obj;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":750,"value":"cardinality = {\n let c = {};\n for (let key of Object.keys(attributes)) {\n c[key] = (attributes[key] === \"nominal\" || attributes[key] === \"ordinal\") \n ? new Set(processedData.map(ele => ele[key])).size\n : Infinity; \n } \n return c;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1229,"value":"categories = {\n let obj = {};\n let categorical_attributes = \n Object.keys(attributes).filter(key => (attributes[key] === \"nominal\" || attributes[key] === \"ordinal\"));\n for (let key of categorical_attributes) {\n obj[key] = processedData.map(ele => ele[key]).filter((v, i, a) => a.indexOf(v) === i);\n }\n return obj;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":738,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":589,"value":"defaultData = (inputData_original[0].ISO_3166_1 === \"4\") // hack to check if new data is loaded","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":17,"value":"md`### Libraries and functions`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":745,"value":"function compare(a, b) {\n \n /*if ((attributes.a === \"nominal\" || attributes.a === \"ordinal\") && cardinality.a < cardinality.b) {\n return 1;\n }\n else if \n return -1;\n */\n if (cardinality[a] < cardinality[b]) {\n return -1;\n }\n if (cardinality[a] > cardinality[b]) {\n return 1;\n }\n // a must be equal to b\n return 0;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":998,"value":"function toHtml (text) {\n return html`${text}`\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":914,"value":"function findDefaultAttribute (attributeSet, typeOrder) {\n const attributeArray = Array.from(attributeSet); \n for (let type of typeOrder) {\n if (attributeArray.filter(d => attributes[d] === type).length) {\n const attr = attributeArray.filter(d => attributes[d] === type).sort(compare)[0];\n attributeSet.delete(attr);\n return attr;\n }\n }\n return null;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":120,"value":"function type(field) {\n const value = processedData[0][field];\n if (typeof (value) === \"number\") {\n return \"quantitative\";\n }\n else if (value instanceof Date) {\n return \"temporal\";\n }\n else { \n return \"nominal\";\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":548,"value":"function getAttributeTypes() {\n let views = [];\n let names = [];\n let defaultType;\n let types = [\n ['quantitative', 'quantitative'],\n ['nominal', 'nominal'],\n ['ordinal', 'ordinal'],\n ['temporal', 'temporal'],\n ];\n let reorderedTypes;\n for (let key of data.attributes.all) {\n defaultType = type(key);\n reorderedTypes = types.sort((a,b) => a[0] === defaultType ? -1 : 0);\n views.push([menu(key, reorderedTypes)]);\n names.push([key]); \n }\n return inputsGroup(views, names);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":83,"value":"locale = {\n return {\n \"decimal\": \",\",\n \"thousands\": \".\",\n \"grouping\": [3],\n \"currency\": [\"€\", \"\"],\n \"dateTime\": \"%a %b %e %X %Y\",\n \"date\": \"%d.%m.%Y\",\n \"time\": \"%H:%M:%S\",\n \"periods\": [\"AM\", \"PM\"],\n \"days\": [\"Sonntag\", \"Montag\", \"Dienstag\", \"Mittwoch\", \"Donnerstag\", \"Freitag\", \"Samstag\"],\n \"shortDays\": [\"So\", \"Mo\", \"Di\", \"Mi\", \"Do\", \"Fr\", \"Sa\"],\n \"months\": [\"Januar\", \"Februar\", \"März\", \"April\", \"Mai\", \"Juni\", \"Juli\", \"August\", \"September\", \"Oktober\", \"November\", \"Dezember\"],\n \"shortMonths\": [\"Jan\", \"Feb\", \"Mär\", \"Apr\", \"Mai\", \"Jun\", \"Jul\", \"Aug\", \"Sep\", \"Okt\", \"Nov\", \"Dez\"]\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":244,"value":"import {fileInput} from \"@mbostock/file-input-with-initial-value\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":19,"value":"import {serialize} from \"@palewire/saving-csv\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":948,"value":"import {Select, Table} from \"@observablehq/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":593,"value":"import {heading} from '@ee2dev/formatting-in-observable'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":541,"value":"import {menu} from \"@ee2dev/dom-utilities\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":535,"value":"import {inputsGroup} from '@bumbeishvili/input-groups'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1029,"value":"import {vl} from '@vega/vega-lite-api-v5'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":21,"value":"d3 = require(\"d3\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"}],"files":"$W34","stdlib":"1","titling":"auto"}} 30:"$2f:body"