26:null 27:["$","title",null,{"children":"IAT355: Do you have all the data you need? Working with data in d3 and Vega | drlynb | Observable"}] 2c:T47b,md `## Join two data sources There are several ways to join two data sources: within a single data file (preprocessing) and within your code (dynamic).
WHich you choose depends on how tighly linked your data are. If you are just adding an attribute to an existing data record, it's probably easier to join it outside the code, by simply adding a column in your Excel or data file. One example of this might be if you have a set of property data, (location, price,type of building) and you want to add another attribute such as year_built, you can just add that to each data record. Often, however, you are in the situation where you don't want to add more to your initial data. An example would be when you have neighbourhood data relating to aggregate income and age that are not specifically related to a building, although the building is in the neighbourhood. Also, you may not have access to actually change the data files you are using. IN that case we can connect the data by either using the data transform{} function in Vega, (an explicit join), or just identify a common field name to connect by a brush (implicit).`2d:[] 2a:{"id":"2db20badf456517c","slug":"iat355-do-you-have-all-the-data-you-need-working-with-data-in-d/3","title":"IAT355: Do you have all the data you need? Working with data in d3 and Vega","thumbnail":"530b572629f6a8b672eeeb2cb15423a6531d6cd85eff7191566dfd2064ae34c3","access_level":"public","create_time":"$D2019-04-02T22:48:26.947Z","update_time":"$D2019-04-15T18:03:01.455Z","view_time":null,"first_public_time":"$D2019-04-02T23:06:50.969Z","fork":{"id":"1b9e8248bcf5a86b","slug":"iat355-do-you-have-all-the-data-you-need-working-with-data-in-d/2","title":"IAT355: Do you have all the data you need? Working with data in d3 and Vega","owner":{"id":"4a0762a387ec748f","login":"drlynb","name":""}},"stars":0,"starred":false,"owner":{"type":"team","id":"4a0762a387ec748f","login":"drlynb","name":"","avatar_url":"https://avatars-next.observableusercontent.com/dd242441d5e4ed40bfe5b09336f04460a3a4974825e2a9cb9088ade14a49fc15","tier":"starter_2024"},"version":502,"publish_version":502,"role":null,"authored":false,"authors":[{"type":"individual","id":"90a5d909f6de8793","login":"drlynb","name":"","avatar_url":"https://avatars-next.observableusercontent.com/dd242441d5e4ed40bfe5b09336f04460a3a4974825e2a9cb9088ade14a49fc15","tier":null}],"members":[{"type":"individual","id":"90a5d909f6de8793","login":"drlynb","name":"","avatar_url":"https://avatars-next.observableusercontent.com/dd242441d5e4ed40bfe5b09336f04460a3a4974825e2a9cb9088ade14a49fc15","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":4,"collection":{"id":"71568a9af940022a","slug":"introduction-to-visual-analytics-spring-2019","title":"Introduction to VIsual Analytics Spring 2019","owner":{"id":"4a0762a387ec748f","login":"drlynb","name":""}},"type":"notebook","body":{"title":"IAT355: Do you have all the data you need? Working with data in d3 and Vega","cells":[{"id":0,"value":"md`# IAT355: Do you have all the data you need? Working with data in d3 and Vega\nI am covering some of the basic in how we can work with data in the Observable/d3/Vega environment.\nYou should look at Mike Bostocks's Introduction to Data workbook before reading this. `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":11,"value":"md `## Modules, Libraries and Data Files\nImport the libraries and data files you need. In this workbook I will be using a number of datasets and showing you how to merge, join and/or reference them in the same visualizations.\nBecause the VPD file is large I am hosting it in my own webspace: the Census data is a gist.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":6,"value":"d3 = require(\"d3@5\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":252,"value":"vegalite = require(\"@observablehq/vega-lite@0.1\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"embed = require('vega-embed@3')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":46,"value":"md `Data files are Vancouver Census DAta from 2011 by neighbourhood (source City of Vancouver Open Data) and non-violent crimes from 2008-2017 (source VPD).`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":32,"value":"vanPop = d3.csv(\"https://www.sfu.ca/~lyn/data/VanPop2011.csv\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":38,"value":"smallvanCrime = d3.csv(\"https://www.sfu.ca/~lyn/data/vpdcrime.csv\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":49,"value":"md `### Get the data into shape\nAll csv data are imported as text so I need to clean them. Thre are several ways:\n

iterate through and change text fields to appropriate formats;\n
change format while importing the d3.csv() function\n
use the d3.map() function\n

\nExamples follow of each.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":124,"value":"md ` ### 1. Iterate through and change text fields to appropriate formats`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":83,"value":"md `

Use the + operator to iteratively change the text Population string in the VanPop array to a number. \n
use the same technique to change the year string in the smallVanCrime array to a number.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":57,"value":"vanPopCleaned = {\n let p1 = 0;\n let i=0;\n \n console.log (\"Starting\"); \n for (i=0;id3.csv function parameters`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":82,"value":"vanPopCleaned2 = \n d3.csv(\"https://www.sfu.ca/~lyn/data/VanPop2011.csv\", function(d) {\n return {\n Area: d.Area,\n Age: d.Age,\n Gender : d.Gender,\n Population: +d.Population\n };\n })\n ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":103,"value":"md `### Use the array.map() to change all types in the array`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":113,"value":"vanCrime2 = smallvanCrime.map( d => {\n var vc = d;\n vc.YEAR = +d.YEAR;\n return vc;\n})\n ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":135,"value":"popMax = {\n let i=0;\n let maxPop=0;\n for (i=0;i maxPop) maxPop = vanPopCleaned[i].Population;\n }\n return(maxPop)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":462,"value":"md `Let's make an array of population totlas jsut to check my values in the later aggregation.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":471,"value":"mp = vanPopCleaned2.filter(function(nhood) {\n return nhood.Area == \"Mount Pleasant\"});\n ","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":479,"value":"rp = vanPopCleaned2.filter(rp => rp.Area == \"Riley Park\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":485,"value":"rpPop = calculatePop(rp)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":487,"value":"mpPop = calculatePop(mp)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":482,"value":"function calculatePop(neighbourhood)\n{\n let pop=0;\n let i=0;\n for (i=0; i < neighbourhood.length;i++)\n {\n pop += neighbourhood[i].Population;\n }\n return pop;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":466,"value":"function SumPops(areas)\n{\n let i=0;\n let j=0;\n let nPop=0;\n let name=\"\";\n let AreaPops = [];\n let currentArea = [];\n for(i=0;i

\n\"data\": \n    {\n    \"name\": \"crime\",\n    \"values\": vanCrimeCleaned\n    },\n    \"transform\": [\n    { \n      \"lookup\": \"NEIGHBOURHOOD\", \n      \"from\": \n      {\n      \"data\": {\"values\": vanPopCleaned}, \n      \"key\": \"Area\", \n      \"fields\": [\"Population\"]\n      }\n    }],\n

\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":388,"value":"mpC = vanCrimeCleaned.filter(mp => mp.NEIGHBOURHOOD == \"Mount Pleasant\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":393,"value":"mpP = vanPopCleaned.filter(mp => mp.Area == \"Mount Pleasant\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":332,"value":"popCrimeView = embedWithSpecificVersions({\n \"name\": \"crimePop\",\n \"data\": \n {\n \"name\": \"crime\",\n \"values\": vanCrimeCleaned\n },\n \"transform\": [\n { \n \"lookup\": \"NEIGHBOURHOOD\", \n \"from\": \n {\n \"data\": {\"values\": populations}, \n \"key\": \"Area\", \n \"fields\": [\"Population\"]\n }\n }],\n \"mark\": {\n \"type\": \"bar\",\n \"clip\": true\n },\n \"encoding\": {\n \"x\": {\n \"aggregate\": \"count\",\n \"field\": \"Offense\",\n \"type\": \"quantitative\",\n \"scale\": {\n \"zero\": false,\n \"domain\": [0,80000]\n }\n },\n \"y\": {\n \"field\": \"NEIGHBOURHOOD\",\n \"type\": \"nominal\",\n \"scale\": {\"rangeStep\": 15}\n },\n \"color\": \n { \n \"type\": \"quantitative\",\n \"field\": \"Population\",\n \"scale\": {\n //\"domain\": [0,9000000,18000000],\n \"round\": true,\n \"type\": \"linear\",\n //\"range\": [\"#8AB9ED\", \"#acbdbb\",\"#FDB813\"]\n }\n }\n }\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":268,"value":"md `I copied the trellis code from the Bostock examples but it's not working yet. What is going on? `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":281,"value":"md `Now let's try the copied trellis code`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":282,"value":"vega_explicit({\n \"name\": \"trellis_crime\",\n \"data\": {\"values\": vanCrime2},\n \"mark\": {\n \"type\": \"point\",\n \"clip\": true\n },\n \"columns\": 2,\n \"encoding\": {\n \"facet\": {\n \"field\": \"NEIGHBOURHOOD\",\n \"type\": \"ordinal\",\n \"sort\": {\"op\": \"count\", \"field\": \"NEIGHBOURHOOD\"}\n },\n \"x\": {\n \"aggregate\": \"count\",\n \"field\": \"NEIGHBOURHOOD\",\n \"type\": \"quantitative\",\n \"scale\": {\n \"zero\": false,\n \"domain\": [0,9000]\n }\n },\n \"y\": {\n \"field\": \"Offense\",\n \"type\": \"nominal\",\n \"scale\": {\"rangeStep\": 12}\n },\n \"color\": {\"field\": \"Offense Type\", \"type\": \"nominal\"}\n },\n \"resolve\": {\"axis\": {\"x\": \"independent\", \"y\": \"independent\"}}\n})\n\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":293,"value":"vega_explicit = {\n const [Vega, VegaLite] = await Promise.all([\n require(\"vega@3/build/vega.min.js\"),\n require(\"vega-lite@3/build/vega-lite.min.js\")\n ]);\n return async spec => {\n const div = document.createElement(\"div\");\n const view = new Vega.View(Vega.parse(VegaLite.compile(spec).spec));\n await view.initialize(div).runAsync();\n return div;\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":374,"value":"embedWithSpecificVersions = require.alias({\n \"vega\": \"vega@5.3.1\",\n \"vega-lite\": \"vega-lite@3.0.0\"\n})(\"vega-embed@4\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W2d","stdlib":"1","titling":"auto"}} 2b:"$2a:body"