2c:null 2d:["$","title",null,{"children":"Visualizing Eurostat grid data using Three.js & D3 | Joe Davies | Observable"}] 32:T515,{ renderer.setSize(viz_width, viz_height); renderer.setPixelRatio(devicePixelRatio); try { while (true) { renderer.render(scene, camera); let html_render = html`

${renderer.domElement}

${tooltip_state.label}

25 km² population grid of Europe

`; yield html_render; } } finally { renderer.dispose(); } }33:T560,viewof color_interpolation = colorInterpolatorPicker({ title: "Color ramp", height: 300, value: "Turbo", interpolators: [ { name: "BrBG" }, { name: "PRGn" }, { name: "PiYG" }, { name: "PuOr" }, { name: "RdBu" }, { name: "RdGy" }, { name: "RdYlBu" }, { name: "RdYlGn" }, { name: "Spectral" }, { name: "Blues" }, { name: "Greens" }, { name: "Greys" }, { name: "Oranges" }, { name: "Purples" }, { name: "Reds" }, { name: "Viridis" }, { name: "Cividis" }, { name: "Inferno" }, { name: "Magma" }, { name: "Plasma" }, { name: "Cool" }, { name: "CubehelixDefault" }, { name: "BuGn" }, { name: "BuPu" }, { name: "GnBu" }, { name: "OrRd" }, { name: "PuBuGn" }, { name: "PuBu" }, { name: "PuRd" }, { name: "RdPu" }, { name: "YlGnBu" }, { name: "YlGn" }, { name: "YlOrBr" }, { name: "YlOrRd" }, { name: "Rainbow" }, { name: "Sinebow" }, { name: "Turbo" } ] })34:T4b0,html`

When faced with the challenge of visualizing a 1km² population grid of Europe whilst rendering everything client-side, my initial attempt was to use a 2D HTML Canvas and call fillRect() for each grid cell. This method however simply couldnt handle the quantity of cells we wanted to show, bearing in mind that a 1km grid of europe has over 2 million cells.

Next up was PixiJS - a 2D WebGL renderer designed for game developers. This also struggled to handle the sheer amount of points we were trying to render. After looking around, it seemed that Three.js was capable of handling huge amounts of points, so I decided to go in that direction, even though we did not need any 3D capability at all. There were other alternatives available such as potree - a point-cloud viewer, or REGL - a lower level library. However although highly performant, they offered less funcionality when compared with Three.js, which will prove useful later on in development when we manipulate the cell attributes and add features such as raycasting.

`35:T4df,points = { let pointsGeometry = new THREE.Geometry(); let colors = []; //let xScale = d3.scaleLinear().domain(d3.extent(cells.map(c => c.position[0]))).range([-1,1]); //let yScale = d3.scaleLinear().domain(d3.extent(cells.map(c => c.position[1]))).range([-1,1]); for (let cell of cells) { // Set vector coordinates from data //let coords = [xScale(cell.position[0]),yScale(cell.position[1])] let coords = [cell.position[0], cell.position[1]]; // TODO: find a cleaner way of converting EPSG 3035 to Vector3 //let vectorCoords = new THREE.Vector3(coords[0],coords[1],0) let vectorCoords = new THREE.Vector3( coords[0] / 100000 - 4, coords[1] / 100000 - 3.5, 0 ); // in Threejs, colors and vertices are added to the geometry object in separate, corresponding arrays pointsGeometry.vertices.push(vectorCoords); let hex = color_interpolation(colorScale(cell.value)); cell.color = hex; //for tooltip colors.push(new THREE.Color(hex)); } pointsGeometry.colors = colors; let pointsMaterial = new THREE.PointsMaterial({ size: point_size, sizeAttenuation: true, vertexColors: THREE.VertexColors }); return new THREE.Points(pointsGeometry, pointsMaterial); }36:[{"id":"b54c87b17a6c4171fefa009f6f00a47194fe64b787ff8c71ede0b4d9912f1dfa2cfc67990fb51e1054f6900d47f4463660fddba113e316bf1d051f5b1d1dd9c4","href":"https://static.observableusercontent.com/files/b54c87b17a6c4171fefa009f6f00a47194fe64b787ff8c71ede0b4d9912f1dfa2cfc67990fb51e1054f6900d47f4463660fddba113e316bf1d051f5b1d1dd9c4","name":"pop_5km.csv","type":"text/csv","lastModified":1580674418528.598,"size":3032980}] 30:{"id":"e0c2cdc3943539fd","slug":"visualizing-eurostat-grid-data-using-three-js-d3","title":"Visualizing Eurostat grid data using Three.js & D3","thumbnail":"d9be72d4cc0d5419999fa8cdbfef0f6defb374e481d61e1b1bf17dc70ba62cb9","access_level":"public","create_time":"$D2020-02-02T10:57:15.529Z","update_time":"$D2022-04-01T16:25:17.046Z","view_time":null,"first_public_time":"$D2020-02-07T15:03:26.636Z","fork":false,"stars":62,"starred":false,"owner":{"type":"team","id":"25fee90bb47fd709","login":"joewdavies","name":"Joe Davies","avatar_url":"https://avatars-next.observableusercontent.com/9b3334fa93f66c399554670429d5728ef2f24d2d28e3933c6c7e1fef7ff8eb7e","tier":"starter_2024"},"version":1013,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"dc51b4f7240f0553","login":"joewdavies","name":"Joe Davies","avatar_url":"https://avatars-next.observableusercontent.com/9b3334fa93f66c399554670429d5728ef2f24d2d28e3933c6c7e1fef7ff8eb7e","tier":null}],"members":[{"type":"individual","id":"dc51b4f7240f0553","login":"joewdavies","name":"Joe Davies","avatar_url":"https://avatars-next.observableusercontent.com/9b3334fa93f66c399554670429d5728ef2f24d2d28e3933c6c7e1fef7ff8eb7e","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":true,"trashed":false,"subscribed":null,"forks":22,"collection":{"id":"61012ff757d99a0b","slug":"eurostat","title":"Eurostat","owner":{"id":"25fee90bb47fd709","login":"joewdavies","name":"Joe Davies"}},"type":"notebook","body":{"title":"Visualizing Eurostat grid data using Three.js & D3","cells":[{"id":0,"value":"html `

Visualizing Eurostat grid data using Three.js & D3

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":384,"value":"html`

The use of WebGL in GIS has increased in recent years, with the likes of CesiumJS, Mapbox GL JS, deck.gl and kepler.gl all gaining popularity, and rightly so. When compared with traditional methods such as SVG and Canvas, WebGL out-performs them significantly, so when presented with the task of visualizing huge amounts of grid cells on a web browser, this seemed the most logical path to go down.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":946,"value":"md`Check out the project on GitHub! [https://github.com/eurostat/gridviz](https://github.com/eurostat/gridviz)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":35,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":950,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":397,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":770,"value":"md `## Cell Colour Interpolation`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":755,"value":"html `

The colour of each cell is defined by an interpolation function from D3 scale-chromatic:

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":760,"value":"colorScale = d3\n .scaleSqrt()\n .domain(d3.extent(csv, (d) => d.value))\n .range([0, 4])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":65,"value":"md `## Loading and Categorising the Grid Data`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":466,"value":"html `The grid is based on the ETRS89 Lambert Azimuthal Equal-Area projection coordinate reference system (EPSG:3035) and the cells are stored in a csv file with three columns: x,y and value - with value in this case being population. The x and y values actually represent the bottom left corner of each grid cell (x min and y min).`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":558,"value":"html`

As to not overload the notebook, the 5km grid shown in this example has around 166,000 points. For a visualization of the 1km grid check out this example.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":477,"value":"html `Firstly, the raw csv data is loaded and parsed using d3-dsv..`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":172,"value":"csv = dsv.csvParse(await FileAttachment(\"pop_5km.csv\").text(), d3.autoType)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":484,"value":"html `

Then we add a few properties to play around with: label is what's shown in the tooltip, position stores the x and y coordinates in an array and value represents the population attribute that will be used for color interpolation.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":67,"value":"cells = {\n let data_points = [];\n for (let i = 0; i < csv.length; i++) {\n let position = [csv[i].x,csv[i].y]; // EPSG:3035\n let label = \"Population: \" + csv[i].value.toLocaleString(); // text shown in tooltip\n let value = csv[i].value;\n let point = { position, label, value };\n data_points.push(point);\n }\n return data_points;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":476,"value":"html `The points class creates a new geometry object and then appends vertices to it for each cell. In this example I am applying a simple manual transformation to the coordinates, along with an offset so that they are closer to the origin of the scene.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":246,"value":"$35","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":741,"value":"point_size = 0.0136","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":545,"value":"md `## Initializing Three.js`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":454,"value":"html `I won't go into too much detail in this notebook about how Three.js works as there are plenty of great tutorials out there, namely this one by Grant Custer, but here are the basic components that went into this visualization:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":44,"value":"md `## Scene`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":43,"value":"scene = {\n let scene = new THREE.Scene();\n scene.background = new THREE.Color(0xffffff);\n scene.add(points);\n return scene;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":139,"value":"renderer = new THREE.WebGLRenderer({antialias: true})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":36,"value":"md `## Camera`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":38,"value":"camera = {\n let aspect = viz_width / viz_height;\n return new THREE.PerspectiveCamera(fov, aspect, near, far + 1);\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":19,"value":"fov =40;","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":26,"value":"near = 0.1;","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":29,"value":"far = 5;","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":89,"value":"mutable viz_width = Math.min(width, 1000)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":918,"value":"viz_height = {\n if (\n /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(\n navigator.userAgent\n )\n ) {\n return 300;\n } else {\n return 800;\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":209,"value":"radius = 25;","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":48,"value":"md `## Defining the Pan & Zoom Functionality with D3`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":61,"value":"view = d3.select(renderer.domElement)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":63,"value":"{\n function setUpZoom() {\n view.call(zoom); \n let initial_scale = getScaleFromZ(far);\n var initial_transform = d3.zoomIdentity.translate(viz_width/2, viz_height/2).scale(initial_scale); \n zoom.transform(view, initial_transform);\n camera.position.set(0, 0, far);\n }\n setUpZoom();\n return setUpZoom;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":60,"value":"zoom = {\n let d3_zoom = d3.zoom()\n \t.scaleExtent([getScaleFromZ(far), getScaleFromZ(near)])\n \t.on('zoom', () => {\n let d3_transform = d3.event.transform;\n zoomHandler(d3_transform);\n });\n return d3_zoom;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":58,"value":"function zoomHandler(d3_transform) {\n let scale = d3_transform.k;\n let x = -(d3_transform.x - viz_width/2) / scale;\n let y = (d3_transform.y - viz_height/2) / scale;\n let z = getZFromScale(scale);\n camera.position.set(x, y, z);\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":50,"value":"function getScaleFromZ (camera_z_position) {\n let half_fov = fov/2;\n let half_fov_radians = toRadians(half_fov);\n let half_fov_height = Math.tan(half_fov_radians) * camera_z_position;\n let fov_height = half_fov_height * 2;\n let scale = viz_height / fov_height; // Divide visualization height by height derived from field of view\n return scale;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":53,"value":"md `### Get scale from d3's zoom`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":51,"value":"function getZFromScale(scale) {\n let half_fov = fov/2;\n let half_fov_radians = toRadians(half_fov);\n let scale_height = viz_height / scale;\n let camera_z_position = scale_height / (2 * Math.tan(half_fov_radians));\n return camera_z_position;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":350,"value":"function toRadians (angle) {\n return angle * (Math.PI / 180);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":82,"value":"md `## Configuring the Tooltip`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":84,"value":"function showTooltip(mouse_position, datum) {\n let tooltip_width = 120;\n let x_offset = -tooltip_width/2;\n let y_offset = 30;\n tooltip_state.display = \"block\";\n tooltip_state.left = mouse_position[0] + x_offset;\n tooltip_state.top = mouse_position[1] + y_offset;\n tooltip_state.label = datum.label;\n tooltip_state.color = datum.color;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":94,"value":"tooltip_state = { return { display: \"none\" } }","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":95,"value":"function hideTooltip() {\n tooltip_state.display = \"none\";\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":97,"value":"md `## Defining the Mouse Hover Event`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":459,"value":"html `The hover functionality leverages Three.js' raycaster class, which is used for mouse picking (working out what objects in the 3d space the mouse is over) amongst other things.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":101,"value":"raycaster = new THREE.Raycaster()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":747,"value":"html `

The raycaster threshold defines the maximum distance the ray can stray from the center of the cell geometry in order to return a value. Think of it as defining a hitbox.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":115,"value":"{\n raycaster.params.Points.threshold = 0.006;\n return raycaster;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":463,"value":"html `Three.js doesn't return the intersected objects in order of their distance from the ray so this is done manually...`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":117,"value":"function sortIntersectsByDistanceToRay(intersects) {\n return _.sortBy(intersects, \"distanceToRay\");\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":102,"value":"{\n view.on(\"mousemove\", () => {\n let [mouseX, mouseY] = d3.mouse(view.node());\n let mouse_position = [mouseX, mouseY];\n\t checkIntersects(mouse_position);\n });\n return view;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":110,"value":"function mouseToThree(mouseX, mouseY) {\n return new THREE.Vector3(\n mouseX / viz_width * 2 - 1,\n -(mouseY / viz_height) * 2 + 1,\n 1\n );\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":111,"value":"function checkIntersects(mouse_position) {\n let mouse_vector = mouseToThree(...mouse_position);\n raycaster.setFromCamera(mouse_vector, camera);\n let intersects = raycaster.intersectObject(points);\n if (intersects[0]) {\n let sorted_intersects = sortIntersectsByDistanceToRay(intersects);\n let intersect = sorted_intersects[0];\n let index = intersect.index;\n let datum = cells[index];\n highlightPoint(datum);\n showTooltip(mouse_position, datum);\n } else {\n removeHighlights();\n hideTooltip();\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":119,"value":"hoverContainer = new THREE.Object3D()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":121,"value":"{\n scene.add(hoverContainer);\n return scene;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":123,"value":"function highlightPoint(datum) {\n removeHighlights();\n \n let geometry = new THREE.Geometry();\n geometry.vertices.push(\n new THREE.Vector3(\n datum.position[0],\n datum.position[1],\n 0\n )\n );\n geometry.colors = [ new THREE.Color(datum.color) ];\n\n let material = new THREE.PointsMaterial({\n size: 0.02,\n sizeAttenuation: true,\n vertexColors: THREE.VertexColors\n });\n \n let point = new THREE.Points(geometry, material);\n hoverContainer.add(point);\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":125,"value":"function removeHighlights() {\n hoverContainer.remove(...hoverContainer.children);\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":127,"value":"{\n view.on(\"mouseleave\", () => {\n removeHighlights()\n })\n return view;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":670,"value":"md `## Next Steps`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":669,"value":"md `* Add raster basemap or boundaries TopoJSON`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":676,"value":"md `* Distort cell sizes according to their values`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":680,"value":"md `* Implement csv tiling capability`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":74,"value":"md `## Dependencies`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":951,"value":"import { viewof colorInterpolatorPicker } from '@zechasault/color-schemes-and-interpolators-picker'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":174,"value":"dsv = require(\"d3-dsv@1\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":78,"value":"THREE = require(\"three@0.89.0/build/three.min.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":79,"value":"d3 = require(\"https://d3js.org/d3.v5.min.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":131,"value":"_ = require('https://cdnjs.cloudflare.com/ajax/libs/lodash.js/4.17.5/lodash.min.js')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":860,"value":"import {select} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":515,"value":"md `# Credits`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":518,"value":"html `

The code used in this notebook is adapted from Gran Custer's excellent tutorial on using Three.js for 2D visualization:

https://observablehq.com/@grantcuster/using-three-js-for-2d-data-visualization`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":526,"value":"html `

Data source: Eurostat population grid

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"}],"files":"$W36","stdlib":"1","titling":"auto"}} 31:"$30:body"