25:["$","title",null,{"children":"Visualizing Eurostat grid data using Three.js & D3 | Ped Ram | Observable"}] 49:T515,{ renderer.setSize(viz_width, viz_height); renderer.setPixelRatio(devicePixelRatio); try { while (true) { renderer.render(scene, camera); let html_render = html`

${renderer.domElement}

${tooltip_state.label}

25 km² population grid of Europe

`; yield html_render; } } finally { renderer.dispose(); } }4a:T560,viewof color_interpolation = colorInterpolatorPicker({ title: "Color ramp", height: 300, value: "Turbo", interpolators: [ { name: "BrBG" }, { name: "PRGn" }, { name: "PiYG" }, { name: "PuOr" }, { name: "RdBu" }, { name: "RdGy" }, { name: "RdYlBu" }, { name: "RdYlGn" }, { name: "Spectral" }, { name: "Blues" }, { name: "Greens" }, { name: "Greys" }, { name: "Oranges" }, { name: "Purples" }, { name: "Reds" }, { name: "Viridis" }, { name: "Cividis" }, { name: "Inferno" }, { name: "Magma" }, { name: "Plasma" }, { name: "Cool" }, { name: "CubehelixDefault" }, { name: "BuGn" }, { name: "BuPu" }, { name: "GnBu" }, { name: "OrRd" }, { name: "PuBuGn" }, { name: "PuBu" }, { name: "PuRd" }, { name: "RdPu" }, { name: "YlGnBu" }, { name: "YlGn" }, { name: "YlOrBr" }, { name: "YlOrRd" }, { name: "Rainbow" }, { name: "Sinebow" }, { name: "Turbo" } ] })4b:T4b0,html`

When faced with the challenge of visualizing a 1km² population grid of Europe whilst rendering everything client-side, my initial attempt was to use a 2D HTML Canvas and call fillRect() for each grid cell. This method however simply couldnt handle the quantity of cells we wanted to show, bearing in mind that a 1km grid of europe has over 2 million cells.

Next up was PixiJS - a 2D WebGL renderer designed for game developers. This also struggled to handle the sheer amount of points we were trying to render. After looking around, it seemed that Three.js was capable of handling huge amounts of points, so I decided to go in that direction, even though we did not need any 3D capability at all. There were other alternatives available such as potree - a point-cloud viewer, or REGL - a lower level library. However although highly performant, they offered less funcionality when compared with Three.js, which will prove useful later on in development when we manipulate the cell attributes and add features such as raycasting.

`4c:T4df,points = { let pointsGeometry = new THREE.Geometry(); let colors = []; //let xScale = d3.scaleLinear().domain(d3.extent(cells.map(c => c.position[0]))).range([-1,1]); //let yScale = d3.scaleLinear().domain(d3.extent(cells.map(c => c.position[1]))).range([-1,1]); for (let cell of cells) { // Set vector coordinates from data //let coords = [xScale(cell.position[0]),yScale(cell.position[1])] let coords = [cell.position[0], cell.position[1]]; // TODO: find a cleaner way of converting EPSG 3035 to Vector3 //let vectorCoords = new THREE.Vector3(coords[0],coords[1],0) let vectorCoords = new THREE.Vector3( coords[0] / 100000 - 4, coords[1] / 100000 - 3.5, 0 ); // in Threejs, colors and vertices are added to the geometry object in separate, corresponding arrays pointsGeometry.vertices.push(vectorCoords); let hex = color_interpolation(colorScale(cell.value)); cell.color = hex; //for tooltip colors.push(new THREE.Color(hex)); } pointsGeometry.colors = colors; let pointsMaterial = new THREE.PointsMaterial({ size: point_size, sizeAttenuation: true, vertexColors: THREE.VertexColors }); return new THREE.Points(pointsGeometry, pointsMaterial); }4d:[{"id":"b54c87b17a6c4171fefa009f6f00a47194fe64b787ff8c71ede0b4d9912f1dfa2cfc67990fb51e1054f6900d47f4463660fddba113e316bf1d051f5b1d1dd9c4","href":"https://static.observableusercontent.com/files/b54c87b17a6c4171fefa009f6f00a47194fe64b787ff8c71ede0b4d9912f1dfa2cfc67990fb51e1054f6900d47f4463660fddba113e316bf1d051f5b1d1dd9c4","name":"pop_5km.csv","type":"text/csv","lastModified":1697766915433.973,"size":3032980}] 29:{"id":"134a699bb78efe9d","slug":null,"title":"Visualizing Eurostat grid data using Three.js & D3","thumbnail":"bde5945c2213caec57d030708b2d8ee64f2ed740255265e4aa7803575e1a6349","access_level":"public","create_time":"$D2023-10-20T01:55:15.433Z","update_time":"$D2023-10-20T01:55:15.433Z","view_time":null,"first_public_time":"$D2023-10-20T01:55:15.433Z","has_unpublished_edits":null,"fork":{"id":"e0c2cdc3943539fd","slug":"visualizing-eurostat-grid-data-using-three-js-d3","title":"Visualizing Eurostat grid data using Three.js & D3","owner":{"id":"25fee90bb47fd709","login":"joewdavies","name":"Joe Davies"}},"stars":0,"starred":false,"owner":{"type":"team","id":"b0bbcf444e46bf02","login":"pedahd","name":"Ped Ram","avatar_url":"https://avatars-next.observableusercontent.com/ef13a6f0f32bd05288b47eb131f8eb1c5f9160b8e901276508df258390e51492","tier":"starter_2024","bio":"","followed_by_you":false,"follows_you":false},"version":1013,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"5456960e585c7bbb","login":"pedahd","name":"Ped Ram","avatar_url":"https://avatars-next.observableusercontent.com/ef13a6f0f32bd05288b47eb131f8eb1c5f9160b8e901276508df258390e51492","tier":null,"bio":"","followed_by_you":false,"follows_you":false}],"members":[{"type":"individual","id":"5456960e585c7bbb","login":"pedahd","name":"Ped Ram","avatar_url":"https://avatars-next.observableusercontent.com/ef13a6f0f32bd05288b47eb131f8eb1c5f9160b8e901276508df258390e51492","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Visualizing Eurostat grid data using Three.js & D3","cells":[{"id":0,"value":"html `

Visualizing Eurostat grid data using Three.js & D3

The use of WebGL in GIS has increased in recent years, with the likes of CesiumJS, Mapbox GL JS, deck.gl and kepler.gl all gaining popularity, and rightly so. When compared with traditional methods such as SVG and Canvas, WebGL out-performs them significantly, so when presented with the task of visualizing huge amounts of grid cells on a web browser, this seemed the most logical path to go down.

The colour of each cell is defined by an interpolation function from D3 scale-chromatic:

As to not overload the notebook, the 5km grid shown in this example has around 166,000 points. For a visualization of the 1km grid check out this example.

Then we add a few properties to play around with: label is what's shown in the tooltip, position stores the x and y coordinates in an array and value represents the population attribute that will be used for color interpolation.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":67,"value":"cells = {\n let data_points = [];\n for (let i = 0; i < csv.length; i++) {\n let position = [csv[i].x,csv[i].y]; // EPSG:3035\n let label = \"Population: \" + csv[i].value.toLocaleString(); // text shown in tooltip\n let value = csv[i].value;\n let point = { position, label, value };\n data_points.push(point);\n }\n return data_points;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":476,"value":"html `The points class creates a new geometry object and then appends vertices to it for each cell. In this example I am applying a simple manual transformation to the coordinates, along with an offset so that they are closer to the origin of the scene.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":246,"value":"$4c","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":741,"value":"point_size = 0.0136","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":545,"value":"md `## Initializing Three.js`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":454,"value":"html `I won't go into too much detail in this notebook about how Three.js works as there are plenty of great tutorials out there, namely this one by Grant Custer, but here are the basic components that went into this visualization:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":44,"value":"md `## Scene`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":43,"value":"scene = {\n let scene = new THREE.Scene();\n scene.background = new THREE.Color(0xffffff);\n scene.add(points);\n return scene;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":139,"value":"renderer = new THREE.WebGLRenderer({antialias: true})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":36,"value":"md `## Camera`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":38,"value":"camera = {\n let aspect = viz_width / viz_height;\n return new THREE.PerspectiveCamera(fov, aspect, near, far + 1);\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":19,"value":"fov =40;","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":26,"value":"near = 0.1;","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":29,"value":"far = 5;","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":89,"value":"mutable viz_width = Math.min(width, 1000)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":918,"value":"viz_height = {\n if (\n /Android|webOS|iPhone|iPad|iPod|BlackBerry|IEMobile|Opera Mini/i.test(\n navigator.userAgent\n )\n ) {\n return 300;\n } else {\n return 800;\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":209,"value":"radius = 25;","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":48,"value":"md `## Defining the Pan & Zoom Functionality with D3`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":61,"value":"view = d3.select(renderer.domElement)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":63,"value":"{\n function setUpZoom() {\n view.call(zoom); \n let initial_scale = getScaleFromZ(far);\n var initial_transform = d3.zoomIdentity.translate(viz_width/2, viz_height/2).scale(initial_scale); \n zoom.transform(view, initial_transform);\n camera.position.set(0, 0, far);\n }\n setUpZoom();\n return setUpZoom;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":60,"value":"zoom = {\n let d3_zoom = d3.zoom()\n \t.scaleExtent([getScaleFromZ(far), getScaleFromZ(near)])\n \t.on('zoom', () => {\n let d3_transform = d3.event.transform;\n zoomHandler(d3_transform);\n });\n return d3_zoom;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":58,"value":"function zoomHandler(d3_transform) {\n let scale = d3_transform.k;\n let x = -(d3_transform.x - viz_width/2) / scale;\n let y = (d3_transform.y - viz_height/2) / scale;\n let z = getZFromScale(scale);\n camera.position.set(x, y, z);\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":50,"value":"function getScaleFromZ (camera_z_position) {\n let half_fov = fov/2;\n let half_fov_radians = toRadians(half_fov);\n let half_fov_height = Math.tan(half_fov_radians) * camera_z_position;\n let fov_height = half_fov_height * 2;\n let scale = viz_height / fov_height; // Divide visualization height by height derived from field of view\n return scale;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":53,"value":"md `### Get scale from d3's zoom`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":51,"value":"function getZFromScale(scale) {\n let half_fov = fov/2;\n let half_fov_radians = toRadians(half_fov);\n let scale_height = viz_height / scale;\n let camera_z_position = scale_height / (2 * Math.tan(half_fov_radians));\n return camera_z_position;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":350,"value":"function toRadians (angle) {\n return angle * (Math.PI / 180);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":82,"value":"md `## Configuring the Tooltip`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":84,"value":"function showTooltip(mouse_position, datum) {\n let tooltip_width = 120;\n let x_offset = -tooltip_width/2;\n let y_offset = 30;\n tooltip_state.display = \"block\";\n tooltip_state.left = mouse_position[0] + x_offset;\n tooltip_state.top = mouse_position[1] + y_offset;\n tooltip_state.label = datum.label;\n tooltip_state.color = datum.color;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":94,"value":"tooltip_state = { return { display: \"none\" } }","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":95,"value":"function hideTooltip() {\n tooltip_state.display = \"none\";\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":97,"value":"md `## Defining the Mouse Hover Event`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":459,"value":"html `The hover functionality leverages Three.js' raycaster class, which is used for mouse picking (working out what objects in the 3d space the mouse is over) amongst other things.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":101,"value":"raycaster = new THREE.Raycaster()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":747,"value":"html `

The raycaster threshold defines the maximum distance the ray can stray from the center of the cell geometry in order to return a value. Think of it as defining a hitbox.

The code used in this notebook is adapted from Gran Custer's excellent tutorial on using Three.js for 2D visualization:

https://observablehq.com/@grantcuster/using-three-js-for-2d-data-visualization`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":526,"value":"html `

Data source: Eurostat population grid