29:null 2a:["$","title",null,{"children":"Visualising large geospatial datasources 🗺️ | Christoph Pahmeyer | Observable"}] 2f:Tf61,viewof map = { const container = html`

`; yield container; mapboxgl.accessToken = "pk.eyJ1IjoidG9mZmkiLCJhIjoiY2lvMDBxMzR3MDB1eHZza2x4NGI2YjI5OSJ9.IEaNA05pWbT92nOu-lEOYw"; let styleUrl = "mapbox://styles/mapbox/light-v10"; if (baselayer === "Satellite") { styleUrl = "mapbox://styles/mapbox/satellite-streets-v9"; } else if (baselayer === "Light") { styleUrl = "mapbox://styles/mapbox/light-v10"; } else { styleUrl = "mapbox://styles/mapbox/dark-v10"; } const map = new mapboxgl.Map({ container: container, style: baselayer === "White" ? { version: 8, sources: {}, layers: [ { id: "background", type: "background", paint: { "background-color": "white" } } ] } : styleUrl, center: [0, 0], zoom: 0 }); const draw = new MapboxDraw({ displayControlsDefault: false, controls: { polygon: true, trash: true }, modes: Object.assign(MapboxDraw.modes, { draw_polygon: DrawRectangle }) }); map.on("draw.create", (f) => { const feature = f.features[0]; mutable bbox = turf.bbox(feature); }); map.addControl(draw, "top-left"); if (!geojson) return; const splits = chunk(geojson.features, chunkSize); const layers = []; map.on("load", () => { const curLayers = map.getStyle().layers; // Find the index of the first symbol layer in the map style. let firstSymbolId; for (const layer of curLayers) { if (layer.type === "symbol") { firstSymbolId = layer.id; break; } } function addGeoJSON(data, index, mapRef) { mapRef.addSource(`data_source_${index}`, { type: "geojson", maxzoom: 12, tolerance: tolerance, data: { type: "FeatureCollection", features: data } }); map.addLayer( { id: `data_layer_${index}`, type: "fill", source: `data_source_${index}`, // reference the data source layout: {}, paint: { "fill-color": viewof color.value, // blue color fill "fill-opacity": viewof opacity.value } }, firstSymbolId ); map.on("click", `data_layer_${index}`, (e) => { new mapboxgl.Popup() .setLngLat(e.lngLat) .setHTML(JSON.stringify(e.features[0].properties, null, 2)) .addTo(map); }); // Change the cursor to a pointer when // the mouse is over the states layer. map.on("mouseenter", `data_layer_${index}`, () => { map.getCanvas().style.cursor = "pointer"; }); // Change the cursor back to a pointer // when it leaves the states layer. map.on("mouseleave", `data_layer_${index}`, () => { map.getCanvas().style.cursor = ""; }); layers.push(`data_layer_${index}`); } splits.forEach((segment, i) => addGeoJSON(segment, i, map)); viewof color.addEventListener("input", () => { const newColor = viewof color.value; layers.forEach((id) => { map.setPaintProperty(id, "fill-color", newColor); }); }); viewof opacity.addEventListener("input", () => { const newOpacity = viewof opacity.value; layers.forEach((id) => { map.setPaintProperty(id, "fill-opacity", newOpacity); }); }); let bbox = []; try { bbox = turf.bbox(geojson); } catch (error) { console.log(error); } if (bbox.length) { map.fitBounds(turf.bbox(geojson), { padding: 30, ...(baselayer === "White" && { duration: 0 }) }); } mutable message = "Done :)"; }); container.value = map; invalidation.then(() => map.remove()); return container; }30:Tf98,geojson = { if (!file) return; let count = 0; if (fileExtension === "json" || fileExtension === "geojson") { mutable message = "Parsing GeoJSON, please be patient..."; const data = await file.json(); mutable message = `Splitting GeoJSON into ${Math.ceil( data.features.length / chunkSize )} chunks and creating vector tiles...`; return data; } else if (fileExtension === "shp") { const source = await shapefile.open(await file.stream(), null); const data = { type: "FeatureCollection", features: [] }; while (true) { const result = await source.read(); if (result.done) break; count++; if (bbox && bbox.length && !polygonIntersectsBBox(result.value, bbox)) { continue; } if (count < startFeature) { continue; } if (data.features.length % 10000 === 0) { mutable message = `Parsed ${data.features.length} features.`; } if (data.features.length >= maxNoFeatures) { break; } if (dropProperties) { delete result.value.properties; result.value.properties = {}; } if (proj4String) { result.value = await Generators.queue( worker(reprojectFeature, { feature: result.value, projection: proj4String }) ); } data.features.push(result.value); } mutable message = `Splitting Shapefile into ${Math.ceil( data.features.length / chunkSize )} chunks and creating vector tiles...`; return data; } else if ( fileExtension === "ndjson" || fileExtension === "ndgeojson" || fileExtension === "geojsonl" ) { mutable message = "Parsing NDGeoJSON, please be patient..."; const stream = ndjsonStream(await file.stream()); const reader = stream.getReader(); let read; const data = { type: "FeatureCollection", features: [] }; await reader.read().then( (read = (result) => { if (result.done) { return; } count++; if (bbox && bbox.length && !polygonIntersectsBBox(result.value, bbox)) { return reader.read().then(read); } if (count < startFeature) { return reader.read().then(read); } if (data.features.length % 10000 === 0) { mutable message = `Parsed ${data.features.length} features.`; } if (data.features.length >= maxNoFeatures) { return; } if (dropProperties) { delete result.value.properties; result.value.properties = {}; } data.features.push(result.value); return reader.read().then(read); }) ); mutable message = `Splitting NDGeoJSON into ${Math.ceil( data.features.length / chunkSize )} chunks and creating vector tiles...`; return data; } else if (fileExtension === "fgb") { const stream = await file.stream(); const data = { type: "FeatureCollection", features: [] }; for await (let feature of flatgeobuf.deserialize(stream)) { count++; if (data.features.length % 10000 === 0) { mutable message = `Parsed ${data.features.length} features.`; } if (bbox && bbox.length && !polygonIntersectsBBox(feature, bbox)) { continue; } if (count < startFeature) continue; if (data.features.length >= maxNoFeatures) { break; } if (dropProperties) { delete feature.properties; feature.properties = {}; } if (proj4String) { feature = await Generators.queue( worker(reprojectFeature, { feature: feature, projection: proj4String }) ); } data.features.push(feature); } mutable message = `Splitting FlatGeobuf into ${Math.ceil( data.features.length / chunkSize )} chunks and creating vector tiles...`; return data; } }31:[] 2d:{"id":"6710f5bf454c8530","slug":"visualising-large-geospatial-datasources","title":"Visualising large geospatial datasources 🗺️","thumbnail":"2934c70c7a503a856c705a3f858d1978b8ce59ccb712e4aa60b8ea115b020620","access_level":"public","create_time":"$D2022-02-19T12:19:58.215Z","update_time":"$D2023-07-06T08:07:02.011Z","view_time":null,"first_public_time":"$D2022-04-29T06:49:38.256Z","has_unpublished_edits":null,"fork":false,"stars":4,"starred":false,"owner":{"type":"team","id":"d2dc6c8d94d1ea2c","login":"chrispahm","name":"Christoph Pahmeyer","avatar_url":"https://avatars-next.observableusercontent.com/5f4a345410b8a1f574bf0beecf9c7b18c1140fb1c085b289e271718ef4ed0d57","tier":"starter_2024"},"version":460,"publish_version":460,"role":null,"authored":false,"authors":[{"type":"individual","id":"e3c99024cde17af0","login":"chrispahm","name":"Christoph Pahmeyer","avatar_url":"https://avatars-next.observableusercontent.com/5f4a345410b8a1f574bf0beecf9c7b18c1140fb1c085b289e271718ef4ed0d57","tier":null}],"members":[{"type":"individual","id":"e3c99024cde17af0","login":"chrispahm","name":"Christoph Pahmeyer","avatar_url":"https://avatars-next.observableusercontent.com/5f4a345410b8a1f574bf0beecf9c7b18c1140fb1c085b289e271718ef4ed0d57","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":true,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":{"id":"6c7ff609de80b6e2","slug":"maps","title":"Geo","owner":{"id":"d2dc6c8d94d1ea2c","login":"chrispahm","name":"Christoph Pahmeyer"}},"type":"notebook","body":{"title":"Visualising large geospatial datasources 🗺️","cells":[{"id":0,"value":"# Visualising large geospatial datasources 🗺️","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":247,"value":"md`This notebook makes use of some of the tips outlined in Mapbox's guide to [working with large GeoJSON data](https://docs.mapbox.com/help/troubleshooting/working-with-large-geojson-data/). Depending on your browsers memory allocation limits, you may visualize GeoJSON files with more than a million features!\n\n*Tip: Firefox allows for displaying larger files than i.e. Chrome*\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":99,"value":"md`#### ${message}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"viewof file = Inputs.file({ label: \"Data\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":166,"value":"viewof baselayer = Inputs.radio([\"Satellite\", \"Light\", \"Dark\", \"White\"], {\n value: \"Light\",\n label: \"Baselayer\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":185,"value":"viewof opacity = Inputs.range([0, 1], {\n value: 0.5,\n step: 0.05,\n label: \"Opacity\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":171,"value":"viewof color = Inputs.color({ label: \"Polygon fill color\", value: \"#4682b4\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":55,"value":"$2f","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":229,"value":"md`### Optional settings`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":219,"value":"viewof chunkSize = Inputs.range([0, 1000000], {\n value: 100000,\n label: \"Chunk size\",\n step: 1000\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":281,"value":"md`The GeoJSON source is split into ${Math.ceil(\n geojson?.features?.length / chunkSize\n)} smaller GeoJSON feature collections consisting of (max) ${chunkSize} features each.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":225,"value":"viewof tolerance = Inputs.range([0, 10], {\n value: 0.375,\n step: 0.005,\n label:\n \"Douglas-Peucker simplification tolerance (higher means simpler geometries and faster performance)\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":270,"value":"viewof dropProperties = Inputs.toggle({\n label: \"Delete properties objects to save additional memory\",\n value: false\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":347,"value":"viewof proj4String = Inputs.text({\n label:\n \"Optional proj4 projection string in order to project coordinates to WGS84\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":353,"value":"md`If you know the projection of your input files, you may look up the corresponding proj4 string from [https://epsg.io/](https://epsg.io/)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":398,"value":"viewof maxNoFeatures = Inputs.range([0, 1000000], {\n value: 100000,\n label: \"Max. no. of features\",\n step: 1000\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":406,"value":"viewof startFeature = Inputs.range([0, 10000000], {\n value: 0,\n label: \"Start reading at feature\",\n step: 1000\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":300,"value":"md`## Data and methods`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":96,"value":"mutable message = \"Start by dragging a GeoJSON (.json/.geojson), NDGeoJSON (.ndjson/.ndgeojson), Flatgeobuf (.fgb), or Shapefile (.shp) into the file handle below!\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":369,"value":"toWGS84 = new proj4.Proj(\"WGS84\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":456,"value":"mutable bbox = []","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":365,"value":"function reprojectFeature({ feature, projection }) {\n turf.coordEach(feature, (coord) => {\n const p = proj4(projection, toWGS84, coord);\n coord.length = 0;\n coord.push(...p);\n });\n return feature;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":33,"value":"$30","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":14,"value":"function chunk(arr, len) {\n var chunks = [],\n i = 0,\n n = arr.length;\n\n while (i < n) {\n chunks.push(arr.slice(i, (i += len)));\n }\n\n return chunks;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":129,"value":"fileExtension = file?.name?.split(\".\").pop()?.toLowerCase()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":121,"value":"ndjsonStream = require(\"https://bundle.run/can-ndjson-stream@1.0.2\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":448,"value":"function rayIntersect(p, p1, p2) {\n return (\n p1[1] > p[1] !== p2[1] > p[1] &&\n p[0] < ((p2[0] - p1[0]) * (p[1] - p1[1])) / (p2[1] - p1[1]) + p1[0]\n );\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":446,"value":"function insidePolygon(rings, p) {\n var inside = false;\n for (var i = 0, len = rings.length; i < len; i++) {\n var ring = rings[i];\n for (var j = 0, len2 = ring.length, k = len2 - 1; j < len2; k = j++) {\n if (rayIntersect(p, ring[j], ring[k])) inside = !inside;\n }\n }\n return inside;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":439,"value":"function polygonIntersectsBBox(polygon, bbox) {\n polygon = polygon.geometry.coordinates\n var bboxCenter = [(bbox[0] + bbox[2]) / 2, (bbox[1] + bbox[3]) / 2];\n if (insidePolygon(polygon, bboxCenter)) return true;\n for (var i = 0; i < polygon.length; i++) {\n if (lineclip(polygon[i], bbox).length > 0) return true;\n }\n return false;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":381,"value":"md`### Imports`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":442,"value":"lineclip = require(\"https://bundle.run/lineclip@1.1.5\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":375,"value":"proj4 = require(\"proj4\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":421,"value":"DrawRectangle = (\n await require(\"https://bundle.run/mapbox-gl-draw-rectangle-mode@1.0.4\")\n).default","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":416,"value":"html`\n(Mapbox Stylesheet)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":413,"value":"MapboxDraw = require(\"@mapbox/mapbox-gl-draw\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":392,"value":"flatgeobuf = require(\"flatgeobuf@3.20.0/dist/flatgeobuf-geojson.min.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":385,"value":"import { worker } from \"@fil/worker\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":310,"value":"shapefile = require(\"shapefile@0.6\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":60,"value":"turf = require(\"@turf/turf\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":256,"value":"mapboxgl = require(\"mapbox-gl@2.7.0\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":51,"value":"html`\n(Mapbox Stylesheet)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W31","stdlib":"1","titling":"auto"}} 2e:"$2d:body"