2f:null 30:["$","title",null,{"children":"Streaming GeoJSONL from your choice of OpenStreetMap extract | Drew Dara-Abrams | Observable"}] 35:T5b4,md` # Streaming GeoJSONL from your choice of OpenStreetMap extract _If you don't want to get an Interline OSM Extracts API token, try the self-contained original version at https://observablehq.com/@bcamper/streaming-geojsonl_ An example of streaming GeoJSONL (line-delimited GeoJSON) in the browser using: - [\`ReadableStream\` consumed from a \`fetch()\` request](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams) - [Async iterator to output the features as they are parsed](https://jakearchibald.com/2017/async-iterators-and-generators/) - [\`for-await\` to loop over them asynchronously](https://jakearchibald.com/2017/async-iterators-and-generators/) Also see: - [GeoJSONL: An optimized format for large geographic datasets](https://www.interline.io/blog/geojsonl-extracts/) - [Even more geospatial tools supporting the "GeoJSONL" format](https://www.interline.io/blog/here-cli-supports-geojsonl/) ## Instructions 1. Fork this notebook so you have a private version. It's necessary to work with a private notebook so you can store your API token as a [secret](https://observablehq.com/@observablehq/secrets) 2. Get a (free) Interline OSM Extracts API token from https://www.interline.io/osm/extracts/ 3. Copy your token to an Observable secret environment variable called INTERLINE_OSM_EXTRACTS_API_TOKEN at https://observablehq.com/settings/secrets 4. Select a city extract from the drop-down below: `36:T491,async function* streamGeoJSONL(url) { const response = await fetch(url); console.log(response.redirected) const reader = response.body.getReader(); // stream reader const decoder = new TextDecoder('utf-8'); // for decoding binary stream data into strings let lineFragment = ''; // incomplete line fragment from previous stream chunk while (true) { // get next stream chunk const { done, value } = await reader.read(); if (!done) { // decode buffer into string, and split into lines const lines = decoder.decode(value).split('\n'); // if we have a line fragment from the previous chunk, prepend it to the first new line if (lineFragment != '') { lines[0] = lineFragment + lines[0]; lineFragment = ''; } // if the last line isn't complete (doesn't end in a newline), save it as a fragment for next chunk const lastLine = lines[lines.length-1]; if (lastLine[lastLine.length-1] !== '\n') { lineFragment = lines.pop(); } // parse and return new features yield lines.map(JSON.parse); } else { break; // done processing stream } } }37:[] 33:{"id":"6b51ffbae5bc41d0","slug":"streaming-geojsonl-from-your-choice-of-openstreetmap-extr","title":"Streaming GeoJSONL from your choice of OpenStreetMap extract","thumbnail":"66eb4b09aad73b8c946efe0e75e91284e968787b6fce9864adbcf881759c4de9","access_level":"public","create_time":"$D2019-03-14T04:58:21.284Z","update_time":"$D2019-03-14T21:39:23.392Z","view_time":null,"first_public_time":"$D2019-03-14T21:42:16.257Z","has_unpublished_edits":null,"fork":{"id":"023366fb0da12624","slug":"streaming-geojsonl","title":"Streaming GeoJSONL","owner":{"id":"c53130f001c03e79","login":"bcamper","name":"Brett Camper"}},"stars":0,"starred":false,"owner":{"type":"team","id":"983c58603a3ffb4e","login":"drewda","name":"Drew Dara-Abrams","avatar_url":"https://avatars-next.observableusercontent.com/ceaaac48a125d829a02c6149683edf7ffab64f29185b7df706665538ed6a0f9e","tier":"starter_2024"},"version":319,"publish_version":319,"role":null,"authored":false,"authors":[{"type":"individual","id":"2cac6074f1bddd1b","login":"drewda","name":"Drew Dara-Abrams","avatar_url":"https://avatars-next.observableusercontent.com/ceaaac48a125d829a02c6149683edf7ffab64f29185b7df706665538ed6a0f9e","tier":null}],"members":[{"type":"individual","id":"2cac6074f1bddd1b","login":"drewda","name":"Drew Dara-Abrams","avatar_url":"https://avatars-next.observableusercontent.com/ceaaac48a125d829a02c6149683edf7ffab64f29185b7df706665538ed6a0f9e","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":5,"collection":null,"type":"notebook","body":{"title":"Streaming GeoJSONL from your choice of OpenStreetMap extract","cells":[{"id":302,"value":"$35","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":224,"value":"viewof selectedCityId = html`\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":44,"value":"{\n const polys = features.filter(f => ['Polygon', 'MultiPolygon'].indexOf(f.geometry.type) > -1).length;\n const lines = features.filter(f => ['LineString', 'MultiLineString'].indexOf(f.geometry.type) > -1).length;\n const points = features.filter(f => ['Point', 'MultiPoint'].indexOf(f.geometry.type) > -1).length;\n \n return md`\n## Geometry Counts\n\n**${features.length.toLocaleString()}** features read so far:\n\n- Polygons: ${polys.toLocaleString()}\n- Lines: ${lines.toLocaleString()}\n- Points: ${points.toLocaleString()}\n`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":121,"value":"{\n // count unique properties\n let props = new Map();\n features.forEach(f => {\n Object.keys(f.properties).forEach(p => {\n if (!props.get(p)) props.set(p, 0);\n props.set(p, props.get(p) + 1);\n });\n });\n props = Array.from(props).sort((a, b) => b[1] - a[1]); // sort by count\n props = props.slice(0, 30); // top thirty properties\n \n // build a table of them\n return html`\n

Most Common Properties

\n \n \n \n \n \n ${props.map(r => html`\n \n \n \n \n `)}\n \n

property	count
${r[0]}	${r[1].toLocaleString()}

\n `;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":20,"value":"features = {\n // build an array of features as they are parsed from the stream\n const f = [];\n for await (const newFeatures of streamGeoJSONL(url)) {\n f.push(...newFeatures);\n yield f;\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"$36","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":278,"value":"// note that this endpoint is really just proxying us through to \n// https://github.com/interline-io/osm-extracts/blob/master/cities.json\n// with CORS enabled\ncities = fetch('https://app.interline.io/osm_extracts/bounding_boxes.geojson')\n .then(response => response.json())\n .then(json => json.features.map (city => city.properties))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":229,"value":"// sign up for a (free) API token at https://www.interline.io/osm/extracts/\n// copy your token in to https://observablehq.com/settings/secrets\n// for help see https://observablehq.com/@observablehq/secrets\napi_token = Secret(\"INTERLINE_OSM_EXTRACTS_API_TOKEN\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"url = `https://app.interline.io/osm_extracts/download_latest?string_id=${selectedCityId}&data_format=geojsonl&api_token=${api_token}`;","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W37","stdlib":"1","titling":"auto"}} 34:"$33:body"