2c:null 2d:["$","title",null,{"children":"Streaming GeoJSONL | Brett Camper | Observable"}] 32:[] 30:{"id":"023366fb0da12624","slug":"streaming-geojsonl","title":"Streaming GeoJSONL","thumbnail":"8bf13f34ba41a73e2ef73102e1f26c989693ab5af34127ca898a7c5e4c098e8b","access_level":"public","create_time":"$D2019-03-14T02:03:38.654Z","update_time":"$D2019-07-06T02:28:30.216Z","view_time":null,"first_public_time":"$D2019-03-14T04:02:44.206Z","has_unpublished_edits":null,"fork":false,"stars":5,"starred":false,"owner":{"type":"team","id":"c53130f001c03e79","login":"bcamper","name":"Brett Camper","avatar_url":"https://avatars-next.observableusercontent.com/1f3a3aead4cc1f200d81120af918d78f076732c23133b2a182f81410bd748058","tier":"starter_2024","bio":""},"version":242,"publish_version":242,"role":null,"authored":false,"authors":[{"type":"individual","id":"9facf0ed218e9eb6","login":"bcamper","name":"Brett Camper","avatar_url":"https://avatars-next.observableusercontent.com/1f3a3aead4cc1f200d81120af918d78f076732c23133b2a182f81410bd748058","tier":null,"bio":""}],"members":[{"type":"individual","id":"9facf0ed218e9eb6","login":"bcamper","name":"Brett Camper","avatar_url":"https://avatars-next.observableusercontent.com/1f3a3aead4cc1f200d81120af918d78f076732c23133b2a182f81410bd748058","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Streaming GeoJSONL","cells":[{"id":0,"value":"md`\n# Streaming GeoJSONL\nAn example of streaming GeoJSONL (line-delimited GeoJSON) in the browser using:\n- [\\`ReadableStream\\` consumed from a \\`fetch()\\` request](https://developer.mozilla.org/en-US/docs/Web/API/Streams_API/Using_readable_streams)\n- [Async iterator to output the features as they are parsed](https://jakearchibald.com/2017/async-iterators-and-generators/)\n- [\\`for-await\\` to loop over them asynchronously](https://jakearchibald.com/2017/async-iterators-and-generators/)\n\nAlso see:\n- [GeoJSONL: An optimized format for large geographic datasets](https://www.interline.io/blog/geojsonl-extracts/)\n- [Even more geospatial tools supporting the \"GeoJSONL\" format](https://www.interline.io/blog/here-cli-supports-geojsonl/)\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":44,"value":"{\n const polys = features.filter(f => ['Polygon', 'MultiPolygon'].indexOf(f.geometry.type) > -1).length;\n const lines = features.filter(f => ['LineString', 'MultiLineString'].indexOf(f.geometry.type) > -1).length;\n const points = features.filter(f => ['Point', 'MultiPoint'].indexOf(f.geometry.type) > -1).length;\n \n return md`\n## Geometry Counts\n\n**${features.length.toLocaleString()}** features read so far:\n\n- Polygons: ${polys.toLocaleString()}\n- Lines: ${lines.toLocaleString()}\n- Points: ${points.toLocaleString()}\n`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":121,"value":"{\n // count unique properties\n let props = new Map();\n features.forEach(f => {\n Object.keys(f.properties).forEach(p => {\n if (!props.get(p)) props.set(p, 0);\n props.set(p, props.get(p) + 1);\n });\n });\n props = Array.from(props).sort((a, b) => b[1] - a[1]); // sort by count\n props = props.slice(0, 30); // top thirty properties\n \n // build a table of them\n return html`\n

Most Common Properties

\n \n \n \n \n \n ${props.map(r => html`\n \n \n \n \n `)}\n \n

property	count
${r[0]}	${r[1].toLocaleString()}

\n `;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":20,"value":"features = {\n // build an array of features as they are parsed from the stream\n const f = [];\n for await (const newFeatures of streamGeoJSONL(url)) {\n f.push(...newFeatures);\n yield f;\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"async function* streamGeoJSONL(url) {\n const response = await fetch(url);\n const reader = response.body.getReader(); // stream reader\n const decoder = new TextDecoder('utf-8'); // for decoding binary stream data into strings\n\n try {\n let lineFragment = ''; // incomplete line fragment from previous stream chunk\n\n while (true) {\n // get next stream chunk\n const { done, value } = await reader.read();\n\n if (!done) {\n // decode buffer into string, and split into lines\n const lines = decoder.decode(value).split('\\n');\n\n // prepend last line fragment from previous to the first new line\n lines[0] = lineFragment + lines[0];\n\n // save any line fragment from this chunk for next one\n lineFragment = lines.pop();\n\n // parse and return new features\n yield lines.map(JSON.parse);\n }\n else {\n break; // done processing stream\n }\n }\n }\n finally {\n reader.releaseLock(); // release lock on stream\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"url = 'https://s3.amazonaws.com/vtiles/honolulu_hawaii.geojsonl';","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W32","stdlib":"1"}} 31:"$30:body"