3f:null 40:["$","title",null,{"children":"Reading Tapalcatl 2 Archives with the Fetch API | Seth Fitzsimmons | Observable"}] 46:T69e,class Archive { constructor(source, zip, extension = "") { this.extension = extension; this.entries = {}; this.metadata = { headers: [] } this.source = source; // private this._ready = false; this._zip = zip; this.openReadStream = util.promisify(zip.openReadStream.bind(zip)); this._initialize(); } _initialize() { const zip = this._zip; zip.on("entry", entry => { this.entries[entry.fileName] = entry; }) zip.on("end", () => this._ready = true) if (zip.comment != null) { try { this.metadata = JSON.parse(zip.comment); } catch (err) { console.warn(`Invalid metadata: "${zip.comment}"\n`, err.stack); } } } async ready() { while (!this._ready) { await pImmediate(); } } // TODO variant (make extension local) async getTile(zoom, x, y) { let filename = `${zoom}/${x}/${y}`; if (this.extension !== "") { filename += `.${this.extension}`; } await this.ready(); const entry = this.entries[filename]; if (entry == null) { return null; } let headers = []; if (entry.fileComment !== "") { try { headers = JSON.parse(entry.fileComment); } catch (err) { console.warn(`Invalid tile headers: "${entry.fileComment}"`, err.stack); } } const rs = await this.openReadStream(entry); const chunks = [] rs.on("data", chunk => chunks.push(chunk)); await pEvent(rs, "end"); return { headers: [...this.metadata.headers, ...headers], // TODO replace this with a readable stream body: buffer.Buffer.concat(chunks) }; } }47:[] 44:{"id":"9c0e50104f80e6e2","slug":"reading-tapalcatl-2-archives-with-fetch","title":"Reading Tapalcatl 2 Archives with the Fetch API","thumbnail":"4e937ba9280aef2811a9045fa181ef5876f97ee1b4bf80807bd58e27ec9be31d","access_level":"public","create_time":"$D2018-10-11T20:41:57.304Z","update_time":"$D2018-10-28T23:06:38.267Z","view_time":null,"first_public_time":"$D2018-10-11T22:07:37.234Z","has_unpublished_edits":null,"fork":false,"stars":2,"starred":false,"owner":{"type":"team","id":"461277f0bca7897c","login":"mojodna","name":"Seth Fitzsimmons","avatar_url":"https://avatars-next.observableusercontent.com/2da6648e4443edddbb03f8be4bbbd7a74211c5c91f4b601246faa6f14d278a5d","tier":"starter_2024","bio":"","following":false},"version":270,"publish_version":270,"role":null,"authored":false,"authors":[{"type":"individual","id":"8981a93cd4c5e490","login":"mojodna","name":"Seth Fitzsimmons","avatar_url":"https://avatars-next.observableusercontent.com/2da6648e4443edddbb03f8be4bbbd7a74211c5c91f4b601246faa6f14d278a5d","tier":null,"bio":"","following":false}],"members":[{"type":"individual","id":"8981a93cd4c5e490","login":"mojodna","name":"Seth Fitzsimmons","avatar_url":"https://avatars-next.observableusercontent.com/2da6648e4443edddbb03f8be4bbbd7a74211c5c91f4b601246faa6f14d278a5d","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Reading Tapalcatl 2 Archives with the Fetch API","cells":[{"id":0,"value":"md`# Reading Tapalcatl 2 Archives with the Fetch API`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":184,"value":"md`This notebook demonstrates how tiles can be read from a Tapalcatl 2 ZIP using the \\`fetch\\` API. No caching is done, so you can watch multiple overlapping requests being made for different byte ranges in the Network Inspector (⌥⌘I).`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":132,"value":"md`The source of a Tapalcatl 2 ZIP:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":81,"value":"source = \"https://mojodna.s3.amazonaws.com/tapalcatl-fixtures/tile.zip\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":135,"value":"md`Create an \\`Archive\\` encapsulating the source (API very subject to change):`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":30,"value":"archive = {\n const { reader, size } = await createHttpRandomAccessReader(source);\n \n const zip = await open(source, reader, size);\n \n return new Archive(source, zip, \"txt\");\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":207,"value":"{\n await archive.ready();\n \n return md`\n## Central Directory\n\n### Comment\n\n\\`\\`\\`json\n${archive._zip.comment}\n\\`\\`\\`\n\n### Entries\n\nFilename | Compressed | Raw | Comment | Modified | CRC32\n- | - | - | - | -\n${Object.keys(archive.entries).map(k => {\n const entry = archive.entries[k];\n return `${k} | ${prettyBytes(entry.compressedSize)} | ${prettyBytes(entry.uncompressedSize)} | \\`${entry.fileComment || \" \"}\\` | ${yauzl.dosDateTimeToDate(entry.lastModFileDate, entry.lastModFileTime).toLocaleString()} | \\`${entry.crc32}\\`\\n`;\n})}\n`\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":137,"value":"md`Load a tile from the source:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":112,"value":"{\n const { headers, body } = await archive.getTile(0, 0, 0);\n \n return md`\n\\`\\`\\`javascript\nawait archive.getTile(0, 0, 0);\n\\`\\`\\`\n\n\\`\\`\\`\n${headers.map(h => `${Object.keys(h)[0]}: ${h[Object.keys(h)[0]]}`).join(\"\\n\")}\n\n${body.toString()}\n\\`\\`\\`\n`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":186,"value":"md`\nSample CORS configuration for use with Amazon S3:\n\n\\`\\`\\`xml\n\n\n\n *\n GET\n HEAD\n 3000\n *\n Content-Length\n Content-Type\n\n\n\\`\\`\\`\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":106,"value":"md`---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":72,"value":"$46","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":160,"value":"md`An HTTP-aware \\`RandomAccessReader\\` (\\`yauzl\\`'s interface for providing access to ZIPs):`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":16,"value":"class HTTPReader extends yauzl.RandomAccessReader {\n constructor(source) {\n super();\n this.source = source;\n }\n\n async fetchInto(start, end, rs) {\n try {\n const rsp = await fetch(this.source, {\n headers: {\n Range: `bytes=${start}-${end - 1}`\n }\n });\n\n // Node.js-style\n // rsp.body.pipe(rs);\n rs.end(buffer.Buffer.from(await rsp.arrayBuffer()));\n } catch (err) {\n rs.emit(\"error\", err);\n }\n }\n\n _readStreamForRange(start, end) {\n // TODO use the web Streams API (https://developer.mozilla.org/en-US/docs/Web/API/Streams_API)\n // to avoid Node shims (buffer, stream)\n const rs = new stream.PassThrough();\n\n this.fetchInto(start, end, rs);\n\n return rs;\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":20,"value":"createHttpRandomAccessReader = async source => {\n const rsp = await fetch(source, {\n method: \"HEAD\"\n });\n\n if (rsp.status !== 200) {\n throw new Error(`ENOENT: no such file or directory, open '${source}'`);\n }\n \n return {\n reader: new HTTPReader(source),\n size: Number(rsp.headers.get(\"content-length\"))\n };\n}\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":31,"value":"async function open(filename, randomAccessReader = null, size = null) {\n if (randomAccessReader != null && size != null) {\n return fromRandomAccessReader(randomAccessReader, size, {\n autoClose: false\n });\n }\n\n return openZip(filename, {\n autoClose: false\n });\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":148,"value":"md`---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":78,"value":"buffer = require('https://bundle.run/buffer@5.2.1')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"stream = require('https://bundle.run/stream-browserify@2.0.1')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":36,"value":"util = require('https://bundle.run/util@0.11.0')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":70,"value":"pEvent = require('https://bundle.run/p-event@2.1.0')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":75,"value":"pImmediate = require('https://bundle.run/p-immediate@2.1.0')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":215,"value":"prettyBytes = require('https://bundle.run/pretty-bytes@5.1.0')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":22,"value":"yauzl = require('https://bundle.run/yauzl@2.10.0')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":116,"value":"md`---`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":34,"value":"openZip = util.promisify(yauzl.open.bind(yauzl))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":39,"value":"fromRandomAccessReader = util.promisify(\n yauzl.fromRandomAccessReader.bind(yauzl)\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W47","stdlib":"1"}} 45:"$44:body"