25:["$","title",null,{"children":"Streaming CSV | Mike Bostock | Observable"}] 4b:T715,viewof url = html``4c:Te30,class CsvParser { constructor(encoding) { this._decoder = new TextDecoder(encoding); this._readOffset = 0; // number of bytes we’ve read from buffer this._readLength = 0; // number of bytes available in buffer this._row = null; // the current row this._state = STATE_FIELD; // parser finite state machine this._lineEmpty = true; // true if the line was completely empty (to skip) this._tokenOffset = 0; // number of bytes we’ve written for the current token this._tokenLength = tokenDefaultLength; // current token length (doubles as needed) this._token = new Uint8Array(this._tokenLength); // current token accumulation buffer } *parse(buffer) { let token; this._readOffset = 0; this._readLength = buffer.length; while ((token = this._readToken(buffer)) != null) { if (this._row === null) this._row = []; this._row.push(this._decoder.decode(token)); if (this._state === STATE_END_OF_LINE) { if (!this._lineEmpty) yield this._row; this._row = null; this._state = STATE_FIELD; this._lineEmpty = true; } } } ended() { return this._row === null; } *end() { if (this._row !== null) { this._row.push(this._decoder.decode(this._readDelimiter())); yield this._row; this._row = null; } } // TODO Optimize special case of non-quoted, not-fragmented field. _readToken(buffer) { while (this._readOffset < this._readLength) { const code = buffer[this._readOffset++]; if (this._state === STATE_FIELD) { if (code === CODE_QUOTE) { // entered a quoted value this._state = STATE_QUOTE; this._lineEmpty = false; } else if (code === CODE_CARRIAGE_RETURN) { // possible CRLF this._state = STATE_AFTER_CARRIAGE_RETURN; } else if (code === CODE_LINE_FEED) { // ended an unquoted field & row this._state = STATE_END_OF_LINE; return this._readDelimiter(); } else if (code === delimiterCode) { // ended an unquoted field this._lineEmpty = false; return this._readDelimiter(); } else { // read an unquoted character this._lineEmpty = false; this._readCharacter(code); continue; } } else if (this._state === STATE_QUOTE) { if (code === CODE_QUOTE) { // read a quote within a quoted value this._state = STATE_AFTER_QUOTE_IN_QUOTE; } else { // read a quoted character this._readCharacter(code); continue; } } else if (this._state === STATE_AFTER_QUOTE_IN_QUOTE) { if (code === CODE_QUOTE) { // read a quoted quote this._state = STATE_QUOTE; this._readCharacter(CODE_QUOTE); } else { // exited a quoted value this._state = STATE_FIELD; --this._readOffset; } } else if (this._state === STATE_AFTER_CARRIAGE_RETURN) { this._state = STATE_END_OF_LINE; if (code === CODE_LINE_FEED) { return this._readDelimiter(); } else { --this._readOffset; } } } return null; } _readCharacter(code) { if (this._tokenOffset >= this._tokenLength) { const token = this._token; this._token = new Uint8Array(this._tokenLength <<= 1); this._token.set(token); } this._token[this._tokenOffset++] = code; } _readDelimiter() { const token = this._token.subarray(0, this._tokenOffset); this._token = new Uint8Array(this._tokenLength = tokenDefaultLength); this._tokenOffset = 0; return token; } }4d:[] 47:{"id":"5a8e8a9c0ab833d9","slug":"streaming-csv","title":"Streaming CSV","thumbnail":"028b9ff35c11578c50982be88dd58510f7c250418497ecb4d655745437c9a0ed","access_level":"public","create_time":"$D2018-10-12T15:42:50.684Z","update_time":"$D2020-09-08T02:11:04.591Z","view_time":null,"first_public_time":"$D2018-10-12T16:57:07.481Z","has_unpublished_edits":null,"fork":false,"stars":21,"starred":false,"owner":{"type":"team","id":"8c2e47252fd4995c","login":"mbostock","name":"Mike Bostock","avatar_url":"https://avatars-next.observableusercontent.com/7dbba99ced40bd5ab8013c99b1a11da61231bce321ba1148a9d16ad0f121550a","tier":"pro_2026","bio":"Visualization toolmaker. Founder @observablehq. Creator @d3. Former @nytgraphics. Pronounced BOSS-tock.","followed_by_you":false,"follows_you":false},"version":203,"publish_version":203,"role":null,"authored":false,"authors":[{"type":"individual","id":"074c414ad1d825f5","login":"mbostock","name":"Mike Bostock","avatar_url":"https://avatars-next.observableusercontent.com/7dbba99ced40bd5ab8013c99b1a11da61231bce321ba1148a9d16ad0f121550a","tier":null,"bio":"Visualization toolmaker. Founder @observablehq. Creator @d3. Former @nytgraphics. Pronounced BOSS-tock.","followed_by_you":false,"follows_you":false}],"members":[{"type":"individual","id":"074c414ad1d825f5","login":"mbostock","name":"Mike Bostock","avatar_url":"https://avatars-next.observableusercontent.com/7dbba99ced40bd5ab8013c99b1a11da61231bce321ba1148a9d16ad0f121550a","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":24,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Streaming CSV","cells":[{"id":0,"value":"md`# Streaming CSV\n\nA CSV (or any delimiter) parser that supports the [Streams API](https://streams.spec.whatwg.org/) and [asynchronous iteration](/@mbostock/introduction-to-asynchronous-iteration). Each iteration yields an array of rows.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":65,"value":"$4b","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":87,"value":"{\n const response = await fetch(url);\n const reader = response.body.getReader();\n for await (const rows of new CsvReader(reader)) {\n yield* rows;\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":167,"value":"class CsvReader {\n constructor(reader, encoding) {\n this._reader = reader;\n this._parser = new CsvParser(encoding);\n }\n [Symbol.asyncIterator]() {\n return this;\n }\n async next() {\n const {done, value} = await this._reader.read();\n return done ? (this._parser.ended() ? {done: true}\n : {done: false, value: this._parser.end()})\n : {done: false, value: this._parser.parse(value)};\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":12,"value":"$4c","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":47,"value":"delimiterCode = CODE_COMMA // character code of delimiter","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":54,"value":"tokenDefaultLength = 1 << 7 // default token length to allocate","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":15,"value":"CODE_QUOTE = '\"'.charCodeAt(0)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":18,"value":"CODE_COMMA = \",\".charCodeAt(0)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":21,"value":"CODE_LINE_FEED = \"\\n\".charCodeAt(0)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":24,"value":"CODE_CARRIAGE_RETURN = \"\\r\".charCodeAt(0)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":27,"value":"STATE_FIELD = Symbol(\"field\") // inside an unquoted value","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":30,"value":"STATE_QUOTE = Symbol(\"quote\") // inside a quoted value","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":33,"value":"STATE_AFTER_QUOTE_IN_QUOTE = Symbol(\"afterquote\") // after a quote (\") in a quoted value","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":41,"value":"STATE_AFTER_CARRIAGE_RETURN = Symbol(\"aftercr\") // after a carriage return in an unquoted value","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":44,"value":"STATE_END_OF_LINE = Symbol(\"eol\") // after the last field in a line","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W4d","stdlib":"1"}} 24:null