2d:null 2e:["$","title",null,{"children":"From Apache Arrow to JavaScript Objects | Jeffrey Heer | Observable"}] 33:[] 31:{"id":"b31da3382af3dbba","slug":"from-apache-arrow-to-javascript-objects","title":"From Apache Arrow to JavaScript Objects","thumbnail":"461cfa805623fe1d851d1618256483a977f97af6ca0bc68bd3af1db793cd1fb6","access_level":"public","create_time":"$D2018-06-30T03:21:50.752Z","update_time":"$D2019-02-08T20:58:56.272Z","view_time":null,"first_public_time":"$D2018-06-30T06:45:36.367Z","has_unpublished_edits":null,"fork":false,"stars":19,"starred":false,"owner":{"type":"team","id":"082c3d8198c3fcdc","login":"jheer","name":"Jeffrey Heer","avatar_url":"https://avatars-next.observableusercontent.com/230cea705fbda819b9f6b8bc87e05ebecf06d314098901fbd9946af2cdc51be4","tier":"starter_2024"},"version":246,"publish_version":246,"role":null,"authored":false,"authors":[{"type":"individual","id":"ecf15dc33ed8cd37","login":"jheer","name":"Jeffrey Heer","avatar_url":"https://avatars-next.observableusercontent.com/230cea705fbda819b9f6b8bc87e05ebecf06d314098901fbd9946af2cdc51be4","tier":null}],"members":[{"type":"individual","id":"ecf15dc33ed8cd37","login":"jheer","name":"Jeffrey Heer","avatar_url":"https://avatars-next.observableusercontent.com/230cea705fbda819b9f6b8bc87e05ebecf06d314098901fbd9946af2cdc51be4","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":5,"collection":null,"type":"notebook","body":{"title":"From Apache Arrow to JavaScript Objects","cells":[{"id":0,"value":"md`# From Apache Arrow to JavaScript Objects`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":215,"value":"md`[Apache Arrow](https://arrow.apache.org/) provides a language independent column-oriented format for tabular data. However, some JS tools (including [Vega](https://vega.github.io/vega)) primarily expect input data in a row-oriented JavaScript Object format. This notebook runs a simple micro-benchmark to compare strategies for mapping Apache Arrow files to JavaScript objects.\n\nThe results below may be sensitive to the browser used, and perform a single run only. They do not account for variance and may be sensitive to resource use on your machine. Adjust the _N_ parameter below to change the number of rows considered.\n\nFor more about Arrow, see [\"Introduction to Apache Arrow\"](https://beta.observablehq.com/@theneuralbit/introduction-to-apache-arrow) by Brian Hulette.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":184,"value":"md`\n| Method (processing ${d3.format(',')(N)} rows) | Build | Scan | Total |\n|--------|------:|-----:|------:|\n| Direct access via _column.get_ | ${timeDirect[1]} ms | ${timeDirect[2]} ms | ${timeDirect[0]} ms |\n| Proxy objects via _defineProperty_ | ${timeProperty[1]} ms | ${timeProperty[2]} ms | ${timeProperty[0]} ms |\n| Proxy objects via _Proxy_ | ${timeProxy[1]} ms | ${timeProxy[2]} ms | ${timeProxy[0]} ms |\n| New objects, column-major copy scan | ${timeCopyCol[1]} ms | ${timeCopyCol[2]} ms | ${timeCopyCol[0]} ms |\n| New objects, row-major copy scan | ${timeCopyRow[1]} ms | ${timeCopyRow[2]} ms | ${timeCopyRow[0]} ms |\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":178,"value":"md`_Do all methods generate the same result_? **${\n timeDirect[3] === timeProperty[3] &&\n timeDirect[3] === timeProxy[3] &&\n timeDirect[3] === timeCopyCol[3] &&\n timeDirect[3] === timeCopyRow[3] ? 'Yes' : 'No'\n}!**`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":61,"value":"viewof N = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":91,"value":"// Baseline for comparison: direct access to Arrow table data using column.get\ntimeDirect = {\n const t0 = Date.now();\n \n const col0 = scrabble.getColumn('winnerscore');\n const col1 = scrabble.getColumn('year');\n \n const t1 = Date.now();\n \n let sum = 0;\n for (let i=0; i d.name);\n const columns = fields.map(d => scrabble.getColumn(d));\n const m = fields.length;\n\n for (let i=0; i d.name);\n const columns = fields.map(d => scrabble.getColumn(d));\n const m = fields.length;\n \n for (let i=0; i d.name);\n \n const ctr = function(index) {\n this.__rowIndex__ = index;\n };\n \n const proto = ctr.prototype;\n\n Object.defineProperty(proto, '__rowIndex__', {\n value: -1,\n writable: true\n });\n\n fields.forEach(function(name) {\n const column = table.getColumn(name);\n\n Object.defineProperty(proto, name, {\n get: function() {\n return column.get(this.__rowIndex__);\n },\n enumerable: true\n });\n });\n\n return function(i) {\n return new ctr(i);\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":117,"value":"buildProxy = function(table) {\n const fields = table.schema.fields.reduce((m, d) => {\n m[d.name] = table.getColumn(d.name);\n return m;\n }, {});\n\n var handler = {\n get: function(obj, prop) {\n if (fields.hasOwnProperty(prop)) {\n return fields[prop].get(obj.__rowIndex__);\n } else {\n return undefined;\n }\n },\n has: function(key) {\n return fields.hasOwnProperty(key);\n },\n ownKeys: function() {\n return Object.keys(fields);\n }\n };\n\n return function(index) {\n return new Proxy({__rowIndex__: index}, handler);\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":6,"value":"scrabble = {\n // Get the arrow file from GitHub gist as an ArrayBuffer\n const buf = await d3.buffer(url);\n \n // Create an arrow Table with it\n return arrow.Table.from([new Uint8Array(buf)]);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"arrow = require('@apache-arrow/es5-umd@0.3.1')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"d3 = require('d3')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"url = \"https://gist.githubusercontent.com/TheNeuralBit/64d8cc13050c9b5743281dcf66059de5/raw/c146baf28a8e78cfe982c6ab5015207c4cbd84e3/scrabble.arrow\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W33","stdlib":"1","titling":"auto"}} 32:"$31:body"