2b:null 2c:["$","title",null,{"children":"From Apache Arrow to JavaScript Objects | Fidocia Wima Adityawarman | Observable"}] 31:[] 2f:{"id":"b62198dd4351673d","slug":null,"title":"From Apache Arrow to JavaScript Objects","thumbnail":"c3a25822cd1cdaff2333aabb46186837d19dd918b27c25e324cd816ec3565368","access_level":"public","create_time":"$D2024-04-25T06:02:52.751Z","update_time":"$D2024-04-25T09:19:48.691Z","view_time":null,"first_public_time":"$D2024-04-25T06:02:52.751Z","has_unpublished_edits":null,"fork":{"id":"b31da3382af3dbba","slug":"from-apache-arrow-to-javascript-objects","title":"From Apache Arrow to JavaScript Objects","owner":{"id":"082c3d8198c3fcdc","login":"jheer","name":"Jeffrey Heer"}},"stars":0,"starred":false,"owner":{"type":"team","id":"5ccc010318f1a583","login":"adityawarmanfw","name":"Fidocia Wima Adityawarman","avatar_url":"https://avatars-next.observableusercontent.com/34056e8b79f06d1bcf945272deddd76ec9048accb3920a74a2fbe7978c64ad61","tier":"starter_2024"},"version":399,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"70dc02ae8b4bbb08","login":"adityawarmanfw","name":"Fidocia Wima Adityawarman","avatar_url":"https://avatars-next.observableusercontent.com/34056e8b79f06d1bcf945272deddd76ec9048accb3920a74a2fbe7978c64ad61","tier":null}],"members":[{"type":"team","id":"5ccc010318f1a583","login":"adityawarmanfw","name":"Fidocia Wima Adityawarman","avatar_url":"https://avatars-next.observableusercontent.com/34056e8b79f06d1bcf945272deddd76ec9048accb3920a74a2fbe7978c64ad61","tier":"starter_2024","role":"editor"},{"type":"individual","id":"70dc02ae8b4bbb08","login":"adityawarmanfw","name":"Fidocia Wima Adityawarman","avatar_url":"https://avatars-next.observableusercontent.com/34056e8b79f06d1bcf945272deddd76ec9048accb3920a74a2fbe7978c64ad61","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"From Apache Arrow to JavaScript Objects","cells":[{"id":0,"value":"md`# From Apache Arrow to JavaScript Objects`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":215,"value":"md`[Apache Arrow](https://arrow.apache.org/) provides a language independent column-oriented format for tabular data. However, some JS tools (including [Vega](https://vega.github.io/vega)) primarily expect input data in a row-oriented JavaScript Object format. This notebook runs a simple micro-benchmark to compare strategies for mapping Apache Arrow files to JavaScript objects.\n\nThe results below may be sensitive to the browser used, and perform a single run only. They do not account for variance and may be sensitive to resource use on your machine. Adjust the _N_ parameter below to change the number of rows considered.\n\nFor more about Arrow, see [\"Introduction to Apache Arrow\"](https://beta.observablehq.com/@theneuralbit/introduction-to-apache-arrow) by Brian Hulette.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":274,"value":"viewof N = Inputs.range([1, 1542642], {label: \"Amount\", step: 1})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":314,"value":"**Processing ${d3.format(',')(N)} rows**","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":296,"value":"Inputs.table([\n [\"Direct access via column.get\", ...timeDirect],\n [\"Proxy objects via defineProperty\",...timeProperty],\n [\"Proxy objects via Proxy\", ...timeProxy],\n [\"New objects, column-major copy scan\", ...timeCopyCol],\n [\"New objects, row-major copy scan\", ...timeCopyRow]\n], {header: ['Method', 'Total (ms)', 'Build (ms)', 'Scan (ms)', 'valueSum'], width: width, layout: \"auto\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":91,"value":"// Baseline for comparison: direct access to Arrow table data using column.get\ntimeDirect = {\n const t0 = performance.now();\n \n const col0 = scrabble.getChild('winnerscore');\n const col1 = scrabble.getChild('year');\n \n const t1 = performance.now();\n \n let sum = 0;\n for (let i=0; i d.name);\n const columns = fields.map(d => scrabble.getChild(d));\n const m = fields.length;\n\n for (let i=0; i d.name);\n const columns = fields.map(d => scrabble.getChild(d));\n const m = fields.length;\n \n for (let i=0; i d.name);\n \n const ctr = function(index) {\n this.__rowIndex__ = index;\n };\n \n const proto = ctr.prototype;\n\n Object.defineProperty(proto, '__rowIndex__', {\n value: -1,\n writable: true\n });\n\n fields.forEach(function(name) {\n const column = table.getChild(name);\n\n Object.defineProperty(proto, name, {\n get: function() {\n return column.get(this.__rowIndex__);\n },\n enumerable: true\n });\n });\n\n return function(i) {\n return new ctr(i);\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":117,"value":"buildProxy = function(table) {\n const fields = table.schema.fields.reduce((m, d) => {\n m[d.name] = table.getChild(d.name);\n return m;\n }, {});\n\n var handler = {\n get: function(obj, prop) {\n if (fields.hasOwnProperty(prop)) {\n return fields[prop].get(obj.__rowIndex__);\n } else {\n return undefined;\n }\n },\n has: function(key) {\n return fields.hasOwnProperty(key);\n },\n ownKeys: function() {\n return Object.keys(fields);\n }\n };\n\n return function(index) {\n return new Proxy({__rowIndex__: index}, handler);\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":6,"value":"scrabble = {\n // Get the arrow file from GitHub gist as an ArrayBuffer\n const buf = await d3.buffer(url);\n \n // Create an arrow Table with it\n return arrow.tableFromIPC([new Uint8Array(buf)]);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"arrow = require('apache-arrow@16.0.0')","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"d3 = require('d3')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"url = \"https://gist.githubusercontent.com/TheNeuralBit/64d8cc13050c9b5743281dcf66059de5/raw/c146baf28a8e78cfe982c6ab5015207c4cbd84e3/scrabble.arrow\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":326,"value":"{\n const data = Array(N);\n const fields = scrabble.schema.fields.map((d) => d.name);\n const columns = fields.map((d) => scrabble.getChild(d));\n const m = fields.length;\n for (let i = 0; i < N; ++i) {\n data[i] = {};\n }\n\n for (let j = 0; j < m; ++j) {\n let name = fields[j],\n col = columns[j];\n for (let i = 0; i < N; ++i) {\n data[i][name] = col.get(i);\n }\n }\n return data;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":364,"value":"scrabble.slice(10,100000)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":362,"value":"viewof start = Inputs.range([1, N], {label: \"Start\", step: 1})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":338,"value":"viewof table = {\n let data = []\n for (let i=N-start; i