2b:null 2c:["$","title",null,{"children":"Benchmarking Arquero, DuckDB-WASM, and VanillaJS backends V2 | Kasi Viswanath | Observable"}] 31:T55c,# Benchmarking Arquero, DuckDB-WASM, and VanillaJS backends V2 This notebook provides an user interface to benchmark the backends such as Arquero, DuckDB-WASM, and VanillaJS You can experiment with [Default Setup](https://observablehq.com/d/e9a77a6cedb791ab#cell-1918) or [Custom Setup](https://observablehq.com/d/e9a77a6cedb791ab#cell-1600) [Step 1](https://observablehq.com/d/e9a77a6cedb791ab#cell-1022): Generate the queries and store them in MongoDB using [Query Planner](https://observablehq.com/@kasivisu4/query-planner)
[Step 2](https://observablehq.com/d/e9a77a6cedb791ab#cell-1650): Retrieve queries from MongoDB, Run the queries utilizing [Query Executor](https://observablehq.com/@kasivisu4/query-executor-v2?collection=@kasivisu4/data-filtering-benchmark), and Record the execution duration of every query for each backend in MongoDB. A **Query** can be described as the specific values chosen for an attribute. **Note**: - Please ensure that your Mongo and Node servers are running locally to run the experiments. You can find the setup instructions [here](https://observablehq.com/@kasivisu4/mongodb-local). - Results of all experiments can be viewed in Heat Map/Violin chart at the [end](https://observablehq.com/d/e9a77a6cedb791ab#cell-153). - For obtaining detailed information on the memory footprint, it is advisable to use Google Chrome.32:T7cc,chartWithFaster = { const height = 400; const hover = vl.selectSingle().on("mouseover").nearest(true).empty("none"); const base = vl .markRule({ color: "#ccc" }) .encode(vl.x().fieldN("key"), vl.detail().count()); const line = base.markLine().encode( vl .color() .fieldN("faster") .scale({ domain: ["Arquero", "duckDB", "nativeJS"], range: ["#1f77b4", "#ff7f0e", "#2ca02c"] }), vl.detail().fieldN("index"), vl.opacity().if(hover, vl.value(1)).value(0.3), vl.y().fieldQ("norm_val").axis(null), vl.tooltip([...attribs, "comb", "faster"]) ); const points = line .markCircle() .select(hover) .encode(vl.size().if(hover, vl.value(50)).value(5)); // Generates a spec to show tick values at an specific value of y0 const tick = (y0) => vl .layer( base.markText({ style: "label" }).encode(vl.text().max("max")), base.markTick({ style: "tick", size: 8, color: "#ccc" }) ) .encode(vl.y().value(y0)); // Create an array with *numTicks* ticks const ticks = Array.from({ length: 3 }).map((_, i) => tick((height / (3 - 1)) * i) ); return vl .layer(base, line, points, ...ticks) .data(data_) .transform( // vl.filter(attribs.map((a) => `datum["${a}"] != null`).join(" && ")), vl.window(vl.count().as("index")), vl.fold(attribs), vl .groupby("key") .joinaggregate(vl.min("value").as("min"), vl.max("value").as("max")), vl .calculate("(datum.value - datum.min) / (datum.max - datum.min)") .as("norm_val"), vl.calculate("(datum.min + datum.max) / 2").as("mid") ) .config({ axisX: { domain: false, labelAngle: 0, tickColor: "#ccc", title: null }, view: { stroke: null }, style: { label: { baseline: "middle", align: "right", dx: -5 }, tick: { orient: "horizontal" } } }) .width(width - 100) .height(height) .render(); }33:T742,chart = { const height = 400; const hover = vl.selectSingle().on("mouseover").nearest(true).empty("none"); const base = vl .markRule({ color: "#ccc" }) .encode(vl.x().fieldN("key"), vl.detail().count()); const line = base .markLine() .encode( vl.color().fieldN("comb"), vl.detail().fieldN("index"), vl.opacity().if(hover, vl.value(1)).value(0.3), vl.y().fieldQ("norm_val").axis(null), vl.tooltip([...attribs, "comb"]) ); const points = line .markCircle() .select(hover) .encode(vl.size().if(hover, vl.value(50)).value(5)); // Generates a spec to show tick values at an specific value of y0 const tick = (y0) => vl .layer( base.markText({ style: "label" }).encode(vl.text().max("max")), base.markTick({ style: "tick", size: 8, color: "#ccc" }) ) .encode(vl.y().value(y0)); // Create an array with *numTicks* ticks const ticks = Array.from({ length: 3 }).map((_, i) => tick((height / (3 - 1)) * i) ); return vl .layer(base, line, points, ...ticks) .data(data_) .transform( // vl.filter(attribs.map((a) => `datum["${a}"] != null`).join(" && ")), vl.window(vl.count().as("index")), vl.fold(attribs), vl .groupby("key") .joinaggregate(vl.min("value").as("min"), vl.max("value").as("max")), vl .calculate("(datum.value - datum.min) / (datum.max - datum.min)") .as("norm_val"), vl.calculate("(datum.min + datum.max) / 2").as("mid") ) .config({ axisX: { domain: false, labelAngle: 0, tickColor: "#ccc", title: null }, view: { stroke: null }, style: { label: { baseline: "middle", align: "right", dx: -5 }, tick: { orient: "horizontal" } } }) .width(width - 100) .height(height) .render(); }34:[] 2f:{"id":"a878f2cd5daf16ea","slug":null,"title":"Benchmarking Arquero, DuckDB-WASM, and VanillaJS backends V2","thumbnail":"09034544e40ec0fbe0f0c48e82c40361f2b5ce71a90b5a1682f4e92e8d4780c7","access_level":"public","create_time":"$D2023-04-29T18:44:01.073Z","update_time":"$D2023-04-30T18:21:14.046Z","view_time":null,"first_public_time":"$D2023-04-29T18:44:01.073Z","has_unpublished_edits":null,"fork":{"id":"e9a77a6cedb791ab","slug":"benchmarking-framework","title":"Benchmark Filtering Performance of In-Browser Data Processing Libraries v1.0","owner":{"id":"b760ea566a896d7d","login":"kasivisu4","name":"Kasi Viswanath"}},"stars":1,"starred":false,"owner":{"type":"team","id":"b760ea566a896d7d","login":"kasivisu4","name":"Kasi Viswanath","avatar_url":"https://avatars-next.observableusercontent.com/48e36723913e7e5b18b906e1d7cde12fad4146715e78bbf74d3db7969a248b56","tier":"starter_2024"},"version":2352,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"fcd94e1c1ca12ae9","login":"kasivisu4","name":"Kasi Viswanath","avatar_url":"https://avatars-next.observableusercontent.com/48e36723913e7e5b18b906e1d7cde12fad4146715e78bbf74d3db7969a248b56","tier":null}],"members":[{"type":"individual","id":"fcd94e1c1ca12ae9","login":"kasivisu4","name":"Kasi Viswanath","avatar_url":"https://avatars-next.observableusercontent.com/48e36723913e7e5b18b906e1d7cde12fad4146715e78bbf74d3db7969a248b56","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"title":"Benchmarking Arquero, DuckDB-WASM, and VanillaJS backends V2","cells":[{"id":0,"value":"$31","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":143,"value":"monitorMemory()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":391,"value":"

","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1918,"value":"# Default Setup","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1906,"value":"- Experiment Steps\n - Step 1: Click the create experiment button for creating the experiment where Query plan will be created in the background\n - Step 2: After the query Plan created, Click the run experiment to start the experiment\n - Step 3: Click on Refresh Chart to see the details of the executed queries for the mentioned experimentID\n- Note:\n - Default Experiment ID is created. Please update in case if you want to change experiment name","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1921,"value":"### Backend Performance Analysis","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":557,"value":"viewof graphRefresh = Inputs.button(\"Refrest chart\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":769,"value":"viewof chartType = Inputs.select(\n [\"Heat-Map-Summarizer\", \"violin-Chart\", \"bar-chart\"],\n {\n label: \"Chart type\"\n }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1925,"value":"Note: The chart will be empty if there are no queries executed. Please run the experiment to check the backend Performance","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1739,"value":"

","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1600,"value":"### Custom Setup","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1200,"value":"

Environment Setup

${viewof useMongo} ${viewof serverUrl } ${viewof mongoDBUrl} ${nodeServerStatus} ${mongoServerStatus} Please ensure that the Node server and MongoDB are connected if you are using Mongo. 🟢 Connected ⛔️ Not Connected

","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":2250,"value":"barChart()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1022,"value":"barChart = async () => {\n const data = await getDataForChart();\n return vl\n .markBar({ tooltip: true, cornerRadius: 10 })\n .encode(\n vl.y().fieldN(\"backend\"),\n vl.x().fieldQ(\"time\"),\n vl.color().fieldN(\"backend\"),\n vl.facet().fieldN(\"comb\").columns(6),\n vl.text().count()\n )\n .data(data)\n .height(100)\n .width(100)\n .render();\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2272,"value":"attribs = [\"arqueroExecTime\", \"duckDBExecTime\", \"nativeJSExecTime\"]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2298,"value":"viewof time = Inputs.range([100, 5000], { step: 1 })","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2304,"value":"viewof comb = {\n let combs = [\n \"1Q-@V\",\n \"1Q-RV\",\n \"1Q-NV\",\n \"1C-@V\",\n \"1C-RV\",\n \"1C-NV\",\n \"1C-@V+1Q-@V\",\n \"1C-@V+1Q-@V+1Q-RV\",\n \"1C-@V+1Q-@V+1Q-RV+1Q-NV\",\n \"1C-RV+1Q-@V\",\n \"1C-RV+1Q-@V+1Q-RV\",\n \"1C-RV+1Q-@V+1Q-RV+1Q-NV\",\n \"1C-NV+1Q-@V\",\n \"1C-NV+1Q-@V+1Q-RV\",\n \"1C-NV+1Q-@V+1Q-RV+1Q-NV\",\n \"2C-@V+1Q-@V\",\n \"2C-@V+1Q-@V+1Q-RV\",\n \"2C-@V+1Q-@V+1Q-RV+1Q-NV\",\n \"2C-RV+1Q-@V\",\n \"2C-RV+1Q-@V+1Q-RV\",\n \"2C-RV+1Q-@V+1Q-RV+1Q-NV\",\n \"2C-NV+1Q-@V\",\n \"2C-NV+1Q-@V+1Q-RV\",\n \"2C-NV+1Q-@V+1Q-RV+1Q-NV\"\n ];\n return searchCheckbox(combs, { label: \"Select Combinations\" });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2345,"value":"$32","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2269,"value":"$33","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2292,"value":"import { vl } from \"@vega/vega-lite-api\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2276,"value":"data_ = {\n function check(a, b, c) {\n if (a <= b && a <= c) {\n return \"Arquero\";\n }\n if (c <= a && c <= b) {\n return \"nativeJS\";\n }\n if (b <= a && b <= c) {\n return \"duckDB\";\n }\n }\n let data_ = data\n .map((d) => {\n return {\n comb: d[\"comb\"],\n arqueroExecTime: d[\"arquero\"][\"filter_time\"],\n duckDBExecTime: d[\"duckDB\"][\"filter_time\"],\n nativeJSExecTime: d[\"nativeJS\"][\"filter_time\"]\n };\n })\n .map((d) => {\n return {\n ...d,\n faster: check(\n d[\"arqueroExecTime\"],\n d[\"duckDBExecTime\"],\n d[\"nativeJSExecTime\"]\n )\n };\n })\n .filter(\n (d) =>\n comb.includes(d[\"comb\"]) &&\n d[\"arqueroExecTime\"] <= time &&\n d[\"duckDBExecTime\"] <= time &&\n d[\"nativeJSExecTime\"] <= time\n );\n return data_;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2278,"value":"data = await mongodbClient({\n db: \"benchmark\",\n collection: \"queryExecutor\",\n operation: \"find\",\n query: {}\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2244,"value":"async function getDataForChart(id = \"defaultExperiment\") {\n let res = await mongodbClient({\n db: \"benchmark\",\n collection: \"queryExecutor\",\n operation: \"find\",\n query: { experimentName: id }\n });\n\n let target = [];\n for (let data of res) {\n let res = {};\n res[\"comb\"] = data.comb;\n res[\"duckdb\"] = data?.duckDB?.filter_time / 1000;\n res[\"arquero\"] = data?.arquero?.filter_time / 1000;\n res[\"nativeJS\"] = data?.nativeJS?.filter_time / 1000;\n res[\"dataset\"] = data.dataset;\n target.push(res);\n }\n\n let data = [];\n if (target.length) {\n data = aq\n .from(target)\n .fold([\"duckdb\", \"arquero\", \"nativeJS\"], { as: [\"backend\", \"time\"] })\n .objects();\n }\n return data;\n return res;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":153,"value":"## Performance Analysis\n\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":177,"value":"Let's analyze backends based on experiments.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1885,"value":"Inputs.bind(Inputs.button(\"Referesh Charts\"), viewof graphRefresh)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":721,"value":"import { multiAutoSelect } from \"@john-guerra/multi-auto-select\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"import {\n constants,\n clipBoardCopy,\n downloadFile,\n feedBackButton,\n monitorMemory\n} from \"@kasivisu4/filterer-utilities\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":64,"value":"import {\n queryExecutor,\n getTestids,\n mongoDetailsQueryPlan,\n mongoDetailsSuccessQuery,\n mongoDetailsFailedQuery,\n arqueroFilterer,\n nativeJSFilterer,\n duckDBFilterer,\n getMaxQueryId\n} from \"@kasivisu4/query-executor-v2\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":156,"value":"import { performaceChart } from \"@kasivisu4/performance-analysis\";","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":15,"value":"import { getDatasets } from \"@kasivisu4/datasets\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":205,"value":"import { searchCheckbox } from \"@john-guerra/search-checkbox\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":226,"value":"import { FacetedSearch } from \"@kasivisu4/faceted-search-filter\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":600,"value":"import { mongodbClient } from \"@kasivisu4/mongodb-local\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2056,"value":"import { queryPlanner } from \"@kasivisu4/query-planner-v2\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W34","stdlib":"1","titling":"auto"}} 30:"$2f:body"