2d:null 2e:["$","title",null,{"children":"Query Executor V3 | Kasi Viswanath | Observable"}] 33:T1dee,async function queryExecutor({ queries, sleepTime = 1, storageClient, // To store and query queries and its results backends = [arqueroFilterer, duckDBFilterer, vanillaJSFilter], log = [], queryPlanCollection = "queryPlanner", queryExecCollection = "queryExecutor", dbName = "benchmark" } = {}) { let queryCounter = { success: 0, failed: 0 }; let expNames = await getExperiments(); let experimentSelect = Inputs.select(expNames, { label: "Select the experiment" }); let runExperiment = Inputs.button("Run Experiment"); let status = html`

Log

`; let target = html`${experimentSelect} ${runExperiment} ${status}`; let statusLog = html``; let queries_ = []; let statusLock = false; let queriesStatus = html``; let datasetsHelper = await getDatasets(); target.log = log; async function runFn() { let initalCounter = 0; if (storageClient) { let response = await storageClient({ db: dbName, collection: queryExecCollection, operation: "find", query: { experimentName: experimentSelect.value }, options: { sort: { queryId: -1 }, limit: 1 } }); initalCounter = response[0] ? response[0].queryId : 0; } let totalQueries; if (storageClient) { let response = await storageClient({ db: dbName, collection: queryPlanCollection, operation: "find", query: { experimentName: experimentSelect.value }, options: { sort: { queryId: -1 }, limit: 1 } }); totalQueries = response[0]?.queryId; } else { totalQueries = Math.max(queries.map((d) => d.queryId)); } log.push({ msg: `Query Executor: Total Queries : ${totalQueries} | Completed Queries ${initalCounter}`, level: 1 }); //Executing queries while (initalCounter < totalQueries) { try { let query; if (storageClient) { let res = await storageClient({ db: dbName, collection: queryPlanCollection, operation: "find", query: { experimentName: experimentSelect.value, queryId: initalCounter + 1 } }); query = res[0]; } else { query = queries[queryPlanCollection][initalCounter]; } log.push({ msg: `Query Executor: Executing query ${query}`, level: 2 }); if (!Object.keys(datasetsHelper.datasets).includes(query["dataset"])) { await datasetsHelper.clearAll(); await datasetsHelper.selectDataset(query["dataset"])(); console.log({ datasetsHelper }); } let data = datasetsHelper.datasets[query["dataset"]]; //lengths to check if all the backend gives the same result based on the length let lengths = new Set(); let resLog = { ...query }; //executing in backends for (let backend of backends) { let filter = await backend(); log.push({ msg: `Query Executor: Executing query ${query} in ${filter.filterer}`, level: 2 }); let init_start = performance.now(); await filter.initialize(data, log); let init_end = performance.now(); //filter the data based on query map let queryMap = new Map(Object.entries(query.query)); console.log({ queryMap }); let filter_start = performance.now(); let res = await filter.filterData(queryMap); let filter_end = performance.now(); lengths.add(res.length); resLog[filter.filterer] = {}; let factors = { init_time: init_end - init_start, filter_time: filter_end - filter_start, length: res.length }; Object.assign(resLog[filter.filterer], factors); filter = null; await new Promise((d) => setTimeout(d, sleepTime)); } //If the size is same then results match if (lengths.size === 1) { if (storageClient) { let res = await storageClient({ db: dbName, collection: queryExecCollection, operation: "insert", data: [resLog] }); if (!res.acknowledged) { throw res; } } else { queries[queryExecCollection].push(resLog); } queryCounter.success += 1; } else { queryCounter.failed += 1; } } catch (err) { log.push({ msg: `Query Executor: Executing queries error with ${err}`, level: 0 }); queryCounter.failed += 1; } finally { initalCounter += 1; } } } runExperiment.addEventListener("click", async () => { log.push({ msg: `Query Executor: Starting Experiment ${experimentSelect.value}`, level: 1 }); runExperiment.childNodes[0].disabled = true; statusLock = true; updateStatus(); await runFn(); target.log = log; //Update the log and status before lock updateStatus(); await new Promise((d) => setTimeout(d, sleepTime)); statusLock = false; runExperiment.childNodes[0].disabled = false; //Queries queriesStatus?.remove(); queriesStatus = htl.html`Query Executor Execution Completed | Success ${queryCounter.success} | Failed ${queryCounter.failed}`; target.appendChild(queriesStatus); log.push({ msg: `Query Executor: Experiment ${experimentSelect.value} Completed`, level: 1 }); }); async function updateStatus() { while (statusLock) { //Queries queriesStatus?.remove(); queriesStatus = htl.html`Running... | Completed ${queryCounter.success} | Failed ${queryCounter.failed}`; target.appendChild(queriesStatus); // Log statusLog?.remove(); statusLog = html`

${[ ...log.filter((d) => d.level < 2).slice(-10) ] .reverse() .map((d, i) => (d.msg.length > 50 ? d.msg.slice(0, 50) + "..." : d.msg)) .join("
")}

`; status.appendChild(statusLog); await new Promise((d) => setTimeout(d, sleepTime)); } } //Functions to get Experiment Names async function getExperiments() { log.push({ msg: `Query Executor: Fetching Experiments`, level: 1 }); let experimentNames = []; try { if (storageClient) { let res = await storageClient({ db: dbName, collection: queryPlanCollection, operation: "aggregate", query: [ { $group: { _id: "$experimentName" } } ] }); if (res instanceof Error) { throw res; } experimentNames = res?.map((d) => d._id); } else { experimentNames = new Set( queries[queryPlanCollection].map((d) => d?.experimentName) ); } } catch (err) { log.push({ msg: `Query Executor: Fetching experiments failed with error ${err} `, level: 0 }); console.error( `Query Executor: Fetching experiments failed with error ${err} ` ); } finally { log.push({ msg: `Query Executor: Fetched Experiments -> ${experimentNames}`, level: 1 }); return experimentNames; } } return target; }34:[] 31:{"id":"100f4df2446b5fd1","slug":"query-executor-v3","title":"Query Executor V3","thumbnail":"c9f3d724ba4a6b1b63eb563faedd1c0a4dae3e2c9074c5b8ca6b5b1b36f50991","access_level":"public","create_time":"$D2023-04-29T23:31:22.318Z","update_time":"$D2023-07-20T06:35:14.167Z","view_time":null,"first_public_time":"$D2023-04-29T23:31:22.318Z","has_unpublished_edits":null,"fork":{"id":"7cba2cbcdb12c8fb","slug":"query-executor-v2","title":"Query Executor V2","owner":{"id":"b760ea566a896d7d","login":"kasivisu4","name":"Kasi Viswanath"}},"stars":0,"starred":false,"owner":{"type":"team","id":"b760ea566a896d7d","login":"kasivisu4","name":"Kasi Viswanath","avatar_url":"https://avatars-next.observableusercontent.com/48e36723913e7e5b18b906e1d7cde12fad4146715e78bbf74d3db7969a248b56","tier":"starter_2024"},"version":3301,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"fcd94e1c1ca12ae9","login":"kasivisu4","name":"Kasi Viswanath","avatar_url":"https://avatars-next.observableusercontent.com/48e36723913e7e5b18b906e1d7cde12fad4146715e78bbf74d3db7969a248b56","tier":null}],"members":[{"type":"individual","id":"fcd94e1c1ca12ae9","login":"kasivisu4","name":"Kasi Viswanath","avatar_url":"https://avatars-next.observableusercontent.com/48e36723913e7e5b18b906e1d7cde12fad4146715e78bbf74d3db7969a248b56","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":{"id":"48467ca908df504b","slug":"archive","title":"Archive","owner":{"id":"b760ea566a896d7d","login":"kasivisu4","name":"Kasi Viswanath"}},"type":"notebook","body":{"title":"Query Executor V3","cells":[{"id":0,"value":"# Query Executor V3\n\nThis notebook provides a user interface to execute the generated queries from [Query Planner](https://observablehq.com/@kasivisu4/query-planner?collection=@kasivisu4/data-filtering-benchmark) with the backends such as Arquero, DuckDB-WASM, and VanillaJS\n\nNote: Generated Queries will be fetched from MongoDB. Make sure you are running Mongo and NodeJS Server. You could find instructions about running Servers [here](https://observablehq.com/@kasivisu4/mongodb-local?collection=@kasivisu4/data-filtering-benchmark)\n\nTo - Do:\n- Initial query counter for execution is default to recent success queryId +1 but should find workaround\n- Generate all permutations of Backends","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1320,"value":"monitorMemory()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":3142,"value":"viewof q = queryExecutor({ storageClient: mongodbClient })","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":3146,"value":"viewof q.log","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":3121,"value":"$33","pinned":true,"hidden":true,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":932,"value":"import { duckDBFilterer } from \"@kasivisu4/duckdb-filterer\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":934,"value":"import { arqueroFilterer } from \"@kasivisu4/arquero-datastore\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1025,"value":"import { mongodbClient } from \"@kasivisu4/mongodb-local\";","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1011,"value":"mongoDetailsQueryPlan = {\n let res = {};\n Object.assign(res, constants.queryPlan);\n Object.assign(res, {\n mongoDBUrl: constants.mongoDBUrl,\n serverUrl: constants.serverUrl\n });\n return res;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1019,"value":"mongoDetailsSuccessQuery = {\n let res = {};\n Object.assign(res, constants.successQueryRes);\n Object.assign(res, {\n mongoDBUrl: constants.mongoDBUrl,\n serverUrl: constants.serverUrl\n });\n return res;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1623,"value":"mongoDetailsFailedQuery = {\n let res = {};\n Object.assign(res, constants.failedQueryRes);\n Object.assign(res, {\n mongoDBUrl: constants.mongoDBUrl,\n serverUrl: constants.serverUrl\n });\n return res;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1013,"value":"import {\n constants,\n clipBoardCopy,\n downloadFile,\n monitorMemory,\n feedBackButton\n} from \"@kasivisu4/filterer-utilities\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1057,"value":"import { getDatasets } from \"@kasivisu4/datasets\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1136,"value":"import { vanillaJSFilter } from \"@kasivisu4/vanillajs-filter\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1792,"value":"//ToDo\nfunction generatePermutations(arr) {\n return [arr];\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W34","stdlib":"1","titling":"auto"}} 32:"$31:body"