Public
Edited
Apr 15, 2023
1 fork
Insert cell
Insert cell
Insert cell
viewof testId = testIdFn()
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
async function getMaxQueryIdRes({ mongoDetails, testId }) {
let res = await mongodbClient({
...mongoDetails,
operation: "find",
query: { testId: testId },
options: { sort: { queryId: -1 }, limit: 1 }
});
return !res.length ? 1 : res[0].queryId + 1;
}
Insert cell
getMaxQueryIdRes({
mongo: mongoDetailsQueryPlan,
testId: "test-2"
})
Insert cell
async function* executeQueries({
testId,
filters = [nativeJSFilterer, arqueroFilterer, duckDBFilterer],
sleepTime = 1000
} = {}) {
//To-Do : get Max Id from Mongodb
function getMaxSuccessQueryId({ mongoDetails }) {
return getMaxQueryIdRes({
mongoDetails,
testId: testId
});
}
let queryCounter = await getMaxSuccessQueryId({
mongoDetails: mongoDetailsSuccessQuery
});
yield html`${await queryTracker({ testId })}`;
let res;
let maxQueries = await getMaxQueryIdRes({
testId,
mongoDetails: mongoDetailsQueryPlan
});
while (runQueries && queryCounter <= maxQueries) {
log(`Running query ${JSON.stringify({ queryCounter, testId })}`);
yield html`${await queryTracker({ testId })} Executing : ${queryCounter}`;
res = await getNextQueryAndRunIt({ filters, queryCounter, testId });
// In memory log
mutable mlog.push({ queryCounter, res });
queryCounter += 1;

//Sleep
await new Promise((res) => setTimeout(res, sleepTime));
}
yield html`${await queryTracker({ testId })} ${
runQueries ? "All Queries Executed" : "Toggle Run Queries"
}`;
}
Insert cell
async function getNextQueryAndRunIt({ filters, queryCounter, testId }) {
//get Queries

log("getNextQueryAndRunIt: getting queries", queryCounter);
let query = await mongodbClient({
...mongoDetailsQueryPlan,
operation: "find",
query: { testId, queryId: queryCounter }
});

log("getNextQueryAndRunIt: got queries", queryCounter, query);

if (!query.length) {
return null;
}

log("getNextQueryAndRunIt: running query", queryCounter, query);

return runQuery(query[0], filters);
}
Insert cell
Insert cell
Insert cell
Insert cell
async function runQuery(query, filters) {
let results = [];
let queryMap = new Map(Object.entries(query.query));

filters = d3.shuffle(filters);
log("runQuery: shuffling filters", filters);
for (let filter of filters) {
log("runQuery: creating backend", filter);
//Initialize
let { initialize, filterData, filterer, client } = await filter();
//initialize
let initStart = performance.now();

log("runQuery: initializing filter", filter);
await initialize(datasets[query.dataset]);
let initEnd = performance.now();
log("runQuery: initialized ", initialize, filterData, filterer);

//Filter
let execStart = performance.now();

log("runQuery: filtering data", queryMap);
let res = await filterData(queryMap);
let execEnd = performance.now();

log("runQuery: data filtered", queryMap);

//Add res to results
results.push({
initTime: initEnd - initStart,
ExecTime: execEnd - execStart,
resLength: res?.length,
res: res,
filterer
});

//explicitly clearing memory
filterData = null;
initialize = null;
filterer = null;
res = null;
client = null;
}

function checkResults(results) {
return new Set(results.map((d) => d.resLength)).size === 1;
}

// Creating the log and checks results
let resFlat = { ...query };
delete resFlat["_id"];
log("runQuery: checking results", results);
resFlat.resMatch = checkResults(results);

for (let result of results) {
resFlat[result.filterer] = {};
resFlat[result.filterer]["initTime"] = result.initTime;
resFlat[result.filterer]["execTime"] = result.ExecTime;
if (!resFlat.resMatch) {
resFlat[result.filterer] = result.res;
} else {
resFlat["length"] = result.resLength;
}
}
let order = results.map((d) => d.filterer);
resFlat.execOrder = order;
log("runQuery: creating the log", resFlat);

log("runQuery: results checked", results);

// Inserting log into Mongo
log("runQuery: inserting log to Mongo", resFlat);

let insertCollection = resFlat.resMatch
? mongoDetailsSuccessQuery
: mongoDetailsFailedQuery;
let response = await mongodbClient({
...insertCollection,
operation: "insert",
data: [resFlat]
});
log("runQuery: inserted acknowledged", response.acknowledged);

if (response.acknowledged) {
return { status: true, resFlat };
} else {
return { status: false, resFlat };
}
}
Insert cell
//ToDo
function generatePermutations(arr) {
return [arr];
}
Insert cell
viewof qe = queryExecutor({
fetchQueriesFn: async (d) => (d < 5 ? d : false),
runFn: (query, backend, successFn, failedFn, log) => {
let res = query % 2 == 0 ? successFn() : failedFn();
if (res === "failed") {
throw new Error("Required");
}
log[res].push({ query, backend, res });
},
successFn: () => "success",
failedFn: () => "failed"
})
Insert cell
randomBackend = {
let backend = ["Arquero", "DuckDB", "VanillaJS"];
return backend[d3.randomInt(3)()];
}
Insert cell
{
qe;
return viewof qe.log;
}
Insert cell
//Executor
async function queryExecutor({
fetchQueriesFn = (d) => d,
//Function to run the query
runFn,
//Function to run once the query ran successfully,
successFn,
//Function to run once the query failed
failedFn,
initialCounter = 1,
backends = ["Arquero", "DuckDB", "VanillaJS"],
//A flag to check all the permutations of backend like [arquero,duckdb] => generates [arquero,duckdb], [duckdb,arquero]
checkBackendPoss = false,
//SleepTime after the query execution
sleepTime = 1000
} = {}) {
let queryCounter = initialCounter;
let backendsComb = checkBackendPoss
? generatePermutations(backends)
: [d3.shuffle(backends)];
let log = { success: [], failed: [], error: [] };
let query;
let runBtn = Inputs.toggle({ label: "Run Queries" });
let target = html`${runBtn}`;
let lock = false;

async function executeQueries() {
lock = true;
let currentBackend = "";
console.log("Executing Queries");
while (runBtn.value) {
try {
//Fetch Query
query = await fetchQueriesFn(queryCounter);
if (!query) {
console.log("Completed execution");
lock = false;
return;
}
for (let backends of backendsComb) {
for (let backend of backends) {
// try { //Inner Try Block
currentBackend = backend;
console.log(`Executing query : ${queryCounter} in ${backend}`);
let res = await runFn(query, backend, successFn, failedFn, log);
target.dispatchEvent(new Event("input", { bubbles: true }));
await new Promise((d) => setTimeout(d, sleepTime));
// yield { poss, queryCounter };
}
// catch (error) {
// log["error"].push({ query, error });
// console.error({ error, query, currentBackend });
// }
}
} catch (error) {
log["error"].push({ query, error });
console.error({ error, query, currentBackend });
} finally {
queryCounter += 1;
}
}
}

runBtn.addEventListener("input", () => {
if (runBtn.value && !lock) {
executeQueries();
}
});

target.log = log;
return target;
}
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
mongoDetailsFailedQuery = {
let res = {};
Object.assign(res, constants.failedQueryRes);
Object.assign(res, {
mongoDBUrl: constants.mongoDBUrl,
serverUrl: constants.serverUrl
});
return res;
}
Insert cell
Insert cell
// datasets = getdatasets("imdb")
Insert cell
Insert cell
Insert cell
Insert cell
import { interval } from "@mootari/range-slider@1786"
Insert cell

Purpose-built for displays of data

Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
Learn more