I love building things that make people's life easier
viewof query_planner = queriesPlanner({
datasetsAccessor: datasetsAccessor,
// customDataFlag: true
})
viewof query_planner.queries
queries
Type Table, then Shift-Enter. Ctrl-space for more options.
async function queriesPlanner({
comb = defaultCombinations,
runFn, //Function to generate the query
sleepTime = 1000,
attributes = [],
log = [],
storageClient,
collectionName = "queryPlanner",
dbName = "benchmark-all",
datasetsAccessor,
customDataFlag = false,
defaultExpName = "defaultExperiment"
} = {}) {
let useCustomCombinations = Inputs.toggle({
label: "Custom Query combinations"
});
let experimentName = Inputs.text({
label: "Experiment Name",
value: defaultExpName
});
let customCombinations;
let noOfExperiments = Inputs.range([0, 100], {
step: 1,
label: "Repeat experiment",
value: 1
});
//
let queryCounter = { success: 0, failed: 0 };
let datasets_ = datasetsAccessor.getNames();
let defaultDatasets = [datasets_[2]];
console.log({ defaultDatasets });
if (customDataFlag) {
defaultDatasets[0] = "custom-data";
}
let selectDatasets = Inputs.checkbox(datasets_, {
label: "Benchmark datasets",
value: defaultDatasets
});
let generateQueriesBtn = Inputs.button("Create Experiment");
let status = html`<details style="max-width: 600px; background: #fffced; box-sizing: border-box; padding: 10px 20px;"><summary style="font-weight: bold; cursor: pointer; outline: none;">Log</details>`;
let target = html`<div style='display:flex;flex-direction:column; gap: 15px;'>${experimentName} ${selectDatasets} ${useCustomCombinations} ${noOfExperiments} ${generateQueriesBtn} ${status}</div>`;
target.log = log;
let statusLog = html``;
let queries_ = [];
let statusLock = false;
let queriesStatus = html``;
target.queries = {};
//Toggle custom combinations
useCustomCombinations.addEventListener("input", (event) => {
if (event.target.type != "checkbox") {
return;
}
if (useCustomCombinations.value) {
//Disable at start
customCombinations = htl.html`<div style='display:flex;flex-direction:column'> ${Inputs.textarea(
{
placeholder: `Ex : ["1C-RV", "1C-@V", "1C-NV", "1Q-@V", "1Q-NV", "1Q-RV", "1Q-RV+1C-@V"]`
}
)} <div> For more information on query combinations please check <a href='https://observablehq.com/@kasivisu4/queries-generator?collection=@kasivisu4/benchmark-tools'>here</a></div>`;
useCustomCombinations.appendChild(customCombinations);
} else {
customCombinations.remove();
}
});
async function runFn({ noOfExperiments, comb, datasets }) {
console.log({ comb });
log.push({ msg: `Query Planner: Started`, level: 1 });
log.push({
msg: `Query Planner: Running for ${JSON.stringify(datasets)}`,
level: 1
});
log.push({
msg: `Query Planner: Combinations ${JSON.stringify(comb)}`,
level: 1
});
for (let dataset of datasets) {
try {
log.push({ msg: `Query Planner: Running for ${dataset}`, level: 1 });
let res = datasetsAccessor;
await res.selectDataset(dataset)();
let data = res.datasets[dataset];
log.push({
msg: `Query Planner : Fetched ${dataset} with recordcount ${
data.length
}. The record count Provided is ${datasetsAccessor.getRecordCount()}`,
level: 1
});
let queries = queriesGenerator(data, {
queryCombinations: comb,
replicaCount: noOfExperiments,
attributes
});
log.push({
msg: `Query Planner: Generated ${
queries?.length || 0
} queries for ${dataset}`,
level: 1
});
log.push({
msg: `Query Planner: Storing ${
queries?.length || 0
} queries for ${dataset} in ${storageClient?.name}`,
level: 1
});
for (let query of queries) {
let comb = Object.keys(query)[0];
let q = Object.fromEntries(query[comb]);
let res = {
comb,
query: q,
dataset,
experimentName: experimentName.value,
queryId: queryCounter.success + 1
};
let response;
if (storageClient) {
response = await storageClient({
db: dbName,
collection: collectionName,
operation: "insert",
data: [res]
});
if (response?.acknowledged) {
queryCounter.success += 1;
} else {
queryCounter.failed += 1;
log.push({
msg: `Query Planner: Storing failed due to ${response} `,
level: 0
});
}
} else {
queries_.push(res);
queryCounter.success += 1;
}
}
log.push({
msg: `Query Planner: Storing Completed ${
queries?.length || 0
} queries for ${dataset} in ${
storageClient ? storageClient : "local"
}`,
level: 1
});
//Store the queries in case of no storage client
if (!storageClient) {
target.queries[experimentName.value] = queries_;
}
} catch (err) {
log.push({
msg: `Query Planner: Failed for ${dataset} with error ${err} `,
level: 0
});
console.error(
`Query Planner: Failed for ${dataset} with error ${err} `
);
}
}
log.push({
msg: `Query Planner: Execution Completed | Success ${queryCounter.success} | Failed ${queryCounter.failed}`,
level: 1
});
}
async function updateStatus() {
while (statusLock) {
//Queries
queriesStatus?.remove();
queriesStatus = htl.html`Running... | Generated ${queryCounter.success} | Failed ${queryCounter.failed}`;
target.appendChild(queriesStatus);
// Log
statusLog?.remove();
statusLog = html`<div style='font-size:15px'>${[
...log.filter((d) => d.level < 2).slice(-10)
]
.reverse()
.map((d, i) => (d.msg.length > 50 ? d.msg.slice(0, 50) + "..." : d.msg))
.join("<br>")}</div>`;
status.appendChild(statusLog);
await new Promise((d) => setTimeout(d, sleepTime));
}
}
updateStatus();
generateQueriesBtn.addEventListener("click", async () => {
generateQueriesBtn.childNodes[0].disabled = true;
statusLock = true;
updateStatus();
await runFn({
noOfExperiments: noOfExperiments.value,
comb: useCustomCombinations.value
? JSON.parse(customCombinations.querySelector("form").value)
: comb,
datasets: selectDatasets.value
});
target.log = log;
//Update the log and status before lock
updateStatus();
await new Promise((d) => setTimeout(d, sleepTime));
statusLock = false;
generateQueriesBtn.childNodes[0].disabled = false;
//Queries
queriesStatus?.remove();
queriesStatus = htl.html`Query Planner Execution Completed | Success ${queryCounter.success} | Failed ${queryCounter.failed}`;
target.appendChild(queriesStatus);
target.dispatchEvent(new Event("input", { bubbles: true }));
});
return target;
}
// John: I think I made it work 👍
function generateCombinations({ max_attributes_for_type = 2 } = {}) {
const types_of_attributes = ["C", "Q"];
const random_value = (max) => d3.randomInt(1, max)();
const number_of_attributes = [
[0, 1],
[1, 0],
[1, 1]
// [
// random_value(max_attributes_for_type),
// random_value(max_attributes_for_type)
// ]
];
const number_of_values = {
C: ["@V", "RV", "NV"],
Q: ["@V", "RV", "NV"]
};
let combinations = [];
for (let [nc, ns] of number_of_attributes) {
let query = "",
at;
if (ns && nc) {
at = "C";
for (let nv of number_of_values["C"]) {
query = `${nc}C-${nv}`;
for (let nvs of number_of_values["Q"]) {
query += `+${ns}Q-${nvs}`;
combinations.push(query);
}
}
} else if (nc) {
at = "C";
for (let nv of number_of_values[at]) {
query = `${nc}${at}-${nv}`;
combinations.push(query);
}
} else if (ns) {
at = "Q";
for (let nv of number_of_values[at]) {
query = `${ns}${at}-${nv}`;
combinations.push(query);
}
}
}
return combinations;
}
datasetsAccessor = getDatasets()
combinations = ["1C-"]
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
