2d:null 2e:["$","title",null,{"children":"Queries Generator | Kasi Viswanath | Observable"}] 33:T61f,Token | Used For - | - C | Categorical Attribute Q | Quantitative Attribute @ | All Values/Full Range R | Random Values/Random Range N | No Values/No Range

Function Parameter

Example

Rationale

-|-|- attributes | ["attr1","att2"] | Sample the scenario only from these attributes schema | {C:["attr1"],Q:["attr2","attr3"]} | Define Manual Schema for the attributes typeFns| { C: (d) => typeof d==="string" } | Define Manually the data type defination combinations | ["1C-RV","1Q-@V","1Q-NV","1Q-RV+1C-@V"] | Syntax : [(no_of_attributes_to_select)(C/Q)-(@/R/N)V]. documentation replicaCount | 10 | Used to create same query Combinations with different use cases uniqueFns | {'C' : (data,attribute)=> Array.from(d3.group(data, (d) => d[attribute]).keys())} | Unique value definations for specific data type selectionFns | C: (values, selected) => { if (selected === "@") { return values}} | Function for Selecting Values(@,N,R) based on each data Type maxRandomValues | 100 | maximum number of random values that can be selected when choosing either All Values or Random values. minValidScenarios | 0.1 | 0 { tempData.push(data[d]); }); data = tempData; } const valuesMap = new Map(); //If dataset have less categorical or quantitative attributes, this would help to create the queries on same attribute in case of multiple value combinations let index = 0; for (let query of queryCombinations) { const filter = new Map(); query = query.replaceAll(" ", ""); for (let comb of query.split("+")) { selectValues({ comb, valuesMap, attributesWithType, uniqueFns, selectionFns, data, filter, index }); index += 1; } console.log(filter, filter.size, query); if (filter.size === query.split("+").length) { queries.push({ [query]: filter }); } else { console.info("Query generation failed", query, filter); } } // Get Random Values function selectValues({ valuesMap, comb, attributesWithType, uniqueFns, selectionFns, filter, index }) { try { const regex = /([0-9]+)([C/Q])-(\d+|N|R|@)V/; const match = comb.match(regex); const noOfAttributes = +match[1]; const type = match[2]; let selected = isNaN(match[3]) ? match[3] : +match[3]; //Incase of creating min valid scenarios we make selected as Full(@) from shuffled data if (minValidQueries) { selected = "@"; } const uniqueFunctions = { C: (data, attribute) => Array.from(d3.group(data, (d) => d[attribute]).keys()), Q: (data, attribute) => d3.extent(data, (d) => d[attribute]) }; Object.assign(uniqueFunctions, uniqueFns); //Attribute shuffle const selectedAttributes = d3 .shuffle(attributesWithType[type]) .slice(0, noOfAttributes); if (selectedAttributes.length != noOfAttributes) { console.info(`Query Combination: Not Matched for ${comb} with dataset`); return; } for (let attr of selectedAttributes) { // Removed as the values shouldn't be same in the experiments // !!valuesMap.get(attr) || // valuesMap.set(attr, uniqueFunctions[type](data, attr)); const values = uniqueFunctions[type](data, attr); //@,N,R const selectionFunctions = { C: (values, selected) => { let noOfSelections = 0; let res; if (selected === "@") { res = values; } else if (selected === "R") { noOfSelections = d3.randomInt(1, values.length)(); noOfSelections = Math.min(noOfSelections, maxRandomValues); console.log({ noOfSelections, values: values.length }); res = // If noOfSelections > values return all values // Bug: selecting the values can have duplicate noOfSelections >= values.length ? values : Array.from({ length: noOfSelections }).map( (_) => values[d3.randomInt(0, values.length)()] ); } else if (+selected > 0) { res = // If noOfSelections > values return all values +selected >= values.length ? values : Array.from({ length: +selected }).map( (_) => values[d3.randomInt(0, values.length)()] ); } else { res = []; } return res; }, Q: (values, selected) => { let range = [0, 0]; if (selected === "@") { range = values; } else if (selected === "R") { let val1 = d3.randomInt(values[0], values[1])(); let val2 = d3.randomInt(values[0], values[1])(); range[0] = Math.min(val1, val2); range[1] = Math.max(val1, val2); } return range; } }; Object.assign(selectionFunctions, selectionFns); let res = selectionFunctions[type](values, selected); //Reducing the data based on size // while (sizeof(res) > memLimforSelection) { // res = res.slice(0, res.length / 2); // } filter.set(attr + "_" + index, { attr, type, selected: res }); } } catch (err) { console.log(err); return err; } } function getAttrType(data, typeFns) { const attributesWithType = {}; const typeFunctions = { C: (d) => isNaN(d), Q: (d) => +d > 0 }; Object.assign(typeFunctions, typeFns); //Initialize with type keys Object.keys(typeFunctions).forEach((type) => { attributesWithType[type] = []; }); // Get the attribute type for first matching type function for (let attr of attributes) { for (let [type, checkfn] of Object.entries(typeFunctions)) { if (checkfn(data[0][attr])) { attributesWithType[type].push(attr); break; } } } return attributesWithType; } return queries; }35:[] 31:{"id":"73c518f97c033885","slug":"queries-generator","title":"Queries Generator","thumbnail":"358830fe63ba502aaaf2708d64a3fd5486249a90de12844258f51450087852f0","access_level":"public","create_time":"$D2023-08-08T05:08:41.854Z","update_time":"$D2023-11-29T19:41:13.393Z","view_time":null,"first_public_time":"$D2023-08-08T05:08:41.854Z","has_unpublished_edits":null,"fork":{"id":"1c58a694295b498c","slug":"generate-queries","title":"Query Generator","owner":{"id":"b760ea566a896d7d","login":"kasivisu4","name":"Kasi Viswanath"}},"stars":0,"starred":false,"owner":{"type":"team","id":"b760ea566a896d7d","login":"kasivisu4","name":"Kasi Viswanath","avatar_url":"https://avatars-next.observableusercontent.com/48e36723913e7e5b18b906e1d7cde12fad4146715e78bbf74d3db7969a248b56","tier":"starter_2024"},"version":1081,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"fcd94e1c1ca12ae9","login":"kasivisu4","name":"Kasi Viswanath","avatar_url":"https://avatars-next.observableusercontent.com/48e36723913e7e5b18b906e1d7cde12fad4146715e78bbf74d3db7969a248b56","tier":null}],"members":[{"type":"individual","id":"be0076fec73aa550","login":"john-guerra","name":"John Alexis Guerra Gómez","avatar_url":"https://avatars-next.observableusercontent.com/21638cee44bb0ea70798da725b57111a5d3e2acebcd14524565992f7d1594aed","tier":null,"role":"editor"},{"type":"individual","id":"fcd94e1c1ca12ae9","login":"kasivisu4","name":"Kasi Viswanath","avatar_url":"https://avatars-next.observableusercontent.com/48e36723913e7e5b18b906e1d7cde12fad4146715e78bbf74d3db7969a248b56","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":{"id":"fb28ec1f4856bd04","slug":"benchmark-tools","title":"Benchmark tools","owner":{"id":"b760ea566a896d7d","login":"kasivisu4","name":"Kasi Viswanath"}},"type":"notebook","body":{"title":"Queries Generator","cells":[{"id":0,"value":"# Queries Generator\n\n${clipBoardCopy(`import {queriesGenerator} from \"@kasivisu4/queries-generator\"`)}\n\n\nThis module samples the provided data and generates queries based on various combinations that can be used for our [benchmarking framework](https://observablehq.com/@kasivisu4/filtering-benchmark-intro?collection=@kasivisu4/data-filtering-benchmark). \n\n\nTo familiarize yourself with the module, Please review the following documentation\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":188,"value":"$33","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":861,"value":"To-Do:\n- Should restructure the return value as key inside the map also defined as attr (see ex1 where key and attr both are name)\n- Check if minValidScenarios is working.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":429,"value":"### Ex 1 : Simple Query - Choosing two car names from the cars dataset","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":611,"value":"queriesGenerator(cars, { queryCombinations: [\"1C-2V\"], attributes: [\"name\"] })\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":812,"value":"### If we want generate 10 different queries","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":513,"value":"data = queriesGenerator(cars, {\n queryCombinations: [\"1C-2V\"],\n attributes: [\"name\"],\n replicaCount: 10\n})\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":819,"value":"### Ex 2 : Complex Query - Choosing two car names with random cylinders","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":824,"value":"queriesGenerator(cars, {\n queryCombinations: [\"1C-2V+1Q-RV\"],\n attributes: [\"name\", \"cylinders\"]\n})\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":864,"value":"### Ex 3 : Create Valid queries from the data based on minValidQueries thresold","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":841,"value":"queriesGenerator(cars, {\n queryCombinations: [\"1C-2V+1Q-RV\"],\n attributes: [\"name\", \"cylinders\"],\n minValidScenarios: 0.1\n})\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":979,"value":"### Ex4:If the number of attributes chosen exceeds the available attributes in the dataset, we will not generate the queries\nIn the cars dataset, there's only one categorical attribute. If the query combination specifies the selection of two attributes, it won't generate any queries","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":960,"value":"queriesGenerator(cars, {\n queryCombinations: [\"2C-2V\"],\n attributes: [\"name\"],\n replicaCount: 10\n})\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":988,"value":"### Ex 5: Lets' try to generate categorical attributes from Flight dataset. But there are no categorical attributes!","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":998,"value":"queriesGenerator(flights, {\n queryCombinations: [\"1C-@V+1Q-@V\"]\n})\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1078,"value":"queriesGenerator(moma, {\n queryCombinations: [\"1C-RV\"],\n attributes: [\"Department\"]\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1037,"value":"comb = [\n \"1C-@V\",\n \"1C-RV\",\n \"1C-NV\",\n \"1C-@V+1C-RV\",\n \"1C-@V+1C-NV\",\n \"1C-NV+1C-RV\",\n \"1C-@V+1C-RV+1C-NV\",\n \"1Q-@V\",\n \"1Q-RV\",\n \"1Q-NV\",\n \"1Q-@V+1Q-RV\",\n \"1Q-@V+1Q-NV\",\n \"1Q-NV+1Q-RV\",\n \"1Q-@V+1Q-RV+1Q-NV\",\n \"1Q-@V+1C-@V\",\n \"1Q-RV+1C-RV\",\n \"1Q-NV+1C-NV\",\n \"1Q-@V+1Q-RV+1C-@V+1C-RV\",\n \"1Q-@V+1Q-NV+1C-@V+1C-NV\",\n \"1Q-NV+1Q-RV+1C-NV+1C-RV\",\n \"1Q-@V+1Q-RV+1Q-NV +1C-@V+1C-RV+1C-NV\"\n]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":142,"value":"$34","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":440,"value":"import { clipBoardCopy } from \"@kasivisu4/filterer-utilities\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":993,"value":"flights = {\n let data = await getDatasets();\n await data.getFlights();\n return data.datasets.Flights;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1041,"value":"moma = {\n let data = await getDatasets();\n await data.getMoma();\n return data.datasets.Moma;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":991,"value":"import { getDatasets } from \"@kasivisu4/datasets\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W35","stdlib":"1","titling":"auto"}} 32:"$31:body"