2b:null 2c:["$","title",null,{"children":"Search Query Tokenization | Fabian Iwand | Observable"}] 31:T767,/** * @param {(string | Function)[] | null} [fields] - An array of property names or getters. Defaults to searching in all values. * @param {Object} options * @param {(v: any) => string} options.coerce - Function to convert field values to strings. */ function filterFields(fields = null, options = {}) { const { coerce = v => String(v).toLowerCase() } = options; const indexed = new Map(); const index = d => { const strings = fields ? fields.flatMap(k => typeof k === "function" ? k(d).map(coerce) : coerce(d[k])) : Object.values(d).map(coerce); // Store a map of property names to values indices. strings._map = Object.fromEntries((fields ?? Object.keys(d)).map((k, i) => [k, i])); indexed.set(d, strings); return strings; }; return q => { const terms = tokenize(q.toLowerCase()); if(!terms.length) return () => true; const matcher = ({term, tags, quoted}) => { const withFilter = filter => tags?.length ? strings => tags.some(t => filter(strings[strings._map[t]] ?? "")) : strings => strings.some(filter); // Treat backtick quoted strings as regular expressions. if(quoted === "`") { try { const pattern = new RegExp(term, "i"); return withFilter(s => pattern.test(s)); } catch { return withFilter(() => false); } } return withFilter(s => s.includes(term)); } const any = terms.filter(d => d.modifier === null).map(matcher); const all = terms.filter(d => d.modifier === "+").map(matcher); const none = terms.filter(d => d.modifier === "-").map(matcher); return d => { const strings = indexed.get(d) ?? index(d); const match = fn => fn(strings); return !none.some(match) && (!all.length || all.every(match)) && (!any.length || any.some(match)) } } }32:T41d,// (modifier)?(tag delimiter){0,}(term | quote term quote) tokenize = (query, { modifiers: prefixChars = "-+", quotes: quoteChars = "\"'`", delimiter = ":", } = {}) => { const modifiers = Object.fromEntries([...prefixChars].map(k => [k, true])); const quotes = Object.fromEntries([...quoteChars].map(k => [k, true])); let term, tags, modifier, quote, quoted; const terms = []; const reset = () => { term = ""; tags = []; modifier = null; quote = null; quoted = null }; const commit = () => { if(term.length) terms.push({term, tags, modifier, quoted}); reset(); }; reset(); for(const c of query) { if(quote === c) quote = null; else if(quote === null && c === " ") commit(); else if(quote === null && !term.length && !tags.length && modifier === null && modifiers[c]) modifier = c; else if(quote === null && term.length && c === delimiter) tags.push(term), term = "", quoted = null; else if(quote === null && quotes[c]) quoted = quote = c; else term += c; } commit(); return terms; }33:[] 2f:{"id":"08ff5a07f4fe3702","slug":"tokenize-query","title":"Search Query Tokenization","thumbnail":"e917f20a490e5d2af8ecff923ca84f5eeb539f6c5e651673feb5a39756194220","access_level":"public","create_time":"$D2024-05-22T00:49:59.779Z","update_time":"$D2025-10-17T19:07:05.859Z","view_time":null,"first_public_time":"$D2024-05-23T09:01:08.673Z","fork":false,"stars":1,"starred":false,"owner":{"type":"team","id":"898d35f8860c7f03","login":"mootari","name":"Fabian Iwand","avatar_url":"https://avatars-next.observableusercontent.com/0d7defa821f38094c03bad23b9b360a5364e6e97e21fc238c39ddc48db7994ad","tier":"starter_2024"},"version":309,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"07362516b5994994","login":"mootari","name":"Fabian Iwand","avatar_url":"https://avatars-next.observableusercontent.com/0d7defa821f38094c03bad23b9b360a5364e6e97e21fc238c39ddc48db7994ad","tier":null}],"members":[{"type":"individual","id":"07362516b5994994","login":"mootari","name":"Fabian Iwand","avatar_url":"https://avatars-next.observableusercontent.com/0d7defa821f38094c03bad23b9b360a5364e6e97e21fc238c39ddc48db7994ad","tier":null,"role":"editor"},{"type":"team","id":"898d35f8860c7f03","login":"mootari","name":"Fabian Iwand","avatar_url":"https://avatars-next.observableusercontent.com/0d7defa821f38094c03bad23b9b360a5364e6e97e21fc238c39ddc48db7994ad","tier":"starter_2024","role":"viewer"}],"comments":[],"custom_thumbnail":true,"trashed":false,"subscribed":null,"forks":2,"collection":{"id":"e3842c7642c32750","slug":"tools","title":"Tools & Techniques","owner":{"id":"898d35f8860c7f03","login":"mootari","name":"Fabian Iwand"}},"type":"notebook","body":{"title":"Search Query Tokenization","cells":[{"id":0,"value":"# Search Query Tokenization","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":173,"value":"This notebook provides a filter callback for `Inputs.search()` as well as a versatile query tokenization helper.\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":235,"value":"---\n## Using the filter callback\n\n~~~js\nimport {filterFields} from \"@mootari/tokenize-query\"\n~~~\n~~~js\nInputs.search(data, {filter: filterFields()})\n~~~\n\nAll terms are matched case insensitive. Query examples:\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":160,"value":"example_queries = {\n const examples = {\n \"cycling judo\": `cycling OR judo`,\n \"cycling judo +aus\": `(cycling OR judo) AND aus`,\n \"cycling judo -aus\": `(cycling OR judo) AND NOT aus`,\n \"\\\"modern pentathlon\\\"\": `field value includes whitespace`,\n \"`abbas$`\": `regular expression`,\n \"weight:`^6[0-3]$`\": `regular expression for .weight`,\n \"-gold:silver:bronze:0\": `none of the properties .gold, .silver or .bronze include \"0\"`,\n };\n const click = query => () => {\n const input = viewof example_results.input;\n input.value = query;\n input.dispatchEvent(new Event(\"input\", {bubbles: true}));\n }\n const apply = query => htl.html`