2b:null 2c:["$","title",null,{"children":"Search Query Tokenization | brettforbes | Observable"}] 31:T767,/** * @param {(string | Function)[] | null} [fields] - An array of property names or getters. Defaults to searching in all values. * @param {Object} options * @param {(v: any) => string} options.coerce - Function to convert field values to strings. */ function filterFields(fields = null, options = {}) { const { coerce = v => String(v).toLowerCase() } = options; const indexed = new Map(); const index = d => { const strings = fields ? fields.flatMap(k => typeof k === "function" ? k(d).map(coerce) : coerce(d[k])) : Object.values(d).map(coerce); // Store a map of property names to values indices. strings._map = Object.fromEntries((fields ?? Object.keys(d)).map((k, i) => [k, i])); indexed.set(d, strings); return strings; }; return q => { const terms = tokenize(q.toLowerCase()); if(!terms.length) return () => true; const matcher = ({term, tags, quoted}) => { const withFilter = filter => tags?.length ? strings => tags.some(t => filter(strings[strings._map[t]] ?? "")) : strings => strings.some(filter); // Treat backtick quoted strings as regular expressions. if(quoted === "`") { try { const pattern = new RegExp(term, "i"); return withFilter(s => pattern.test(s)); } catch { return withFilter(() => false); } } return withFilter(s => s.includes(term)); } const any = terms.filter(d => d.modifier === null).map(matcher); const all = terms.filter(d => d.modifier === "+").map(matcher); const none = terms.filter(d => d.modifier === "-").map(matcher); return d => { const strings = indexed.get(d) ?? index(d); const match = fn => fn(strings); return !none.some(match) && (!all.length || all.every(match)) && (!any.length || any.some(match)) } } }32:T41d,// (modifier)?(tag delimiter){0,}(term | quote term quote) tokenize = (query, { modifiers: prefixChars = "-+", quotes: quoteChars = "\"'`", delimiter = ":", } = {}) => { const modifiers = Object.fromEntries([...prefixChars].map(k => [k, true])); const quotes = Object.fromEntries([...quoteChars].map(k => [k, true])); let term, tags, modifier, quote, quoted; const terms = []; const reset = () => { term = ""; tags = []; modifier = null; quote = null; quoted = null }; const commit = () => { if(term.length) terms.push({term, tags, modifier, quoted}); reset(); }; reset(); for(const c of query) { if(quote === c) quote = null; else if(quote === null && c === " ") commit(); else if(quote === null && !term.length && !tags.length && modifier === null && modifiers[c]) modifier = c; else if(quote === null && term.length && c === delimiter) tags.push(term), term = "", quoted = null; else if(quote === null && quotes[c]) quoted = quote = c; else term += c; } commit(); return terms; }33:[] 2f:{"id":"2f7bad814948a218","slug":null,"title":"Search Query Tokenization","thumbnail":"51d81f5d2f62e2d6ec8d07ab795e07559f4682628bcbf9e0eefee6dd524b0c9c","access_level":"public","create_time":"$D2026-06-12T11:34:06.797Z","update_time":"$D2026-06-12T11:34:06.797Z","view_time":null,"first_public_time":"$D2026-06-12T11:34:06.797Z","has_unpublished_edits":null,"fork":{"id":"08ff5a07f4fe3702","slug":"tokenize-query","title":"Search Query Tokenization","owner":{"id":"898d35f8860c7f03","login":"mootari","name":"Fabian Iwand"}},"stars":0,"starred":false,"owner":{"type":"team","id":"f846f459b144e893","login":"brettforbes","name":"","avatar_url":"https://avatars-next.observableusercontent.com/90abd4233f342d3bde898a6f28cf0f1a64652b13845939fedb9e12eda77f7bd0","tier":"starter_2024"},"version":309,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"e3ae8b6a8e4f693b","login":"brettforbes","name":"","avatar_url":"https://avatars-next.observableusercontent.com/90abd4233f342d3bde898a6f28cf0f1a64652b13845939fedb9e12eda77f7bd0","tier":null}],"members":[{"type":"individual","id":"e3ae8b6a8e4f693b","login":"brettforbes","name":"","avatar_url":"https://avatars-next.observableusercontent.com/90abd4233f342d3bde898a6f28cf0f1a64652b13845939fedb9e12eda77f7bd0","tier":null,"role":"editor"},{"type":"team","id":"f846f459b144e893","login":"brettforbes","name":"","avatar_url":"https://avatars-next.observableusercontent.com/90abd4233f342d3bde898a6f28cf0f1a64652b13845939fedb9e12eda77f7bd0","tier":"starter_2024","role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"Search Query Tokenization","cells":[{"id":0,"value":"# Search Query Tokenization","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":173,"value":"This notebook provides a filter callback for `Inputs.search()` as well as a versatile query tokenization helper.\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":235,"value":"---\n## Using the filter callback\n\n~~~js\nimport {filterFields} from \"@mootari/tokenize-query\"\n~~~\n~~~js\nInputs.search(data, {filter: filterFields()})\n~~~\n\nAll terms are matched case insensitive. Query examples:\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":160,"value":"example_queries = {\n const examples = {\n \"cycling judo\": `cycling OR judo`,\n \"cycling judo +aus\": `(cycling OR judo) AND aus`,\n \"cycling judo -aus\": `(cycling OR judo) AND NOT aus`,\n \"\\\"modern pentathlon\\\"\": `field value includes whitespace`,\n \"`abbas$`\": `regular expression`,\n \"weight:`^6[0-3]$`\": `regular expression for .weight`,\n \"-gold:silver:bronze:0\": `none of the properties .gold, .silver or .bronze include \"0\"`,\n };\n const click = query => () => {\n const input = viewof example_results.input;\n input.value = query;\n input.dispatchEvent(new Event(\"input\", {bubbles: true}));\n }\n const apply = query => htl.html`