EXTERNAL_TOKENIZER = (name, terms) => {
const CHAR_SPACE = " ".codePointAt(0);
const CHAR_DOT = ".".codePointAt(0);
const CHAR_STAR = "*".codePointAt(0);
const CHAR_LBRACKET = "{".codePointAt(0);
return new ExternalTokenizer(
(input, stack) => {
if (
(input.peek(-1) === CHAR_SPACE || input.peek(0) === CHAR_SPACE) &&
stack.canShift(terms.whitespace)
) {
input.acceptToken(terms.whitespace, 0);
return;
}
},
{ extend: true }
);
}