Hey friends! I'm a fullstack engineering based out of Brooklyn, New York. I'm really interested in programming as a communication medium, tools for thought, and building more humane software.
Public
Edited
Jun 1, 2021
6 stars
function tokenize(src, sourceMapping = false) {
const tokens = [];
const chars = src.split('');
let buffer = '';
let line = 0;
let col = 0;
// Containers for token values which are set at the start of buffer creation
let posStart = 0;
let lineStart = 0;
let colStart = 0;
/**
* Writes to the buffer and sets any relavent positional context that may be needed
*/
const writeBuffer = (text, pos, colNum, lineNum) => {
if (buffer.length === 0) {
buffer = text;
posStart = pos;
colStart = colNum;
lineStart = lineNum;
} else {
buffer += text;
}
col++;
};
// Writes characters in the buffer as a token, takes the positions, and clears the buffer
const flush = () => {
if (buffer.length) {
const colEnd = colStart + buffer.length - 1;
const endPos = posStart + buffer.length - 1;
tokens.push(
sourceMapping
? {
value: buffer,
line: lineStart,
colStart,
colEnd,
start: posStart,
end: endPos
}
: buffer
);
}
buffer = '';
};
for (let i = 0; i < chars.length; ++i) {
const char = chars[i];
const nextChar = chars[i + 1];
if (char === '\n') {
line += 1;
col = 0;
continue;
}
// Support the ability to escape symbols
if (char === "\\") {
writeBuffer(char + nextChar, i, col, line);
i += 1; // skip lexing the next character altogether
continue;
}
// Tokenize grouping symbols
if (char === "(" || char === ")" || char === '"') {
flush();
writeBuffer(char, i, col, line);
flush();
continue;
}
// Build up tokens
if (char.match(/\S/)) {
writeBuffer(char, i, col, line);
} else {
flush();
col++; // account for whitespace
}
}
// Tokenize anything remaining in the buffer
flush();
return tokens;
}
function parse(src) {
const tokens = tokenize(src, true);
let result = consumeTokens(tokens, src);
// If there's only one defined expression, return early
if (!tokens.length) return result;
result = [result];
while (tokens.length) {
result.push(consumeTokens(tokens, src));
}
return result;
}
function consumeTokens(tokens, src) {
if (!tokens.length) throw new ParseError(PARSE_ERROR.EOF);
const token = tokens.shift();
switch (token.value) {
case "(": {
const expression = [];
while (tokens[0].value !== ")") {
expression.push(consumeTokens(tokens, src));
if (!tokens.length) throw new ParseError(PARSE_ERROR.EOF);
}
tokens.shift();
return expression;
}
case '"':
/**
* Strings are a little different. Given that strings are whitespace sensitive and tokenization
* discards whitespaces, we simply capture the start and end positions and use that to grab the string.
*/
const stringStart = token.start + 1;
while (tokens[0].value !== '"' && tokens[0].value !== "\n") {
tokens.shift(); // discard token
if (!tokens.length) throw new ParseError(PARSE_ERROR.EOF);
}
if (tokens[0].value === "\n") throw new ParseError(PARSE_ERROR.expectedClosingQuotes)
const stringEnd = tokens.shift().end;
return src.slice(stringStart, stringEnd);
case ")":
throw new ParseError(PARSE_ERROR.unexpectedClosing);
default:
const { value } = token;
// Convert token a number if it's number-like
return isNaN(+value) ? value : +value;
}
}
globals = ({
"+": (scope, ...operands) => operands.reduce((acc, curr) => acc + curr, 0),
"-": (scope, ...operands) => operands.reduce((acc, curr) => acc - curr),
"*": (scope, ...operands) => operands.reduce((acc, curr) => acc * curr, 1)
})
specialForms = ({
'let': (e, scope, variable, value, expression) => {
const lexicalScope = { ...scope }; // Should deep copy eventually
lexicalScope[variable] = value;
return e(expression, lexicalScope);
},
fn: (e, scope, functionName, argNames, body) => {
scope[functionName] = (fnScope, ...args) => {
const argNameValuePairs = argNames.map((arg, index) => [
arg,
args[index],
]);
console.log(argNameValuePairs);
console.log(
argNameValuePairs
);
const fnLocalScope = {
...fnScope,
...argNames
.map((arg, index) => [arg, args[index]])
.reduce((acc, [arg, val]) => ({ ...acc, [arg]: val }), {})
};
return e(body, fnLocalScope);
};
},
})
function evalExpression(astExpression, scope) {
if (!Array.isArray(astExpression)) {
return typeof astExpression === "string" && astExpression in scope
? scope[astExpression]
: astExpression;
}
const operation = astExpression[0];
if (operation in specialForms) {
return specialForms[operation](
evalExpression,
scope,
...astExpression.slice(1)
);
}
for (const [index, node] of astExpression.entries()) {
astExpression[index] = evalExpression(node, scope);
}
if (!(operation in scope)) return astExpression;
return scope[operation](scope, ...astExpression.slice(1));
}
function evaluate(src) {
const ast = parse(src);
// Handle case of multiple expressions
if (Array.isArray(ast[0])) {
let result;
let scope = { ...globals };
for (const expression of ast) {
result = evalExpression(expression, scope);
}
// Returns the result of the last expression
return result;
}
// If it's only a single expression, eval that and return
return evalExpression(ast, { ...globals });
}
tests(evaluate, [
["Evals a simple expression", "(+ 2 2)", 4],
["Evals a complex expression", "(+ 2 (+ 3 5))", 10],
["Evals subtraction", "(- 2 2)", 0],
["Evals subtraction chain", "(- 10 4 2)", 4],
["Evals a multiplication complex expression", "(+ 3 (* 3 2))", 9],
["Evals a stand-alone expression", '1', 1],
["Evals a simple variable assignement", "(let x 10 (+ 5 x))", 15],
["Enables simple fn declarations", "(fn add (a b) (+ a b)) (add 1 2)", 3]
])
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
