spans = await generateSpans(text)
async function generateSpans(text) {
const tokens = encode(text).slice(1);
const htmlPromises = tokens.map(async (t) => {
let text = decode([1, t]);
return `<span style="
padding: 3px;
border-right: 3px solid white;
line-height: 3em;
font-family: courier;
background-color: ${await stringToPastelColor(text)};
position: relative;
"><span style="position: absolute; top: 5.5ch; line-height: 1em; left: -0.5px; font-size: 0.45em">${t}</span>${text}</span>`;
});
const html = await Promise.all(htmlPromises);
return html.join("");
}
llamatok = import(
"https://belladoreai.github.io/llama-tokenizer-js/llama-tokenizer.js"
)
llamatok.default.encode("one and two and three")
function encode(text) {
return llamatok.default.encode(text);
}
function decode(tokens) {
return llamatok.default.decode(tokens);
}
decode([1, 697])
async function stringToPastelColor(str) {
const msgUint8 = new TextEncoder().encode(str);
const hashBuffer = await crypto.subtle.digest("SHA-256", msgUint8);
const hashArray = Array.from(new Uint8Array(hashBuffer)).slice(0, 3); // Taking only the first 3 bytes
let color = "#";
for (const value of hashArray) {
let adjustedValue = Math.floor(value / 2) + 128; // Adjust the range to be between 128 and 256
color += ("00" + adjustedValue.toString(16)).substr(-2);
}
return color;
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
