Public
Edited
Jun 8, 2023
19 forks
43 stars
Inputs.table(
matches.map((t) => ({ integer: t[0], token: t[1].replace("Ġ", "\u00A0") }))
)
viewof prompt_raw = Inputs.textarea({ rows: 10 })
html`<pre>${
prompt.prompt && decode(prompt.prompt[0])
}</pre><p><strong>Logits</strong>${
prompt.logit_bias &&
Object.keys(prompt.logit_bias)
.map((k) => `<br>${k} - "${decode([k])}": ${prompt.logit_bias[k]}`)
.join("")
}`
spans = await generateSpans(text)
async function generateSpans(text) {
const tokens = encode(text);
const htmlPromises = bundleTokens(tokens).map(
async (t) =>
`<span title="${t.tokens}" style="
padding: 3px;
border-right: 3px solid white;
line-height: 3em;
font-family: courier;
background-color: ${await stringToPastelColor(t.text)};
position: relative;
"><span style="position: absolute; top: 5.5ch; line-height: 1em; left: -0.5px; font-size: 0.45em">${t.tokens.join(
"<br>"
)}</span>${t.text.replace("\n", "<br>").replace(" ", " ")}</span>`
);
const html = await Promise.all(htmlPromises);
return html.join("");
}
bundleTokens(encode(text))
prompt = {
try {
return JSON.parse(prompt_raw);
} catch (e) {
return {};
}
}
import {
encode,
byte_encoder,
decodeStr,
decoder
} from "@codingwithfire/gpt-3-encoder"
encode("a and b and c")
byte_decoder = {
let tmp = {};
Object.keys(byte_encoder).map((x) => {
tmp[byte_encoder[x]] = x;
});
return tmp;
}
function decode(tokens) {
let text = tokens.map((x) => decoder[x]).join("");
text = decodeStr(text.split("").map((x) => byte_decoder[x]));
return text;
}
decode([1276])
async function stringToPastelColor(str) {
const msgUint8 = new TextEncoder().encode(str);
const hashBuffer = await crypto.subtle.digest("SHA-256", msgUint8);
const hashArray = Array.from(new Uint8Array(hashBuffer)).slice(0, 3); // Taking only the first 3 bytes
let color = "#";
for (const value of hashArray) {
let adjustedValue = Math.floor(value / 2) + 128; // Adjust the range to be between 128 and 256
color += ("00" + adjustedValue.toString(16)).substr(-2);
}
return color;
}
[decode([17312]).charCodeAt(0), decode([103]).charCodeAt(0)]
String.fromCharCode(65533)
[decode([17312, 103]).charCodeAt(0), decode([17312, 103])]
function bundleTokens(tokens) {
// Given an array of integer tokens, return an array of {"string": "...", "tokens" [1, 3]}
let bundles = [];
let copied = Array.from(tokens);
let current = [];
while (copied.length) {
current.push(copied.shift());
if (decode(current).charCodeAt(0) == 65533) {
continue;
}
bundles.push({ text: decode(current), tokens: current });
current = [];
}
return bundles;
}
encode("本 本")
JSON.stringify(bundleTokens([17312, 105, 42164, 105]))
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.