Public
Edited
Jun 8, 2023
19 forks
44 stars
Inputs.table(
matches.map((t) => ({ integer: t[0], token: t[1].replace("Ġ", "\u00A0") }))
)
viewof prompt_raw = Inputs.textarea({ rows: 10 })
html`<pre>${
prompt.prompt && decode(prompt.prompt[0])
}</pre><p><strong>Logits</strong>${
prompt.logit_bias &&
Object.keys(prompt.logit_bias)
.map((k) => `<br>${k} - "${decode([k])}": ${prompt.logit_bias[k]}`)
.join("")
}`
spans = await generateSpans(text)
async function generateSpans(text) {
const tokens = encode(text);
const htmlPromises = bundleTokens(tokens).map(
async (t) =>
`<span title="${t.tokens}" style="
padding: 3px;
border-right: 3px solid white;
line-height: 3em;
font-family: courier;
background-color: ${await stringToPastelColor(t.text)};
position: relative;
"><span style="position: absolute; top: 5.5ch; line-height: 1em; left: -0.5px; font-size: 0.45em">${t.tokens.join(
"<br>"
)}</span>${t.text.replace("\n", "<br>").replace(" ", " ")}</span>`
);
const html = await Promise.all(htmlPromises);
return html.join("");
}
bundleTokens(encode(text))
prompt = {
try {
return JSON.parse(prompt_raw);
} catch (e) {
return {};
}
}
import {
encode,
byte_encoder,
decodeStr,
decoder
} from "@codingwithfire/gpt-3-encoder"
encode("a and b and c")
byte_decoder = {
let tmp = {};
Object.keys(byte_encoder).map((x) => {
tmp[byte_encoder[x]] = x;
});
return tmp;
}
function decode(tokens) {
let text = tokens.map((x) => decoder[x]).join("");
text = decodeStr(text.split("").map((x) => byte_decoder[x]));
return text;
}
decode([1276])
async function stringToPastelColor(str) {
const msgUint8 = new TextEncoder().encode(str);
const hashBuffer = await crypto.subtle.digest("SHA-256", msgUint8);
const hashArray = Array.from(new Uint8Array(hashBuffer)).slice(0, 3); // Taking only the first 3 bytes
let color = "#";
for (const value of hashArray) {
let adjustedValue = Math.floor(value / 2) + 128; // Adjust the range to be between 128 and 256
color += ("00" + adjustedValue.toString(16)).substr(-2);
}
return color;
}
[decode([17312]).charCodeAt(0), decode([103]).charCodeAt(0)]
String.fromCharCode(65533)
[decode([17312, 103]).charCodeAt(0), decode([17312, 103])]
function bundleTokens(tokens) {
// Given an array of integer tokens, return an array of {"string": "...", "tokens" [1, 3]}
let bundles = [];
let copied = Array.from(tokens);
let current = [];
while (copied.length) {
current.push(copied.shift());
if (decode(current).charCodeAt(0) == 65533) {
continue;
}
bundles.push({ text: decode(current), tokens: current });
current = [];
}
return bundles;
}
encode("本 本")
JSON.stringify(bundleTokens([17312, 105, 42164, 105]))
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
