2b:["$","title",null,{"children":"GPT token encoder and decoder | Simon Willison | Observable"}] 30:[] 2e:{"id":"5523771744f27e16","slug":"gpt-tokenizer","title":"GPT token encoder and decoder","thumbnail":"c88a2338f43d2db4c66dc75f5b2b242f0829b66e64b366df96c1d92bdb24a4f8","access_level":"public","create_time":"$D2023-04-27T21:35:46.222Z","update_time":"$D2023-06-08T21:01:20.993Z","view_time":null,"first_public_time":"$D2023-04-27T21:35:46.222Z","has_unpublished_edits":null,"fork":false,"stars":46,"starred":false,"owner":{"type":"team","id":"3b974c2db4452cfc","login":"simonw","name":"Simon Willison","avatar_url":"https://avatars-next.observableusercontent.com/78a40181b372a172b86abe69479cab717df036b9cdcbaab5a197e45a76ffb4f3","tier":"starter_2024"},"version":352,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"dc026342127b807b","login":"simonw","name":"Simon Willison","avatar_url":"https://avatars-next.observableusercontent.com/78a40181b372a172b86abe69479cab717df036b9cdcbaab5a197e45a76ffb4f3","tier":null}],"members":[{"type":"individual","id":"dc026342127b807b","login":"simonw","name":"Simon Willison","avatar_url":"https://avatars-next.observableusercontent.com/78a40181b372a172b86abe69479cab717df036b9cdcbaab5a197e45a76ffb4f3","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":29,"collection":null,"type":"notebook","body":{"titling":"auto","title":"GPT token encoder and decoder","cells":[{"id":72,"value":"# GPT token encoder and decoder\n\nFor more information on this tool, read [Understanding GPT tokenizers](https://simonwillison.net/2023/Jun/8/gpt-tokenizers/)\n\nEnter text to tokenize it:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":77,"value":"viewof text = Inputs.textarea({ value: \"Example text is here\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":82,"value":"html`${encode(text).join(\" \")}
${\n encode(text).length\n} token${encode(text).length == 1 ? \"\" : \"s\"}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":160,"value":"html`${spans}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":88,"value":"
Or convert tokens to text:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":93,"value":"viewof tokens = Inputs.textarea({\n placeholder: \"Paste space separated tokens here\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":337,"value":"Note that this tool uses the GPT-2 tokenizer, which differs slightly from the tokenizer used by more recent models. This is useful primarily as an educational tool for understanding how tokenization works.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":106,"value":"html`

${decode(tokens.split(/[^\\d+]/g).map((t) => parseInt(t, 10)))}

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":135,"value":"## Search tokens","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":125,"value":"viewof matches = Inputs.search(Object.entries(decoder), {\n placeholder: \"Search for a token\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":131,"value":"Inputs.table(\n matches.map((t) => ({ integer: t[0], token: t[1].replace(\"Ġ\", \"\\u00A0\") }))\n)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":0,"value":"Or... paste a JSON prompt in below, e.g.\n\n```json\n{\n \"model\": \"text-davinci-003\",\n \"prompt\": [\n [\n 50256, 14832, 25, 775, 550, 257, 1049, 2652, 13, 367,\n 14132, 287, 262, 12269, 373, 28294, 290, 262, 2057,\n 318, 1107, 922, 13, 198, 48, 25, 1867, 318, 262,\n 10238, 15598, 286, 428, 2423, 290, 1521, 30, 198, 32, 25\n ]\n ],\n \"max_tokens\": 64,\n \"temperature\": 0,\n \"logprobs\": 5,\n \"user\": \"lmql\",\n \"stream\": true,\n \"echo\": true,\n \"logit_bias\": {\n \"198\": -100,\n \"628\": -100,\n \"44320\": -100\n }\n}\n```","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":31,"value":"viewof prompt_raw = Inputs.textarea({ rows: 10 })","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":45,"value":"html`

${\n  prompt.prompt && decode(prompt.prompt[0])\n}

Logits${\n prompt.logit_bias &&\n Object.keys(prompt.logit_bias)\n .map((k) => `
${k} - \"${decode([k])}\": ${prompt.logit_bias[k]}`)\n .join(\"\")\n}`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":185,"value":"spans = await generateSpans(text)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":162,"value":"async function generateSpans(text) {\n const tokens = encode(text);\n const htmlPromises = bundleTokens(tokens).map(\n async (t) =>\n `${t.tokens.join(\n \"
\"\n )}${t.text.replace(\"\\n\", \"
\").replace(\" \", \" \")}`\n );\n const html = await Promise.all(htmlPromises);\n return html.join(\"\");\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":309,"value":"bundleTokens(encode(text))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":37,"value":"prompt = {\n try {\n return JSON.parse(prompt_raw);\n } catch (e) {\n return {};\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"import {\n encode,\n byte_encoder,\n decodeStr,\n decoder\n} from \"@codingwithfire/gpt-3-encoder\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":9,"value":"encode(\"a and b and c\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":22,"value":"byte_decoder = {\n let tmp = {};\n Object.keys(byte_encoder).map((x) => {\n tmp[byte_encoder[x]] = x;\n });\n return tmp;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":25,"value":"function decode(tokens) {\n let text = tokens.map((x) => decoder[x]).join(\"\");\n text = decodeStr(text.split(\"\").map((x) => byte_decoder[x]));\n return text;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":154,"value":"decode([1276])\n","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":170,"value":"async function stringToPastelColor(str) {\n const msgUint8 = new TextEncoder().encode(str);\n const hashBuffer = await crypto.subtle.digest(\"SHA-256\", msgUint8);\n const hashArray = Array.from(new Uint8Array(hashBuffer)).slice(0, 3); // Taking only the first 3 bytes\n let color = \"#\";\n for (const value of hashArray) {\n let adjustedValue = Math.floor(value / 2) + 128; // Adjust the range to be between 128 and 256\n color += (\"00\" + adjustedValue.toString(16)).substr(-2);\n }\n return color;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":258,"value":"## Solving the problem with multi-token characters, e.g. Japanese\n\nThis tool wasn't working correctly with Japanese characters. The character 本 for example was rendered like this:\n\n � �\n\nThis is because 本 is represented as two tokens - 17312 105 - and when you revers decode those tokens individually you get back an invalid character. The following expression:\n\n decode([17312]).charCodeAt(0) // This does the same thing for 105 as well\n\nReturns 65533 - which `String.fromCharCode(65533)` confirms is the Unicode \"replacement character\" indicating an invalid character.\n\nThis is a problem because the tool likes to wrap each decoded token in a `` in order to colour highlight it.\n\nThe solution is to scan the entire response and try to find any tokens which, on their own, decode to 65533 but when combined with the subsequent token instead decode to a valid character.\n\nThese character sequences can then be bundled together into a single ``","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":230,"value":"[decode([17312]).charCodeAt(0), decode([103]).charCodeAt(0)]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":249,"value":"String.fromCharCode(65533)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":270,"value":"[decode([17312, 103]).charCodeAt(0), decode([17312, 103])]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":277,"value":"function bundleTokens(tokens) {\n // Given an array of integer tokens, return an array of {\"string\": \"...\", \"tokens\" [1, 3]}\n let bundles = [];\n let copied = Array.from(tokens);\n let current = [];\n while (copied.length) {\n current.push(copied.shift());\n if (decode(current).charCodeAt(0) == 65533) {\n continue;\n }\n bundles.push({ text: decode(current), tokens: current });\n current = [];\n }\n return bundles;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":324,"value":"Here's a bug I still need to resolve - the string \"本本\" breaks things, as seen below:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":326,"value":"encode(\"本本\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":297,"value":"JSON.stringify(bundleTokens([17312, 105, 42164, 105]))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W30","stdlib":"1"}} 2f:"$2e:body" 2a:E{"digest":"NEXT_REDIRECT;replace;/@simonw/gpt-tokenizer;307;"}