25:["$","title",null,{"children":"GPT-3 token encoder and decoder | sadsadada | Observable"}] 49:[] 29:{"id":"56770b1be94f610d","slug":null,"title":"GPT-3 token encoder and decoder","thumbnail":"579a453f9f11af734352f5ac680f48126c539728fb91f3de8f089e1e020caa9a","access_level":"public","create_time":"$D2023-04-30T05:42:08.160Z","update_time":"$D2023-04-30T05:43:12.539Z","view_time":null,"first_public_time":"$D2023-04-30T05:42:08.160Z","has_unpublished_edits":null,"fork":{"id":"5523771744f27e16","slug":"gpt-tokenizer","title":"GPT token encoder and decoder","owner":{"id":"3b974c2db4452cfc","login":"simonw","name":"Simon Willison"}},"stars":0,"starred":false,"owner":{"type":"team","id":"8e6289f14d61644e","login":"asdadad","name":"sadsadada","avatar_url":"https://avatars-next.observableusercontent.com/255db70486c337e478f9eb7683239300073e0cf87711d8d0bd44a0f7ca78730f","tier":"starter_2024","bio":"","followed_by_you":false,"follows_you":false},"version":154,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"ca7083d390df9d80","login":"hs2002singhal","name":"Harsh Singhal","avatar_url":"https://avatars-next.observableusercontent.com/41b350e8a9395020c37914a541963ffc5befd244a77b622988e527d2a8b914a2","tier":null,"bio":"bruh loves cats","followed_by_you":false,"follows_you":false}],"members":[{"type":"team","id":"8e6289f14d61644e","login":"asdadad","name":"sadsadada","avatar_url":"https://avatars-next.observableusercontent.com/255db70486c337e478f9eb7683239300073e0cf87711d8d0bd44a0f7ca78730f","tier":"starter_2024","role":"editor"},{"type":"individual","id":"ca7083d390df9d80","login":"hs2002singhal","name":"Harsh Singhal","avatar_url":"https://avatars-next.observableusercontent.com/41b350e8a9395020c37914a541963ffc5befd244a77b622988e527d2a8b914a2","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"GPT-3 token encoder and decoder","cells":[{"id":72,"value":"# GPT-3 token encoder and decoder\n\nEnter text to tokenize it:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":77,"value":"viewof text = Inputs.textarea({ value: \"Example text is here\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":82,"value":"html`

${encode(text).join(\" \")}

${decode(tokens.split(/[^\\d+]/g).map((t) => parseInt(t, 10)))}

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":135,"value":"## Search tokens","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":125,"value":"viewof matches = Inputs.search(Object.entries(decoder), {\n placeholder: \"Search for a token\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":131,"value":"Inputs.table(matches.map((t) => ({ integer: t[0], token: t[1] })))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":0,"value":"Or... paste a JSON prompt in below, e.g.\n\n```json\n{\n \"model\": \"text-davinci-003\",\n \"prompt\": [\n [\n 50256, 14832, 25, 775, 550, 257, 1049, 2652, 13, 367,\n 14132, 287, 262, 12269, 373, 28294, 290, 262, 2057,\n 318, 1107, 922, 13, 198, 48, 25, 1867, 318, 262,\n 10238, 15598, 286, 428, 2423, 290, 1521, 30, 198, 32, 25\n ]\n ],\n \"max_tokens\": 64,\n \"temperature\": 0,\n \"logprobs\": 5,\n \"user\": \"lmql\",\n \"stream\": true,\n \"echo\": true,\n \"logit_bias\": {\n \"198\": -100,\n \"628\": -100,\n \"44320\": -100\n }\n}\n```","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":31,"value":"viewof prompt_raw = Inputs.textarea({ rows: 10 })","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":45,"value":"html`

${\n  prompt.prompt && decode(prompt.prompt[0])\n}

Logits${\n prompt.logit_bias &&\n Object.keys(prompt.logit_bias)\n .map((k) => `
${k} - \"${decode([k])}\": ${prompt.logit_bias[k]}`)\n .join(\"\")\n}`","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":57,"value":"decode([44320])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":37,"value":"prompt = {\n try {\n return JSON.parse(prompt_raw);\n } catch (e) {\n return {};\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"import {\n encode,\n byte_encoder,\n decodeStr,\n decoder\n} from \"@codingwithfire/gpt-3-encoder\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":9,"value":"encode(\"a and b and c\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":22,"value":"byte_decoder = {\n let tmp = {};\n Object.keys(byte_encoder).map((x) => {\n tmp[byte_encoder[x]] = x;\n });\n return tmp;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":25,"value":"function decode(tokens) {\n let text = tokens.map((x) => decoder[x]).join(\"\");\n text = decodeStr(text.split(\"\").map((x) => byte_decoder[x]));\n return text;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W49","stdlib":"1"}} 24:null