29:null 2a:["$","title",null,{"children":"The Pile - Test set token counts | Ian Johnson | Observable"}] 2f:[{"id":"29e91dd5e364404fd2955ec26ceaa01d6942d9ac3d0de67488ca1273ac6e4fa43fd8a817ad82627261ce9d3b4c5f3aac02c363d2bffaba1c1efc03ff1026d5de","href":"https://static.observableusercontent.com/files/29e91dd5e364404fd2955ec26ceaa01d6942d9ac3d0de67488ca1273ac6e4fa43fd8a817ad82627261ce9d3b4c5f3aac02c363d2bffaba1c1efc03ff1026d5de","name":"test-counts.parquet","type":"application/octet-stream","lastModified":1684679978966.188,"size":566342}] 2d:{"id":"c14148a1b0a96eb6","slug":"the-pile-test-set-token-counts","title":"The Pile - Test set token counts","thumbnail":"1487faf77db1a7577540eaee1efa6f61d28c72f089dfcc23b24ec0b267542728","access_level":"public","create_time":"$D2023-05-21T14:39:17.660Z","update_time":"$D2023-05-21T15:34:47.744Z","view_time":null,"first_public_time":"$D2023-05-21T16:03:49.257Z","has_unpublished_edits":null,"fork":false,"stars":1,"starred":false,"owner":{"type":"team","id":"cce0fa6e063d2bee","login":"enjalot","name":"Ian Johnson","avatar_url":"https://avatars-next.observableusercontent.com/549bdb0c368027004b51c4e95459e9c224331e0fec9c4cc97d181f83e55ab7f9","tier":"starter_2024"},"version":130,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"595405df3c534894","login":"enjalot","name":"Ian Johnson","avatar_url":"https://avatars-next.observableusercontent.com/549bdb0c368027004b51c4e95459e9c224331e0fec9c4cc97d181f83e55ab7f9","tier":null}],"members":[{"type":"individual","id":"595405df3c534894","login":"enjalot","name":"Ian Johnson","avatar_url":"https://avatars-next.observableusercontent.com/549bdb0c368027004b51c4e95459e9c224331e0fec9c4cc97d181f83e55ab7f9","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":{"id":"36699cd7a496ce71","slug":"the-pile","title":"The Pile","owner":{"id":"cce0fa6e063d2bee","login":"enjalot","name":"Ian Johnson"}},"type":"notebook","body":{"title":"The Pile - Test set token counts","cells":[{"id":0,"value":"# The Pile - Test set token counts\n[The Pile](https://pile.eleuther.ai/)'s test dataset contains ${nformat(counts.length)} documents from 22 datasets. When encoded with the [tiktoken](https://github.com/openai/tiktoken) `cl100k_base` BPE encoder (used for OpenAI embeddings) we get a total of ${nformat(d3.sum(counts, d => d.tokens))} tokens.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":39,"value":"Attached to this notebook is a parquet file with the pile_set_name for each row of the test dataset and the count of tokens. The full test dataset is ~450MB compressed so not practical to explore in the browser.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":5,"value":"counts = await FileAttachment(\"test-counts.parquet\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"counts","database":"$undefined"},{"id":7,"value":"totals = Plot.plot({\n caption: `Total tokens per pile set`,\n marks: [\n Plot.barX(counts, Plot.groupY({x: \"sum\"}, {y: \"pile_set_name\", x: \"tokens\", fill: \"pile_set_name\", sort: {y: \"x\", reverse: true, limit: 22}})),\n Plot.ruleX([0])\n ],\n marginLeft: 120,\n y: {\n label: null\n },\n grid: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":91,"value":"The longest document has ${nformat(d3.max(counts, d => d.tokens))} tokens, and the shortest has ${nformat(d3.min(counts, d => d.tokens))} tokens. There are ${counts.filter(d => d.tokens < 10).length} documents with 0 or 1 tokens.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":58,"value":"Let's look at the distribution of documents by how many tokens they have. Lets start with short documents, as some training regimes are filtering documents with less than 200 tokens. There are ${nformat(counts.filter(d => d.tokens < 200).length)} of these:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":65,"value":"tokenCounts(200)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":102,"value":"The OpenAI embeddings (and gpt-3.5-turbo and widely-available gpt-4) can work with 4000 tokens. There are ${nformat(counts.filter(d => d.tokens < 4000).length)} documents with less than 4k tokens:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":69,"value":"tokenCounts(4000)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":105,"value":"Some people have access to 8k tokens for gpt-4, that means they could handle an additional ${nformat(counts.filter(d => d.tokens > 4000 && d.tokens < 8000).length)} documents:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":72,"value":"tokenCounts(8000, 4000)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":108,"value":"With the 32k token context window you could handle an additional ${nformat(counts.filter(d => d.tokens > 8000 && d.tokens < 32000).length)} documents from the following sources:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":74,"value":"tokenCounts(32000, 8000)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":113,"value":"There are ${nformat(counts.filter(d => d.tokens > 32000).length)} documents with more than 32k tokens","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":85,"value":"tokenCounts(1000000, 32000)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":120,"value":"The above charts can be seen in proportion to each other in this faceted chart:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":41,"value":"Plot.plot({\n caption: `Count of documents within certain token thresholds`,\n facet: {\n data: counts,\n y: d => {\n if(d.tokens <= 200) {\n return \"sub200\"\n } else if(d.tokens < 4000) {\n return \"sub4k\"\n } else if(d.tokens < 8000) {\n return \"sub8k\"\n } else if(d.tokens < 32000) {\n return \"sub32k\"\n } else {\n return \"lots\"\n }\n }\n },\n marks: [\n Plot.barX(\n counts,\n Plot.groupY(\n { x: \"count\", title: \"count\" },\n {\n y: \"pile_set_name\",\n x: \"tokens\",\n fill: \"pile_set_name\",\n sort: { y: \"x\", reverse: true, limit: 22 }\n }\n )\n ),\n Plot.ruleX([0])\n ],\n marginLeft: 120,\n marginRight: 80,\n fy: {\n domain: [\"sub200\", \"sub4k\", \"sub8k\", \"sub32k\", \"lots\"]\n },\n y: {\n label: null\n },\n grid: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":30,"value":"---\n## Appendix","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":60,"value":"function tokenCounts(n, gt = 0){\nreturn Plot.plot({\n caption: `Count of documents with less than ${n} tokens` + (gt ? ` and more than ${gt} tokens.` : ``),\n marks: [\n Plot.barX(\n counts.filter(d => d.tokens <= n && d.tokens > gt),\n Plot.groupY(\n { x: \"count\", title: \"count\" },\n {\n y: \"pile_set_name\",\n x: \"tokens\",\n fill: \"pile_set_name\",\n sort: { y: \"x\", reverse: true, limit: 22 }\n }\n )\n ),\n Plot.ruleX([0])\n ],\n marginLeft: 120,\n marginRight: 80,\n fy: {\n domain: [\"sub200\", \"sub4k\", \"sub8k\", \"sub32k\", \"lots\"]\n },\n y: {\n label: null\n },\n color: { domain: colors.domain },\n grid: true,\n height: 300\n})\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":32,"value":"nformat = d3.format(\",d\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":23,"value":"colors = totals.scale(\"color\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W2f","stdlib":"1","titling":"auto"}} 2e:"$2d:body"