2d:null 2e:["$","title",null,{"children":"Presentation: On-device LLM with Gemini Nano | Ryan Seddon | Observable"}] 33:T1380,deck = [ `# On-device LLM with Gemini Nano Chrome has an experiment to allow a language model run entirely on your device!`, `## What is it? - It's an early preview program - Only in Chrome Dev+ with flags enabled - It's roughly ~2GB (uncompressed to 22GB on fs!) - Surprisingly powerful`, `## How do I get it - https://observablehq.com/@ryanseddon/chrome-ai - chrome://flags/#optimization-guide-on-device-model - chrome://flags/#prompt-api-for-gemini-nano \`\`\`js (await ai.languageModel.capabilities()).available // -> "readily" | "after-download" | "no" \`\`\` `, md`## APIs \`\`\`js const session = await ai.languageModel.create(); session.prompt('Write me a joke about JavaScript') \`\`\` ${promptJokeButton} `, `## APIs \`prompt()\` will probably never ship :( \`\`\`js ai.summarizer() ai.writer() // Generate text, tone, context ai.rewriter() // Rewrite text in certain style translator.translate() \`\`\` `, `

`, ` \`\`\`js await ai.createTextSession() await ai.createGenericSession() await ai.assistant.create() await ai.languageModel.create() \`\`\` `, `## \`prompt()\` focus - Most interesting - Works how you're used to working with LLMs - Is for finding use cases - Has \`promptStreaming()\` for faster response times `, `## Why? - Privacy, all happens on device - Compliance, PII, sensitive data - No network latency, promptStreaming has sub ms repsonse times - Trade off is device resources - Access across all origins, so better than WebGPU models `, `## Resources From my less than scientific investigations: - Uses ~2GB of memory while actively generating text - With an active session uses ~0.8GB of memory - promptStreaming would return a chunk every 0.05ms on average `, `## Limitations - Prompt has a 1024 tokens per prompt - The actual context window is 4096 tokens - Has a sliding window of context - System prompt is kept in the window - It's bad at jokes and factual stuff`, `## Is it just a toy? - Maybe - It is quite capable with the task specific APIs - Think of it as a layered approach - This may never ship and was just a fun thing `, `## Advanced use-cases Let's take a look at some more advanced stuff we can do`, `## N-Shot prompting \`\`\`js await ai.languageModel.create({ initialPrompts: [ { role: "system", content: "Predict up to 5 emojis as a response to a comment. Output emojis, comma-separated." }, { role: "user", content: "This is amazing!" }, { role: "assistant", content: "❤️, ➕" }, { role: "user", content: "LGTM" }, { role: "assistant", content: "👍, 🚢" } ] }); \`\`\` `, md`## Emojis ${emojiPrediction} `, `

`, `## JSON \`\`\`js await ai.languageModel.create({ initialPrompts: [ { role: "system", content: "Predict the sentiment of the text. Output either neutral, postive or negative." }, { role: "user", content: "This is amazing!" }, { role: "assistant", content: '{"sentiment": "positive"}' } ] }); \`\`\` `, md`## JSON ${jsonFormat}`, `## Session cloning - The trick here is to use \`session.clone()\` - This allows you to not accidentally overflow you guidance - No clone would mean the json example could be lost due to windowing `, `## Complex JSON - Open discussion on the [prompt-api explainer repo](https://github.com/explainers-by-googlers/prompt-api/issues/35) - Existing projects: [Guidance-ts](https://github.com/mmoskal/guidance-ts), [AiBrow](https://github.com/axonzeta/aibrow), [TypeChat](https://github.com/microsoft/TypeChat/tree/main/typescript) - TypeChat sort of works with Nano now - Others require JSON Schema, GBNF Grammars `, `

`, gist("ryanseddon", "703ef4eefbe56e3f92c50c5298b2dc18"), gist("ryanseddon", "8f43d21196b83bf75db5495b7ed891c9"), `## Curling Nano A lot of eval tools such as TypeChat assume a server side LLM - So I made a node script that spins up playwright and calls prompt api - Now I can directly curl or use tools that assume an endpoint - Follows the Open AI \`/v1/chat/completions\` schema`, `

`, md`# Thanks! ${llmPoem}`, { theme: ["mytheme", "dark"] } ]34:T44c,css = ` .mytheme-slide { font-family: sans-serif; color: #d5d5d5; background: linear-gradient(to bottom, #000000, #1a1a1a, #333333);ackground-color: yellow; h1 { font-size: 6vw; font-weight: bold; background: linear-gradient(to right, #4db8ff, #b380ff); /* Use similar colors to the screenshot */ -webkit-background-clip: text; color: transparent; text-wrap: balance; } :is(h2,h3,h4) { font-weight: bold; background: linear-gradient(to bottom, #ffffff, #808080, #333333); -webkit-background-clip: text; -webkit-text-fill-color: transparent; text-wrap: balance; } h2 { font-size: 4.5vw; } h3 { font-size: 3.5vw; } h4 { font-size: 2.5vw; } p, button, textarea { font-size: 2vmin; } button { background: linear-gradient(to right, #4db8ff, #b380ff); border: 1px solid #fff; border-radius: 6px; box-shadow: rgba(27, 31, 35, .1) 0 1px 0; box-sizing: border-box; color: #fff; cursor: pointer; &:hover { background-color: #2c974b; } } blockquote, ol, ul { max-width: none; } `35:[] 31:{"id":"d39e424aaf18379f","slug":"melbjs-nov-24","title":"Presentation: On-device LLM with Gemini Nano","thumbnail":"c0a0dfd8c1d5b6d0c401ce17a4f2b9a44441f22ed147af7a358e0a4a7cc95533","access_level":"public","create_time":"$D2024-11-09T02:07:41.127Z","update_time":"$D2024-11-14T09:22:10.281Z","view_time":null,"first_public_time":"$D2024-11-13T08:59:21.925Z","has_unpublished_edits":null,"fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"cd78d380af44b04c","login":"ryanseddon","name":"Ryan Seddon","avatar_url":"https://avatars-next.observableusercontent.com/b2f992859f577d48af545def264d0fb872c02baf1f96b8fa789f9926f6797ecd","tier":"starter_2024"},"version":562,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"484caa07491278f7","login":"ryanseddon","name":"Ryan Seddon","avatar_url":"https://avatars-next.observableusercontent.com/b2f992859f577d48af545def264d0fb872c02baf1f96b8fa789f9926f6797ecd","tier":null}],"members":[{"type":"individual","id":"484caa07491278f7","login":"ryanseddon","name":"Ryan Seddon","avatar_url":"https://avatars-next.observableusercontent.com/b2f992859f577d48af545def264d0fb872c02baf1f96b8fa789f9926f6797ecd","tier":null,"role":"editor"},{"type":"team","id":"cd78d380af44b04c","login":"ryanseddon","name":"Ryan Seddon","avatar_url":"https://avatars-next.observableusercontent.com/b2f992859f577d48af545def264d0fb872c02baf1f96b8fa789f9926f6797ecd","tier":"starter_2024","role":"viewer"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"Presentation: On-device LLM with Gemini Nano","cells":[{"id":96,"value":"# Presentation: On-device LLM with Gemini Nano\n\nA presentation I did for MelbJS Nov 13, 2024.\n\nNote: You need to have the Chrome AI model all setup otherwise this presentation may not work","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":548,"value":"instructions = htl.html`

Setup Instructions

**Install Chrome Dev**: Ensure you have version 127. [Download Chrome Dev](https://google.com/chrome/dev/).
Check that you’re on 127.0.6512.0 or above
Enable two flags:\n
- chrome://flags/#optimization-guide-on-device-model - BypassPerfRequirement
- chrome://flags/#prompt-api-for-gemini-nano - Enabled
\n
Relaunch Chrome
Navigate to chrome://components
Check that Optimization Guide On Device Model is downloading or force download if not\n Might take a few minutes for this component to even appear
Open dev tools and type \\`(await ai.languageModel.capabilities()).available\\`, should return \"readily\" when all good

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":15,"value":"presentation = slideshow","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":339,"value":"currentSlide","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":21,"value":"$33","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":127,"value":"## Slide show stuff\n-----\n\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":0,"value":"import { slideshow, styles, currentSlide } with {\n deck,\n css,\n theme,\n title,\n select\n} from \"@jerryjappinen/slides\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":470,"value":"import { embed, gist, youtube } from \"@jerryjappinen/embed\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":69,"value":"title = width > 350 ? \"Ryan Seddon | Nov 24 | MelbJS\" : \"\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":200,"value":"select = true","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":51,"value":"theme = [\"mytheme\", \"dark\"]","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":24,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":36,"value":"slideshowStyles = styles","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":232,"value":"promptJokeButton = actionInput(\n \"Do it!\",\n \"Tell me a joke about JavaScript, 100 characters max.\",\n async (e, prompt, output) => {\n e.stopPropagation();\n const joke = await nShotSession(prompt, [], true);\n\n for await (const chunk of joke) {\n output.value = chunk;\n }\n }\n)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":362,"value":"emojiPrediction = actionInput(\n \"Get emojis\",\n \"Giving a talk at a local meetup\",\n async (e, prompt, output) => {\n e.stopPropagation();\n\n const emojis = await nShotSession(prompt, [\n {\n role: \"system\",\n content:\n \"Predict up to 5 emojis as a response to a comment. Output emojis, comma-separated.\"\n },\n { role: \"user\", content: \"This is amazing!\" },\n { role: \"assistant\", content: \"❤️, ➕\" },\n { role: \"user\", content: \"LGTM\" },\n { role: \"assistant\", content: \"👍, 🚢\" }\n ]);\n\n output.value = emojis;\n }\n)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":411,"value":"jsonFormat = actionInput(\n \"Get sentiment\",\n \"Why is my order lost!\",\n async (e, prompt, output) => {\n e.stopPropagation();\n\n const json = await nShotSession(prompt, [\n {\n role: \"system\",\n content:\n \"Predict the sentiment of the text. Output either neutral, postive or negative.\"\n },\n { role: \"user\", content: \"This is amazing!\" },\n { role: \"assistant\", content: '{\"sentiment\": \"positive\"}' }\n ]);\n\n output.value = json;\n }\n)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":502,"value":"llmPoem = actionInput(\n \"Do it!\",\n \"Write a haiku about LLMs in the browser.\",\n async (e, prompt, output) => {\n e.stopPropagation();\n const haiku = await nShotSession(prompt, [], true);\n\n for await (const chunk of haiku) {\n output.value = chunk;\n }\n }\n)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":281,"value":"session = {\n /*if (canCreate) {\n const LM = await ai.languageModel.create({\n systemPrompt: \"You're a comedian who writes very short punchy jokes\"\n });\n\n return LM;\n } else {\n return \"Chrome AI not available\";\n }*/\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":131,"value":"actionButton = (label, action) => {\n const node = htl.html`

`;\n const button = node.querySelector(\"button\");\n\n button.addEventListener(\"click\", async (e) => await action(e));\n\n return node;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":402,"value":"actionInput = (label, prompt, action) => {\n const styles = \"width: 100%; field-sizing: content;\";\n const node = htl.html`

`;\n const button = node.querySelector(\"button\");\n const textarea = node.querySelector(\"#prompt\");\n const output = node.querySelector(\"#output\");\n\n button.addEventListener(\n \"click\",\n async (e) => await action(e, textarea.value, output)\n );\n textarea.addEventListener(\"click\", (e) => e.stopPropagation());\n output.addEventListener(\"click\", (e) => e.stopPropagation());\n\n return node;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":355,"value":"nShotSession = async (prompt, intialPrompts, streamMode = false) => {\n //session.destroy();\n const newSession = await window?.ai.languageModel.create({\n initialPrompts: intialPrompts\n });\n\n // Clone an existing session for efficiency, instead of recreating one each time.\n async function predictEmoji(comment) {\n const freshSession = await newSession.clone();\n if (streamMode) {\n return await freshSession.promptStreaming(comment);\n }\n\n return await freshSession.prompt(comment);\n }\n\n const res = await predictEmoji(prompt);\n\n return res;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W35","stdlib":"1","titling":"auto"}} 32:"$31:body"