2c:null 2d:["$","title",null,{"children":"Embeddings playground (WIP) | John Alexis Guerra Gómez | Observable"}] 32:Tc58,create_parameter_form = (dr_method) => { switch (dr_method) { case "ISOMAP": case "LLE": case "LTSA": return html``; break; case "TSNE": return html``; break; case "UMAP": return html``; break; case "TriMap": return html``; break; case "LSP": return html``; case "SQDMDS": return html``; break; case "LDA": return html``; break; default: return html``; break; } }33:[] 30:{"id":"ba03b00b84e4a29e","slug":"embeddings-playground","title":"Embeddings playground (WIP)","thumbnail":"a6155231b20a6ea1243cf6ba26bae944c66095b457fb5cd0c4bb3805543e0809","access_level":"public","create_time":"$D2023-11-21T05:01:42.980Z","update_time":"$D2026-09-18T18:40:13.599Z","view_time":null,"first_public_time":"$D2024-10-24T21:30:45.950Z","has_unpublished_edits":null,"fork":false,"stars":2,"starred":false,"owner":{"type":"team","id":"cd3c1e49fc84d010","login":"john-guerra","name":"John Alexis Guerra Gómez","avatar_url":"https://avatars-next.observableusercontent.com/21638cee44bb0ea70798da725b57111a5d3e2acebcd14524565992f7d1594aed","tier":"starter_2024"},"version":601,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"be0076fec73aa550","login":"john-guerra","name":"John Alexis Guerra Gómez","avatar_url":"https://avatars-next.observableusercontent.com/21638cee44bb0ea70798da725b57111a5d3e2acebcd14524565992f7d1594aed","tier":null}],"members":[{"type":"individual","id":"be0076fec73aa550","login":"john-guerra","name":"John Alexis Guerra Gómez","avatar_url":"https://avatars-next.observableusercontent.com/21638cee44bb0ea70798da725b57111a5d3e2acebcd14524565992f7d1594aed","tier":null,"role":"editor"},{"type":"team","id":"cd3c1e49fc84d010","login":"john-guerra","name":"John Alexis Guerra Gómez","avatar_url":"https://avatars-next.observableusercontent.com/21638cee44bb0ea70798da725b57111a5d3e2acebcd14524565992f7d1594aed","tier":"starter_2024","role":"viewer"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Embeddings playground (WIP)","cells":[{"id":0,"value":"# Embeddings playground (WIP)\n\nA simple example to play with sentence embeddings and dimensionality reduction","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":7,"value":"viewof query = Inputs.text({label: \"Semantic search using embeddings\", value: \"\", submit: \"Submit\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":47,"value":"{\n // Execute after these two\n updateHistory;\n resetResults;\n\n console.log(\"Do navio\");\n \n return mutable history.length && navio(mutable history, {\n showSelectedAttrib: false,\n showSequenceIDAttrib: false,\n attribWidth: 2,\n addAllAttribsRecursionLevel: 0,\n height: 200,\n y0: 10\n });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":67,"value":"viewof reset = Inputs.button(\"reset\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":385,"value":"## Dimensionality Reduction","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":318,"value":"vl\n .markText({size: 18, fontFamily: \"Sans Serif\"})\n .encode(\n vl.text().fieldN(\"query\"),\n vl.x().fieldQ(\"x\").scale({zero: null}).axis(null),\n vl.y().fieldQ(\"y\").scale({zero: null}).axis(null)\n )\n .data(\n mutable history.map((d, i) => ({\n query: d.query,\n x: positions[i][0],\n y: positions[i][1]\n }))\n )\n .width(400)\n .height(400)\n .render()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":447,"value":"viewof druid_method","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":497,"value":"viewof reducerParams = form(create_parameter_form(druid_method))\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":277,"value":"### Query History","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":8,"value":"queryEmbedding = await extractor(query, {\n pooling: \"mean\",\n normalize: true\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"extractor = await transformers.pipeline(\n \"feature-extraction\",\n \"Xenova/all-MiniLM-L6-v2\",\n // \"Xenova/bert-base-uncased\",\n // \"Xenova/gte-small\",\n // \"allenai/specter2\",\n { pooling: \"mean\", quantized: false }\n)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":551,"value":"history","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":42,"value":"mutable history = []","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":453,"value":"// positions = {\n// history;\n// druid_method;\n// reducerParams;\n// history\n// const data = mutable history.map(d => Array.from(d.embedding))\n// console.log(\"yield dataForUmap \", data);\n// if (data.length) {\n// yield* druidGenerator(data, druid_method, reducerParams);\n// } else {\n// yield [];\n// }\n// }","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":43,"value":"updateHistory = {\n console.log(\"Do embedding\");\n if (query) {\n const embeddingsObj = Object.fromEntries(\n Array.from(queryEmbedding.data).map((d, i) => [i, d])\n );\n mutable history.push({ ...embeddingsObj, query, embedding: queryEmbedding.data });\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":78,"value":"resetResults = {\n reset;\n console.log(\"Reset history\");\n mutable history = [];\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":312,"value":"## UMAP","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":338,"value":"viewof nNeighbors = Inputs.range([0, 10], {value: 1, step: 0.01, label: \"UMAP neighborhoods\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":342,"value":"viewof nComponents = Inputs.range([0, 10], {value: 2, step: 1, label: \"UMAP nComponents\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":346,"value":"viewof minDist = Inputs.range([0, 20], {value: 0.1, label: \"UMAP minDist\", step: 0.01})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":310,"value":"config = ({\n nComponents,\n minDist,\n nNeighbors\n})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":299,"value":"// positions = {\n// updateHistory;\n// resetResults;\n \n// return Generators.observe(\n// worker(\n// fit,\n// { data: dataForUmap, show_dynamic: true, config },\n// `\n// const window = {};\n// importScripts(\"https://unpkg.com/umap-js@1.3.3/lib/umap-js.js\");\n// importScripts(\"https://unpkg.com/d3-random@2\");\n// const UMAP = window.UMAP;\n// Math.random = d3.randomLcg(${23});\n// `\n// )\n// );\n// }","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":538,"value":"positions = {\n updateHistory;\n resetResults;\n\n const dataForUmap = mutable history.map((d) => Array.from(d.embedding));\n\n yield* Generators.observe(\n worker(\n function* dr({ method, data, parameters }) {\n console.log(\"Initializing druid...\", data.length);\n yield [];\n yield* self.druid[method].generator(data, parameters); // show the steps\n },\n {\n method: druid_method,\n data: dataForUmap,\n parameters: reducerParams\n },\n\n `\n importScripts(${JSON.stringify(\n await require.resolve(\"@saehrimnir/druidjs@^0.7.3\")\n )});\n `\n )\n );\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":463,"value":"dataForUmap = mutable history.map(d => Array.from(d.embedding))","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":587,"value":"default_method = \"PCA\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":425,"value":"import {\n druidGenerator,\n chart as chartDR,\n viewof druid_method,\n \n} with { dataForUmap as dataAsMatrix , default_method} from \"@john-guerra/druidjs-playground\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":19,"value":"{\n // Execute after these two\n updateHistory;\n query;\n \n return Inputs.table(mutable history);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":9,"value":"transformers = import(\"https://cdn.jsdelivr.net/npm/@xenova/transformers@2.6.0\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":263,"value":"import {chart} from \"@john-guerra/umap-playground\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":264,"value":"import {\n worker,\n fit\n} with { dataForUmap as data } from \"@fil/umap-js-worker\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":17,"value":"import { navio } from \"@john-guerra/navio\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":496,"value":"$32","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":500,"value":"import { form } from \"@mbostock/form-input\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W33","stdlib":"1"}} 31:"$30:body"