2e:null 2f:["$","title",null,{"children":"Observable Notebook Compressed RAG Index + Vector Search | Tom Larkworthy | Observable"}] 34:T48f,vector_search = ({ prompt: "similarity search needs to return the cell_id, cell_path of the top matches", time: 1727050000000 } && async function vector_search(query, options = {}) { const n = options.n || 5; const embedding_full = await get_vector_embedding(query); const embedding = embedding_to_pca(embedding_full, means, proj); const rows = await embedding_db.query( `SELECT cell_id, cell_path, pca FROM compressed_embeddings` ); function cosine_similarity(a, b) { const dot = a.reduce((acc, val, i) => acc + val * b[i], 0); const normA = Math.sqrt(a.reduce((acc, val) => acc + val * val, 0)); const normB = Math.sqrt(b.reduce((acc, val) => acc + val * val, 0)); return dot / (normA * normB); } const similarities = rows.map((row) => { const sim = cosine_similarity(embedding, [...row.pca]); return { cell_id: row.cell_id, cell_path: row.cell_path, similarity: sim }; }); similarities.sort((a, b) => b.similarity - a.similarity); // Return only cell_id and cell_path return Promise.all(similarities.slice(0, n).map(get_code)); })35:Tcca,extract_definitions = ({ prompt: 'extract_definitions is not quite right. the define function variable number of args. The last arg is the definition, and the 2nd to last is the dependancies andsometimes a name is defined. Can you rewrite that, lets extract the dependancie string names as well. Rewrite extract_definitions, the matching logic is correct\n\n\n```\nfunction variable_define(name, inputs, definition) {\n switch (arguments.length) {\n case 1: {\n definition = name, name = inputs = null;\n break;\n }\n case 2: {\n definition = inputs;\n if (typeof name === "string") inputs = null;\n else inputs = name, name = null;\n break;\n }\n }\n return variable_defineImpl.call(this,\n name == null ? null : String(name),\n inputs == null ? [] : map.call(inputs, this._resolve, this),\n typeof definition === "function" ? definition : constant(definition)\n );\n}\n```', time: 1726922431995 } && function extract_definitions(notebook) { const definitions = {}; function traverse(node) { if ( node.type === "CallExpression" && node.callee.type === "MemberExpression" && node.callee.property.type === "Identifier" && node.callee.property.name === "define" ) { const args = node.arguments; let name = null; let dependencies = []; let definition = null; if (args.length === 1) { // define(definition) definition = args[0]; } else if (args.length === 2) { if (args[0].type === "ArrayExpression") { // define(dependencies, definition) dependencies = args[0].elements .filter( (el) => el.type === "Literal" && typeof el.value === "string" ) .map((el) => el.value); definition = args[1]; } else if ( args[0].type === "Literal" && typeof args[0].value === "string" ) { // define(name, definition) name = args[0].value; definition = args[1]; } } else if (args.length >= 3) { // define(name, dependencies, definition) if (args[0].type === "Literal" && typeof args[0].value === "string") { name = args[0].value; } if (args[1].type === "ArrayExpression") { dependencies = args[1].elements .filter( (el) => el.type === "Literal" && typeof el.value === "string" ) .map((el) => el.value); } definition = args[args.length - 1]; } if (definition) { if (definition.type == "Identifier") { definition = definition.name; } definitions[definition] = { name: name, dependencies: dependencies, definition: definition }; } } for (const key in node) { const child = node[key]; if (Array.isArray(child)) { child.forEach(traverse); } else if (typeof child === "object" && child !== null) { traverse(child); } } } traverse(notebook.export.fn); return definitions; })36:T442,create_tables = { ({ prompt: "Now insert into the database a tables called notebooks, and cells which contain the all the information in cells. \n\nNotebooks should contain the path (thats their primary key)\n\nCells should contain a reference to their notebook (foreign key), their id, their cell_name, their code.\n\nCross dependancies between cells should be in a separate table called deps", time: 1726936221270 }); const kb = await DuckDBClient.of({}); kb.query("SET memory_limit = '14GB';"); kb.query(` CREATE TABLE IF NOT EXISTS notebooks ( path TEXT PRIMARY KEY );`); kb.query(`CREATE TABLE IF NOT EXISTS cells ( id INTEGER, cell_name TEXT, code TEXT, notebook_path TEXT, FOREIGN KEY (notebook_path) REFERENCES notebooks(path), PRIMARY KEY (id, notebook_path) );`); kb.query(`CREATE TABLE IF NOT EXISTS deps ( cell_id INTEGER, cell_path TEXT, depends_on TEXT, PRIMARY KEY (cell_id, cell_path, depends_on), FOREIGN KEY (cell_id, cell_path) REFERENCES cells(id, notebook_path) );`); return kb; }37:T463,insertLinks = ({ prompt: "Now insert the data from linkedContent", time: 1726936427272 } && Promise.all( linkedContent.map(async (notebook) => { create_tables; await create_tables.query( `INSERT INTO notebooks (path) VALUES (?) ON CONFLICT(path) DO NOTHING`, [notebook.path] ); await Promise.all( notebook.cells.map(async (cell) => { await create_tables.query( `INSERT INTO cells (id, cell_name, code, notebook_path) VALUES (?, ?, ?, ?) ON CONFLICT(id, notebook_path) DO UPDATE SET cell_name = excluded.cell_name, code = excluded.code`, [cell.id, cell.cell_name, cell.code, notebook.path] ); if (cell.dependencies && cell.dependencies.length > 0) { cell.dependencies.map(async (dep) => { return await create_tables.query( `INSERT INTO deps (cell_id, cell_path, depends_on) VALUES (?, ?, ?) ON CONFLICT(cell_id, cell_path, depends_on) DO NOTHING`, [cell.id, notebook.path, dep] ); }); } }) ); }) ))38:T61b,embedding_to_pca = ({ prompt: "write a JS native way for embedding_to_pca(vector, components, means, scale, matrix). The libraries implementation is \n\n\n```\npublic predict(dataset: MaybeMatrix, options: PredictOptions = {}): Matrix {\n const { nComponents = (this.U as Matrix).columns } = options;\n let datasetmatrix;\n if (Array.isArray(dataset)) {\n datasetmatrix = new Matrix(dataset);\n } else {\n datasetmatrix = new Matrix(dataset);\n }\n if (this.center) {\n datasetmatrix.subRowVector(this.means as number[]);\n if (this.scale) {\n for (let i of this.excludedFeatures) {\n datasetmatrix.removeColumn(i);\n }\n datasetmatrix.divRowVector(this.stdevs as number[]);\n }\n }\n let predictions = datasetmatrix.mmul(this.U as Matrix);\n return predictions.subMatrix(0, predictions.rows - 1, 0, nComponents - 1);\n }\n```\n", time: 1727980602225 } && function embedding_to_pca(vector, means, components) { // Center the vector by subtracting the means let n = components.length / means.length; let centered = vector.map((val, i) => val - means[i]); // Perform matrix multiplication: centered vector * components matrix // Assuming components is an array of arrays (each sub-array is a component) const projection = Array.from({ length: cutoff }).map((_, i) => { let sum = 0; for (let j = 0; j < means.length; j++) { sum += components[j * n + i] * centered[j]; } return sum; }); return projection; })39:[] 32:{"id":"d76c2180ae7b30c9","slug":"notebook-rag","title":"Observable Notebook Compressed RAG Index + Vector Search","thumbnail":"ad7632ec517cc7b452e83e96ed2c0833a6bf42b4e46842bc1a89313c5d0bceab","access_level":"public","create_time":"$D2024-09-20T18:44:24.777Z","update_time":"$D2024-11-28T17:16:50.256Z","view_time":null,"first_public_time":"$D2024-10-03T21:21:40.577Z","has_unpublished_edits":null,"fork":{"id":"ae2987be1d73ac7e","slug":"robocoop-blank-slate","title":"Roboco-op: Blank Slate ","owner":{"id":"7db5ed2b0697d645","login":"tomlarkworthy","name":"Tom Larkworthy"}},"stars":1,"starred":false,"owner":{"type":"team","id":"7db5ed2b0697d645","login":"tomlarkworthy","name":"Tom Larkworthy","avatar_url":"https://avatars-next.observableusercontent.com/47327a8bc1966f2186dcb3ebf4b7ee6e4e7ab9a5c2a07405aff57200ea778f71","tier":"starter_2024"},"version":4435,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"5215f6ec4a999d40","login":"tomlarkworthy","name":"Tom Larkworthy","avatar_url":"https://avatars-next.observableusercontent.com/47327a8bc1966f2186dcb3ebf4b7ee6e4e7ab9a5c2a07405aff57200ea778f71","tier":null}],"members":[{"type":"individual","id":"5215f6ec4a999d40","login":"tomlarkworthy","name":"Tom Larkworthy","avatar_url":"https://avatars-next.observableusercontent.com/47327a8bc1966f2186dcb3ebf4b7ee6e4e7ab9a5c2a07405aff57200ea778f71","tier":null,"role":"editor"},{"type":"team","id":"7db5ed2b0697d645","login":"tomlarkworthy","name":"Tom Larkworthy","avatar_url":"https://avatars-next.observableusercontent.com/47327a8bc1966f2186dcb3ebf4b7ee6e4e7ab9a5c2a07405aff57200ea778f71","tier":"starter_2024","role":"editor"}],"comments":[],"custom_thumbnail":true,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":{"id":"ef392f92725a92d7","slug":"ai","title":"AI","owner":{"id":"7db5ed2b0697d645","login":"tomlarkworthy","name":"Tom Larkworthy"}},"type":"notebook","body":{"title":"Observable Notebook Compressed RAG Index + Vector Search","cells":[{"id":0,"value":"# Observable Notebook Compressed RAG Index + Vector Search\n\nUsing embeddings of notebook cells so we can do semantic similarity search on the cell level. This notebook has a toy vector search engine using a pre-prepared index, but the main point is that it builds the index.\n\nThe uncompressed index is 14Mb for 2600 cells. Using PCA we achieve 3.7Mb including cell source code, vectors, and the PCA projection. matrix.\n\nUsed to generate Roboco-op's [RAG index](https://observablehq.com/@tomlarkworthy/rag-extension?collection=@tomlarkworthy/robocoop)\n\nnote this requires an OpenAI API key to fetch the embeddings for a search term.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":3444,"value":"### Vector search","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":3489,"value":"viewof query = Inputs.text({\n value: \"plot the top 3 athletes\",\n width: \"100%\",\n label: \"query\",\n submit: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4033,"value":"md`${matches.map(\n (row) => md`\n

\n\nhttps://observablehq.com/d/${extractHexcode(row.cell_path)}\n\n

\n${md`~~~js\n${row.code}\n~~~`}\n

`\n)}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4046,"value":"extractHexcode = ({\n prompt:\n \"extract the last segment hexcode before the @ in `@tomlarkworthy/adapting-dataviz/32eeadb67cb4cbcb@1472.js`\",\n time: 1727977760396\n} &&\n function extractHexcode(filename) {\n const match = filename.match(/\\/([^/@]+)@/);\n return match ? match[1] : null;\n })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":3480,"value":"matches = vector_search(query, { n: 10 })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4418,"value":"get_code = async ({ cell_id, cell_path }) => {\n const result = await embedding_db.query(\n `SELECT code FROM cells WHERE id = ? AND notebook_path = ?`,\n [cell_id, cell_path]\n );\n return result.length > 0\n ? {\n cell_id,\n cell_path,\n code: result[0].code\n }\n : null;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":3460,"value":"$34","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":3453,"value":"viewof embedding_db = Inputs.input()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":3449,"value":"viewof embedding_file = Inputs.file({\n label: \"Upload embedding file\",\n accept: \".zip\",\n required: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":3451,"value":"{\n viewof embedding_db.value = await importDuckDB(embedding_file);\n viewof embedding_db.dispatchEvent(new Event(\"input\"));\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":3284,"value":"db = {\n const db = await DuckDBClient.of({});\n db.query(\"SET memory_limit = '10GB';\");\n return db;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":3948,"value":"### Retreve indexes from S3","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":4332,"value":"load_compressed_embeddings = {\n const embedding_file = await fetch(\n \"https://tomlarkworthy-access-aws.s3.eu-central-1.amazonaws.com/datasets/compressed.zip\"\n );\n viewof embedding_db.value = await importDuckDB(embedding_file);\n viewof embedding_db.dispatchEvent(new Event(\"input\"));\n return viewof embedding_db.value;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4335,"value":"load_pca_proj = (\n await fetch(\n \"https://tomlarkworthy-access-aws.s3.eu-central-1.amazonaws.com/datasets/pca_projection.json\"\n )\n).json()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4413,"value":"load_indexes = {\n load_compressed_embeddings;\n load_pca_proj;\n return \"Loaded\";\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":3939,"value":"# Indexing\n\nTo build the index we fetch notebook exports and run them through OpenAI's vector embeddings. Due to RAM constraints of the browser, this had to be done in steps with a fresh DB each step.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":3129,"value":"## Step 1: Build Corpus","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":2845,"value":"viewof file = Inputs.file({\n label: \"Upload content file\",\n accept: \".zip\",\n required: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2871,"value":"{\n const kb = importDuckDB(file, { client: db });\n viewof kb.value = kb;\n viewof kb.dispatchEvent(new Event(\"input\"));\n return kb;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2829,"value":"## Import from notebook Backups: [endpointservices/observable-notebooks](https://github.com/endpointservices/observable-notebooks)","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":2852,"value":"viewof fetchFromGithub = Inputs.button(\"import from Github\", {\n required: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2740,"value":"sources = {\n ({\n prompt:\n 'list all files in \"https://github.com/endpointservices/observable-notebooks\"',\n time: 1726857989123\n });\n fetchFromGithub; // Only if asked\n const owner = \"endpointservices\";\n const repo = \"observable-notebooks\";\n const branch = \"main\";\n\n const response = await fetch(\n `https://api.github.com/repos/${owner}/${repo}/git/trees/${branch}?recursive=1`\n );\n const data = await response.json();\n const files = data.tree\n .filter((item) => item.type === \"blob\")\n .map((item) => item.path)\n .filter((item) => item.endsWith(\".js\"));\n return files;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2753,"value":"latestSources = {\n ({\n prompt: \"reduce the source list to just the latest version of a given file\",\n time: 1726858685611\n });\n const fileMap = {};\n\n sources.forEach((path) => {\n // Example path: \"@endpointservices/auth/027541187c96745d@147.js\"\n const parts = path.split(\"/\");\n const fileNameWithVersion = parts[parts.length - 1];\n const match = fileNameWithVersion.match(/^(.*)@(\\d+)\\.js$/);\n\n if (match) {\n const baseName = match[1] + \".js\";\n const version = parseInt(match[2], 10);\n\n if (!fileMap[baseName] || version > fileMap[baseName].version) {\n fileMap[baseName] = { path, version };\n }\n }\n });\n\n return Object.values(fileMap).map((entry) => entry.path);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2759,"value":"extendedSources = {\n ({\n prompt:\n \"we have the latest versions of some files, but we are missing the module indexes like `@endpointservices/cellstore/index.js` we should also bring them into RAG\",\n time: 1726859116211\n });\n\n const moduleSet = new Set();\n\n // Extract unique module paths from latestSources\n latestSources.forEach((path) => {\n const parts = path.split(\"/\");\n if (parts.length >= 2) {\n const modulePath = parts.slice(0, 2).join(\"/\");\n moduleSet.add(`${modulePath}/index.js`);\n }\n });\n\n // Combine latestSources with the module index files\n const extendedSources = Array.from(new Set([...latestSources, ...moduleSet]));\n\n return extendedSources;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2756,"value":"highlight(latestSources)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2768,"value":"importSources = {\n ({\n prompt: \"create a duckdb database to index our files \",\n time: 1726859446011\n });\n await db.query(`\n CREATE TABLE IF NOT EXISTS files (\n path TEXT PRIMARY KEY,\n content TEXT\n )\n `);\n\n return await Promise.all(\n latestSources.map(async (path) => {\n const response = await fetch(\n `https://raw.githubusercontent.com/endpointservices/observable-notebooks/main/${path}`\n );\n const content = await response.text();\n await db.query(\n `INSERT INTO files (path, content) VALUES (?, ?) ON CONFLICT(path) DO UPDATE SET content=excluded.content`,\n [path, content]\n );\n })\n );\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2831,"value":"## Export corpus from file","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":2816,"value":"