3e:null 3f:["$","title",null,{"children":"ObservableHQ Notebooks: Building and Compiling | Mikhail Kotelnikov | Observable"}] 45:T4cd,{ // Prepare DIV container where the results of the executions are shown const container = html`

`; const define = async (runtime, observer) => { const id = "main"; // const notebook = buildNotebook(id, code); // const compiledModules = await compileNotebook({ // runtime, // observer, // notebook, // resolve : resolveModule // }); // return compiledModules[0]; // Transform the text code to an array of JSON objects with cell definitions. // All cell methods are serialized as simple text. const moduleDefinition = buildModule(id, code); // Transform methods code to executable functions. return await compileModule({ runtime, observer, moduleDefinition, resolve: resolveModule // This function resolves imports }); }; // Create a new runtime const nrt = new rt.Runtime(); // Create a new module const main = nrt.module(define, (name) => { // if (name && name.indexOf('mutable ') === 0) return ; const div = html`

`; container.appendChild(div); return new rt.Inspector(div); }); invalidation.then(() => nrt.dispose()); return container; }46:T52d,async function resolveModule({ source, runtime, observer }) { const path = extractPath(source); console.log("RESOLVE MODULE URL", source, path); if (source[0] === "#") { console.log("DEFINE MODULE", path); const main = runtime.module(); main .variable(observer("totot")) .define( "totot", ["html", "name"], (html, name) => html`Hello ${name}` ); main.variable(observer("name")).define("name", function () { return "Totot"; }); return main; } const moduleUrl = `https://api.observablehq.com/${path}.js?v=3`; const module = (await new Function(`return import("${moduleUrl}")`)()) .default; return module(runtime, (name) => { if (!name) return false; console.log("IMPORT", path, name); return true; }); function extractPath(path) { let source = path; let m; // "https://api.observablehq.com/@jashkenas/inputs.js?v=3" => strip off ".js" if ((m = /\.js(\?|$)/i.exec(source))) source = source.slice(0, m.index); // "74f872c4fde62e35" => "d/..." if ((m = /^[0-9a-f]{16}$/i.test(source))) source = `d/${source}`; // link of notebook if ((m = /^https:\/\/(api\.|beta\.|)observablehq\.com\//i.exec(source))) source = source.slice(m[0].length); return source; } }47:T5d2,function getCellDefinitions(cell) { const { references, code } = getCellCodeAndReferences(cell); const TYPE = "cell"; let name = null; if (cell.id && cell.id.name) name = cell.id.name; else if (cell.id && cell.id.id && cell.id.id.name) name = cell.id.id.name; // if (!name) name = `$cell_${getCellDefinitions.id = (getCellDefinitions.id || 0) + 1}`; const definitions = []; if (cell.id && cell.id.type === "ViewExpression") { definitions.push({ type: TYPE, name: `viewof ${name}`, references, method: code }); definitions.push({ type: TYPE, name: `${name}`, references: ["Generators", `viewof ${name}`], method: `function value_${name}(Generators, $) { return Generators.input($); }` }); } else if (cell.id && cell.id.type === "MutableExpression") { definitions.push({ type: TYPE, name: `mutable initial ${name}`, references, method: code }); definitions.push({ type: TYPE, name: `mutable ${name}`, references: ["Mutable", `mutable initial ${name}`], method: `function mutable_${name}(Mutable, $) { return new Mutable($); }` }); definitions.push({ type: TYPE, name: `${name}`, references: [`mutable ${name}`], method: `function value_${name}($) { return $.generator; }` }); } else { definitions.push({ type: TYPE, name, references, method: code }); } return definitions; }48:T8f0,function getCellCodeAndReferences(cell) { const { walk } = parser; const { simple } = acornWalk; let bodyText = cell.input.substring(cell.body.start, cell.body.end); const _getRefName = (ref) => { if (!ref) return null; if (ref.type === "ViewExpression") { return "viewof " + ref.id.name; } else if (ref.type === "MutableExpression") { return "mutable " + ref.id.name; } else return ref.name; }; const references = (cell.references || []).map(_getRefName); let $count = 0; let indexShift = 0; const _updateText = (node, $string) => { const start = node.start - cell.body.start; const end = node.end - cell.body.start; bodyText = bodyText.slice(0, start + indexShift) + $string + bodyText.slice(end + indexShift); indexShift += $string.length - (end - start); }; const codeReferences = (cell.references || []).map((ref) => { if (ref.type === "ViewExpression") { const $string = "$" + $count++; // replace "viewof X" in bodyText with "$($count)" simple( cell.body, { ViewExpression(node) { _updateText(node, $string); } }, walk ); return $string; } else if (ref.type === "MutableExpression") { const $string = "$" + $count++; // replace "mutable Y" in bodyText with "$($count).value" simple( cell.body, { MutableExpression(node) { _updateText(node, $string + ".value"); } }, walk ); return $string; } else return ref.name; }); let code; if (cell.body.type !== "BlockStatement") { if (cell.async) code = `${bodyText}`; else code = `{\nreturn (${bodyText});\n}`; } else code = `${bodyText}`; const refs = codeReferences.join(","); const name = _getRefName(cell.id); const functionName = name ? name.replace(/\s/gim, "_") : ``; if (cell.generator && cell.async) code = `async function* ${functionName}(${refs}) ${code}`; else if (cell.async) code = `async function ${functionName}(${refs}) ${code}`; else if (cell.generator) code = `function* ${functionName}(${refs}) ${code}`; else code = `function ${functionName}(${refs}) ${code} `; return { references, code }; }49:[] 43:{"id":"a903fc89735a14fe","slug":"observablehq-notebooks-building-and-compiling","title":"ObservableHQ Notebooks: Building and Compiling","thumbnail":"7b184d5c809934e78a7b9bff6a60c94059a2e9629895423bfddff80c281e26c3","access_level":"public","create_time":"$D2019-11-09T22:16:22.469Z","update_time":"$D2021-07-02T22:37:13.181Z","view_time":null,"first_public_time":"$D2019-11-10T14:11:42.615Z","has_unpublished_edits":null,"fork":true,"stars":13,"starred":false,"owner":{"type":"team","id":"d4c65cdf77f8f4fd","login":"kotelnikov","name":"Mikhail Kotelnikov","avatar_url":"https://avatars-next.observableusercontent.com/529fa53c5b4acc1414273473d4e049e502c44c1bdbb1a0af81f7faa463dcff4f","tier":"starter_2024","bio":""},"version":736,"publish_version":736,"role":null,"authored":false,"authors":[{"type":"individual","id":"cdbfd308e8310490","login":"kotelnikov","name":"Mikhail Kotelnikov","avatar_url":"https://avatars-next.observableusercontent.com/529fa53c5b4acc1414273473d4e049e502c44c1bdbb1a0af81f7faa463dcff4f","tier":null,"bio":""}],"members":[{"type":"individual","id":"cdbfd308e8310490","login":"kotelnikov","name":"Mikhail Kotelnikov","avatar_url":"https://avatars-next.observableusercontent.com/529fa53c5b4acc1414273473d4e049e502c44c1bdbb1a0af81f7faa463dcff4f","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"titling":"auto","title":"ObservableHQ Notebooks: Building and Compiling","cells":[{"id":0,"value":"md`# ObservableHQ Notebooks: Building and Compiling`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":439,"value":"md`\nThis notebook contains compiler code for ObservableHQ notebooks. \n\nCode processing works in two steps. At first step (\"building\") text code is transformed to a JSON object with cell functons serialized as a plain text. At the next step (\"compilation\") the JSON description is compiled to real executable code. This separation of steps allows to serialize/deserialize JSON cell descriptions and generate plain javascript for notebooks. Which is very useful for notebooks compilation on the server-side in Node. \n\nCode of this notebook is based on the Unofficial ObservableHQ Complier which can be found here: https://github.com/asg017/unofficial-observablehq-compiler. Code was slightly refactored to separate the whole process to building and compilation phases.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":442,"value":"md`## Example Of Usage`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":16,"value":"code = `\nhello=\"World\"\n\nimport { totot } from '#helloworld'\nmd\\`This is the \"\\${totot}\" value!\\`\n\n// qmlskdjflmkqjsdf\n\nmutable N=142\n\nimport { resolveModule } from '@kotelnikov/observablehq-notebooks-building-and-compiling'\n\nhtml\\`

\\${resolveModule}

\\`\n\n\nmd\\`\nLength of the array is \\${len} elements.\nLast element is \"\\${mutable N}\".\n\\`\n\nabc={\n // console.log('?', invalidation); \n const array = [];\n for (let i = 0; i < len; i++) {\n const val = helloWorld(\\`\\${hello}-\\${i}\\`);\n mutable N = val;\n array.push(val);\n }\n return array;\n}\n\nmd\\`This is \"a\": \\${JSON.stringify(a)}\\`\n\na\n\nviewof len=html\\`\\`\n\nfunction helloWorld(msg) {\n return \\`Hello \\${msg}!\\`;\n}\n\nd3=require('d3')\n\nmd\\`This is b2 cell value imported from another notebook: \\${b2}!\\`\n\nviewof myA=wrap(html\\`\\`)\n\nimport { wrap } from '8135ff976018a995'\n\nimport { viewof a, b as b2 } with {\n viewof myA as a\n} from '8135ff976018a995';\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":84,"value":"$45","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":398,"value":"md`## Runtime Dependencies And Code`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":373,"value":"$46","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":78,"value":"rt = require(\"@observablehq/runtime\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":403,"value":"md`## Compilation to code\n\nThis block contains the \"compileNotebook\" and \"compileModule\" methods transforming JSON module description \nto runnable code.\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":550,"value":"async function compileNotebook({ runtime, observer, notebook, resolve }) {\n const index = {};\n let counter = 0;\n notebook.modules.forEach(m => {\n m.id = m.id || `id-${Date.now()}-${counter++}`;\n index[m.id] = runtime.module();\n });\n const res = async ({ source, runtime, observer }) => {\n const m = index[source] = index[source] || await resolve({ source, runtime, observer });\n return m;\n }\n const result = [];\n for (let moduleDefinition of notebook.modules) {\n const compiled = await compileModule({\n runtime,\n module : index[moduleDefinition.id],\n observer,\n moduleDefinition,\n resolve : res\n });\n result.push(compiled);\n } \n return result;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":356,"value":"async function compileModule({\n runtime,\n observer,\n module,\n moduleDefinition,\n resolve\n}) {\n module = module || runtime.module();\n const { definitions } = moduleDefinition;\n for (let i = 0; i < definitions.length; i++) {\n const cell = definitions[i];\n compileCell({ runtime, observer, module, cell, resolve });\n }\n return module;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":648,"value":"async function compileCell({\n runtime, observer, module, cell, resolve\n}) {\n if (cell.type === \"import\") {\n const { source, injections, specifiers } = cell;\n let importedModule = await resolve({ source, runtime, observer });\n if (injections.length) {\n importedModule = importedModule.derive(injections, module);\n }\n specifiers.map((specifier) => {\n module.import(specifier.name, specifier.alias, importedModule);\n });\n } else {\n const { name, references, method } = cell;\n const m =\n typeof method === \"string\"\n ? new Function([], `return (${method})`)()\n : method;\n module.variable(observer(name)).define(name, references, m);\n }\n return module;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":406,"value":"md`## Module Building\n\nThis section contains functions transforming module code to JSON objects describing\nindividual observable cells. Each object can have one of the following types:\n* type=\"cell\" - contains fields:\n * \"name\" - name of the module\n * \"references\" - list of depenendencies on other cells \n * \"method\" - javascript code of the method providing value of this cell\n* type=\"import\" - contains description of a cell with imports\n * \"source\" - URI of the imported module\n * \"injections\" - list of injections\n * \"specifiers\" - list of variables specifiers\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":414,"value":"// Example:\nbuildModule(\n \"main\",\n `\n{\n const myData = FileAttachment(\"MyData.json\");\n const mySecret = Secret(\"current user or me\");\n const db = DatabaseClient(\"hello-db\");\n db.save(myData, mySecret, Secret(\"AllMyKeys\"));\n}\n\nimport { foo as Foo } with { bar as Bar } from '#toto';\n\n\ntoto=myA + 3\nSecret(\"AllMyKeys\")\n\nmyA\n\nviewof myA=html\\`\\`\n\n`\n)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":411,"value":"md`### Builder Methods`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":552,"value":"function buildNotebook(id, code) {\n const module = buildModule(id, code);\n return {\n id,\n modules: [ module ]\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":292,"value":"function buildModule(id, code) {\n const ast = parser.parseModule(code);\n const definitions = ast.cells.reduce((list, cell) => list.concat(buildCell(cell)), []);\n return {\n id,\n ast,\n definitions\n };\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":234,"value":"function buildCell(cell) {\n return (cell.body.type === \"ImportDeclaration\")\n ? getCellImports(cell)\n : getCellDefinitions(cell);\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":340,"value":"function getCellImports(cell) {\n const addTo = (list, d) => {\n if (d.view) {\n list.push({\n name: 'viewof ' + d.imported.name,\n alias: 'viewof ' + d.local.name\n });\n } else if (d.mutable) {\n list.push({\n name: 'mutable ' + d.imported.name,\n alias: 'mutable ' + d.local.name\n });\n }\n list.push({\n name: d.imported.name,\n alias: d.local.name\n });\n return list;\n }\n const specifiers = (cell.body.specifiers || []).reduce(addTo, []);\n const injections = (cell.body.injections || []).reduce(addTo, []);\n const source = cell.body.source.value;\n const imports = [];\n imports.push({\n type : 'import',\n source,\n specifiers,\n injections,\n });\n return imports;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":319,"value":"$47","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":310,"value":"$48","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"md`## Builder Dependencies`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":11,"value":"parser = import(\"@observablehq/parser\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":230,"value":"acornWalk = import(\"acorn-walk\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W49","stdlib":"1"}} 44:"$43:body"