Published
Edited
May 2, 2020
Importers
18 stars
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
scratchTruth = d3.text('https://api.observablehq.com/d/10ffc3f281b8b72f@745.js')
Insert cell
Insert cell
moduleScratchTruth = importModule('https://api.observablehq.com/d/10ffc3f281b8b72f@745.js')
Insert cell
Insert cell
// v1 runtime
{
const hello = html`<div style="height:400px; overflow:scroll;border: thin solid #555;padding:25px">`;
observable.Runtime.load(moduleScratchTruth.default, lib, observable.Inspector.into(hello));
return hello;
}
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
scratchV1 = toV1(nbScratch)
Insert cell
Insert cell
moduleScratchNew = importModule(`data:text/javascript;base64,${utoa(scratchV1)}`)
Insert cell
Insert cell
// v1 runtime
{
const hello = html`<div style="height:400px; overflow:scroll;border: thin solid #555;padding:25px">`;
observable.Runtime.load(moduleScratchNew.default, lib, observable.Inspector.into(hello));
return hello;
}
Insert cell
Insert cell
Insert cell
Insert cell
{
if (navigator.userAgent.match('HeadlessChrome')) return;
const hello = html`<div style="height:1000px; overflow:scroll;border: thin solid #555;padding:25px">`;
const coord = await toV1(nbCoord);
const moduleCoord = await importModule(
`data:text/javascript;base64,${utoa(coord)}`
);
observable.Runtime.load(
moduleCoord.default,
lib,
observable.Inspector.into(hello)
);
return hello;
}
Insert cell
Insert cell
// parser = require('https://iamprettydamn.cool/parser.js') // dist/parser.js hosted on Alex Garcia's site
// parser = require('https://gistcdn.githack.com/bryangingechen/70cbbc03de1c598cb9ce0fef88fc383d/raw/8c27c7f48d1b0a6ae1ce432db0e2f0cc5618467a/parser.js') // re-hosted on raw.githack.com
parser = require("@observablehq/parser@1")
Insert cell
Insert cell
scratchParsed = nbScratch.nodes.map(({value}) => parser.parseCell(value))
Insert cell
Insert cell
// nb is an object like the one returned by the Observable API /document/id endpoint
// it must have id, title, creator.name, version, and nodes fields
// if tag is true:
// - add node_id to the variables
// - add fake cells for comment-only cells
// - add nodes object to the module and export it for convenience
// (keeping this notebook-level data lets us style pinned vs unpinned cells)
async function toV1(nb, tag = false) {
const url = nb.id.match(/\//) ? nb.id : 'd/'+nb.id;
const prefix = `// URL: https://observablehq.com/${url}
// Title: ${nb.title}
// Author: ${nb.creator && nb.creator.name}
// Version: ${nb.version}
// Runtime version: 1
`;
const nbVersionedID = `${nb.id}@${nb.version}`;
const {importStatements, variablesStr} = mainModule(nb.nodes, nbVersionedID, tag);
const vars = `
const m0 = {
id: "${nbVersionedID}",
variables: [
${variablesStr}
]
};
`;
// fill in nbModules with module IDs and variable strings,
// using the V1 modules served by the API
const nbModules = await getCellsFromModule(importStatements, nbVersionedID);
// write the strings here:
const modules = nbModules.map((module,i) => {
return `const m${i+1} = {
id: "${module.moduleID}",
variables: [
${module.str.join('\n')}
]
};`;
}).join('\n');
const end = `${tag ? `
const nodes = ${JSON.stringify(nb.nodes)};
` : ''}
const notebook = {
id: "${nbVersionedID}",
modules: [m0${nbModules.map((d,i) => `,m${i+1}`).join('')}]${tag ? `,
nodes` : ''}
};
export default notebook;
`;
return prefix+vars+modules+end;
}
Insert cell
function mainModule(nodes, nbVersionedID, tag) {
const importStatements = [];
const variablesStr = nodes.map((node,i) => { // loop over cells
const {value} = node;
let parsed;
try {
parsed = parser.parseCell(value); // parse
}
catch (e) {
throw new Error(`${e}, in cell ${i}`);
}
if (parsed.hasOwnProperty('references')) { // if NOT an import statement
return notImportCell(value, parsed, node, tag);
}
else if (parsed.body && parsed.body.type === "ImportDeclaration") { // no "references" field => notebook import (?)
importStatements.push(parsed);
return importCell(nbVersionedID, importStatements, parsed, node, tag);
}
else if (parsed.body === null) {
// comment-only cell?
if (tag) return ` {
node_id: ${node.id}
},`;
}
else {
throw new Error(`unknown type at cell ${i}: ${JSON.stringify(node)}`);
}
}).join('\n');
return {importStatements, variablesStr};
}
Insert cell
LibraryKeys = {
const L = new observable.Library;
const p = [];
for (const l in L)
p.push(l);
return p;
}
Insert cell
function notImportCell(value, parsed, node, tag) {
const {inputs, fInputs, newValue} = getInputs(value, parsed);
// pass "parsed" here since it gets mutated by "getInputs"
// value will be "newValue"
function varStr(parsed, value, cellName) {
return ` {${cellName ? `
name: "${cellName}",` : ''}${inputs.length ? `
inputs: ${JSON.stringify(inputs)},` : ''}
value: (${parsed.async ? 'async ' : ''}function${parsed.generator ? '*' : ''}(${fInputs})${parsed.body.type !== 'BlockStatement' ? `{return(` : ''}
${value.slice(parsed.body.start, parsed.body.end + parsed.indexShift)}
)${parsed.body.type !== 'BlockStatement' ? `;})` : ''}${tag ? `,
node_id: ${node.id}`
: ''}
},`;
}

if (parsed.id === null || parsed.id.type === "Identifier") // normal cell
return varStr(parsed, newValue, parsed.id && parsed.id.name);
else if (parsed.id.type === "ViewExpression") {
// similar to the above, but we have an extra cell
const viewofName = parsed.id.id.name;
return `${varStr(parsed, newValue, `viewof ${viewofName}`)}
{
name: "${viewofName}",
inputs: ["Generators","viewof ${viewofName}"],
value: (G, _) => G.input(_)${tag ? `,
node_id: ${node.id}`
: ''}
},`;
}
else if (parsed.id.type === "MutableExpression") {
// similar to the above, but we have two extra cells
const mutableName = parsed.id.id.name;
return `${varStr(parsed, newValue, `initial ${mutableName}`)}
{
name: "mutable ${mutableName}",
inputs: ["Mutable","initial ${mutableName}"],
value: (M, _) => new M(_)${tag ? `,
node_id: ${node.id}`
: ''}
},
{
name: "${mutableName}",
inputs: ["mutable ${mutableName}"],
value: _ => _.generator${tag ? `,
node_id: ${node.id}`
: ''}
},`;
}
}
Insert cell
// mutates "parsed" (adds field "indexShift")
function getInputs(value, parsed) {
const fInputs = []; // array of function inputs
parsed.indexShift = 0; // character offset in `value`, after "viewof X" and "mutable Y" replacements
let j = 0; // used to name the tokens that replace "viewof X" and "mutable Y" references in the cell
const inputs = parsed.references.map((d)=> {
// check references to see if any point to ViewExpressions or MutableExpressions
if (d.type === "Identifier") {
// ordinary input, function input is named the same
fInputs.push(d.name);
return d.name;
}
else if (d.type === "ViewExpression") {
// "viewof" input, need to perform replacement
const inputName = 'viewof '+d.id.name;
// go through `value` and replace all "viewof XX" with "$j"
walk.simple(parsed.body,{
ViewExpression(node) {
const newName = `\$${j}`;
value = value.slice(0,node.start + parsed.indexShift) +
newName +
value.slice(node.end + parsed.indexShift);
parsed.indexShift += newName.length - (node.end - node.start);
}
}, cellWalk);
fInputs.push(`\$${j++}`);
return inputName;
}
else if (d.type === "MutableExpression") {
// "mutable" input, need to perform replacement
const inputName = 'mutable '+d.id.name;
// go through `value` and replace all "mutable XX" with "$j.value"
walk.simple(parsed.body,{
MutableExpression(node) {
const newName = `\$${j}.value`;
value = value.slice(0,node.start + parsed.indexShift) +
newName +
value.slice(node.end + parsed.indexShift);
parsed.indexShift += newName.length - (node.end - node.start);
}
}, cellWalk);
fInputs.push(`\$${j++}`);
return inputName;
}
else console.log('unknown input type:', d);
});
return {inputs, fInputs, newValue:value};
}
Insert cell
function importCell(nbVersionedID, importStatements, parsed, node, tag) {
const {specifiers, source:{value:importedID}, injections} = parsed.body;
const fromID = injections ? // the ID changes if the injections field is present (even if it's empty []!)
`${nbVersionedID}/${importStatements.length}` :
importedID;
function importStr(localName, importedName) {
return ` {
from: "${fromID}",
name: "${localName}",
remote: "${importedName}"${tag ? `,
node_id: ${node.id}`
: ''}
},`;
}

return specifiers.map((spec) => {
const {imported:{name:importedName}, local:{name:localName}, view} = spec;
return `${importStr(localName, importedName)}${view ? // extra cell if we import viewof
'\n' + importStr(`viewof ${localName}`, `viewof ${importedName}`) :
''}${spec.mutable ? // extra cell if we import mutable
'\n' + importStr(`mutable ${localName}`, `mutable ${importedName}`) : ''}`
}).join('\n');
}
Insert cell
// Let's trust that importing the module served by the API gets us everything.
async function getCellsFromModule(importStatements, parentID) {
const imported = new Map();
// fetch and import all necessary modules
await Promise.all(
importStatements.map(({ body: { source: { value: id } } }) => {
const url = id.match(/\//) ? id : 'd/' + id;
if (!imported.has(id))
return importModule(`https://api.observablehq.com/${url}.js`).then(
module => {
imported.set(id, module);
}
);
})
);

const nbModules = [];
const funcs = new Map();
// first pass, set up initial "top modules" and initial funcs
for (let j = 0; j < importStatements.length; j++) {
const {
specifiers: funcs,
source: { value: id },
injections
} = importStatements[j].body;
const topModuleID = injections ? `${parentID}/${j + 1}` : id;
// we may find an import statement that has the same ID as one we already created
// (this will not happen to imports with injections)
let topModule = nbModules.find(({ moduleID }) => moduleID === topModuleID);
if (topModule !== undefined) {
// if we do, add our functions to that notebook module
const newFuncNames = [
...new Set(funcs.map(({ imported: { name } }) => name))
];
topModule.funcNames = [
...new Set(topModule.funcNames.concat(newFuncNames))
]; // remove duplicates again
for (let k = 0; k < newFuncNames.length; k++)
topModule.queued.add(newFuncNames[k]);
} // otherwise, we create a new notebook module
else {
const funcNames = [
...new Set(funcs.map(({ imported: { name } }) => name))
];
const queued = new Set(funcNames.concat(LibraryKeys));
const injMap = new Map(
injections ? injections.map(inj => [inj.imported.name, inj]) : []
);
topModule = {
funcNames,
injMap,
queued,
str: [],
moduleID: topModuleID,
module: imported.get(id),
moduleIndex: 0
};
nbModules.push(topModule);
}
}
// process modules / variables
let currModule;
while (
(currModule = nbModules.find(({ funcNames }) => funcNames.length > 0))
) {
const { funcNames, injMap, queued, str, module, moduleIndex } = currModule;
// `funcNames` is a queue that begins with the functions that are imported;
// their dependencies will be added recursively
// `queued` marks the functions that have already been added
while (funcNames.length > 0) {
const name = funcNames.shift(); // could use .pop() here too;
// .shift() just gets the order closer to what the Observable API returns?
if (injMap.has(name)) {
// if this definition is injected from the base notebook using import...with
str.push(` {
from: "${parentID}",
name: "${name}",
remote: "${injMap.get(name).local.name}"
},`);
} else if (name === "FileAttachment") {
throw new Error('FileAttachment API not supported!');
}
// some DOM / Web APIs show up in "inputs" but aren't real cells, we should ignore them
// however, 'define' and 'hljs' are things added to the window by Observable, so we should not ignore those!
// (Probably comment those out if running this in a non-notebook environment)
else if (!(window[name] && name !== 'define' && name !== 'hljs')) {
// WARNING: for cells imported from "broken" notebooks where the same name is given to two different cells,
// the following line grabs the first cell with that name, which differs from current Observable behavior.
const modObj = module.default.modules[moduleIndex].variables.find(
({ name: varName }) => varName === name
);
if (modObj.from) {
// there's an import in your import
// write import variable
str.push(` {
from: "${modObj.from}",
name: "${name}",
remote: "${modObj.remote}"
},`);
// where are you 'from'
if (modObj.from !== parentID) {
// if it's the parent, do nothing
// search in other nbModules
let innerModule = nbModules.find(
({ moduleID }) => moduleID === modObj.from
);
if (innerModule !== undefined) {
// the module is already in nbModules, so check "queued" for the remote name...
if (!innerModule.queued.has(modObj.remote)) {
// not in "queued" so it's not a duplicate
innerModule.funcNames = innerModule.funcNames.push(
modObj.remote
);
innerModule.queued.add(modObj.remote);
}
} else {
// otherwise, we create a new notebook module
// dive into the other modules inside this import
let newModuleIndex = module.default.modules.findIndex(
(mod, ind) =>
ind !== moduleIndex &&
mod.variables.some(
({ name: varName }) => varName === modObj.remote
)
);
if (newModuleIndex > 0) {
// if found, push the remote name and module to nbModules
const newFuncNames = [modObj.remote];
const newQueued = new Set(newFuncNames.concat(LibraryKeys));
innerModule = {
funcNames: newFuncNames,
injMap: new Map(),
queued: newQueued,
str: [],
moduleID: modObj.from,
module,
moduleIndex: newModuleIndex
};
nbModules.push(innerModule);
} else console.log(`not found: ${modObj}`);
// if this happens then probably the import is missing something!
// let the runtime throw with some "could not resolve"
}
}
} else {
// non-import / "normal" variable
if (modObj.inputs) {
// queue the dependencies into this module's "funcNames"
modObj.inputs
.filter(str => !queued.has(str))
.forEach(newName => {
funcNames.push(newName);
queued.add(newName);
});
}
// append the variable definition
str.push(` {
name: "${name}",${
modObj.inputs
? `
inputs: ${JSON.stringify(modObj.inputs)},`
: ''
}
value: ${modObj.value.toString()}
},`);
}
}
}
}
return nbModules;
}
Insert cell
Insert cell
Insert cell
// https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/btoa#Unicode_strings
// possibly deprecated...
function utoa(str) {
return window.btoa(unescape(encodeURIComponent(str)));
}
Insert cell
d3 = require_('d3-fetch@1')
Insert cell
lib = {
toV1; // delay creating this until after toV1 is done
// once the "require" in "lib" is called, the built-in require should be considered broken
return new observable.Library;
}
Insert cell
require_ = lib.require()
Insert cell
md_ = lib.md()
Insert cell
Insert cell
import {importModule} from '@bryangingechen/dynamic-import-polyfill'
Insert cell
walk = require('acorn-walk@7')
Insert cell
Insert cell
cellWalk = parser.walk //walk.make(parser.walk)
Insert cell
Insert cell
Insert cell

Purpose-built for displays of data

Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
Learn more