2a:null 2b:["$","title",null,{"children":"Exploring the universal approximation theorem | Nick Strayer | Observable"}] 30:T566,viewof inputData = { wipe_drawing; const height = figHeight; const context = DOM.context2d(width, height); const xScale = d3.scaleLinear().range([-1,1]).domain([0,width]) const yScale = d3.scaleLinear().range([0,1]).domain([figHeight, 0]); let currentStroke = []; // Create a new empty stroke at the start of a drag gesture. function dragsubject() { return currentStroke; } // Add to the stroke when dragging, render, and report the new value. function dragged() { const stroke = d3.event.subject; stroke.push([d3.event.x, d3.event.y]); context.clearRect(0, 0, width, height); for (let i = 1; i < stroke.length; ++i) { context.beginPath(); context.arc(stroke[i][0], stroke[i][1], 1, 0, 2 * Math.PI, true); context.stroke(); } context.canvas.value = stroke; currentStroke = stroke; } function sendDragData(){ const ys = dl.tensor(currentStroke.map(d => yScale(d[1]))); const xs = dl.tensor(currentStroke.map(d => xScale(d[0]))); //const draggedData = dl.tensor(currentStroke); context.canvas.value = {xs, ys}; context.canvas.dispatchEvent(new CustomEvent("input")); } d3.select(context.canvas).call(d3.drag() .container(context.canvas) .subject(dragsubject) .on("start drag", dragged) .on("end", sendDragData)); return context.canvas; }31:T638,{ if(model_predictions.preds.length === 0) return html`

Training your model!

Epoch ${model_predictions.iter} out of ${iterations}

While you wait, may we interest you in the dance of a Seal?

Taking a long time? Drawing less data will speed up the fit process. Since we're literally trying to overfit don't worry about it.

` const xvals = await inputData.xs.data(); const yvals = await inputData.ys.data(); const predvals = model_predictions.preds; const values = Array.from(yvals).map((y, i) => { return { 'x': xvals[i], 'y': yvals[i], 'pred': predvals[i] }; }); const lineStyle = { color: 'orangered' }; const pointStyle = { opacity: 0.5 }; const dataPoints = { mark: "circle", encoding: { x: {"field": "x", "type": "quantitative"}, y: {"field": "y", "type": "quantitative", scale: {domain: [0,1]}} } }; const predictionLine = { mark: "line", encoding: { "x": {field: "x", type: "quantitative"}, "y": {field: "pred", type: "quantitative"} } }; return vegalite({ width: width, height: figHeight, title: `Iteration: ${model_predictions.iter + 1}`, data: { values }, layer: [ dataPoints, predictionLine ], config: { style: { line:lineStyle, circle:pointStyle } } }) }32:T99a,model_predictions = { ++modelTracker.count const numObs = inputData.xs.size, ymin = inputData.ys.min(), ymax = inputData.ys.max(), yrange = ymax.sub(ymin), ynormalized = inputData.ys.sub(ymin).div(yrange); // We start by initializing weights and biases for our two layers: // layer from 1d input to hidden layer const w1_init_var = 2/(1 + hidden_size); const W1 = dl.variable(dl.randomNormal([1,hidden_size], 0, Math.sqrt(w1_init_var))); const b1 = dl.variable(dl.zeros([1,hidden_size])); // layer from hidden layer to 1d output. const w2_init_var = 2/(hidden_size + 1) const W2 = dl.variable(dl.randomNormal([hidden_size, 1], 0, Math.sqrt(w2_init_var))); const b2 = dl.variable(dl.zeros([1,1])); // Next, we define out model. It takes a scalar input (because out input is 1d) // and spits out a scalar output. const netModel = x => dl.tidy(() => { const first_layer = act_func(x.mul(W1).add(b1)) const second_layer = first_layer.matMul(W2).add(b2) return second_layer.asScalar() }) // The objective function: mean squared error between prediction and actual. const loss = (preds, label) => preds.sub(label).square().mean() // set our optimizer based on cell above let optimizer; switch(optimizer_type){ case 'sgd': optimizer = dl.train.momentum(learningRate, momentum); break; case 'adam': optimizer = dl.train.adam(); break; case 'adamax': optimizer = dl.train.adamax(); break; } // Train the model. async function train(xs, ys, iterations){ for(let i=0; i loss( netModel(extractObs(xs, i)), extractObs(ys, i) )); } } const runNumAtStart = modelTracker.count; for (let iter = 0; iter < iterations; iter++) { if(runNumAtStart !== modelTracker.count) break; // checks to see if we have reinstantiated new run await train(inputData.xs, ynormalized, iterations); await dl.nextFrame(); yield {iter, preds: []} console.log(`Epoch ${iter} | Run ${modelTracker.count}`) } const modelOutput = []; for(let i=0; i`\n \n div.style = \"border-radius:5px; width: 100px; height: 30px; color: white; background-color: orangered;\";\n return div\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":775,"value":"md `### Let's see if the model can learn!\n\nDrag the iterations slider up to a decent number (maybe start with 20?) and watch the model try and learn the underlying function you drew. How did it do? Was it easy to fool?\n\nPotential questions to ask: \n- How does changing the activation function change the function the model learns?\n- Does momentum manifest itself in the way you would intuitively think it would?\n- How does adding another layer change things? What about taking one away?\n- Does the entire line get fit at once or does it start from one side?\n\n_As a warning: this will slow down your browser unless you've got a pretty beefy gpu hooked up. I'm pretty sure I've got a memory leak somewhere but can't figure out where exactly. If it crashes just reload your browser and you should be good. Sorry!_`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":323,"value":"$31","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":552,"value":"//iterations = 50\nviewof iterations = slider({\n message:'Number of epochs to run', \n min: 0, max:350, start:5, step:1,\n int:true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1619,"value":"viewof hidden_size = slider({\n message:'Number of nodes in hidden layer', \n min: 1, max:256, start:5, step:1,\n int:true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2096,"value":"viewof activation_type = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2100,"value":"viewof optimizer_type = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2094,"value":"md`Note that the following will only apply if you have set your \\`optimizer_type\\` to 'sgd w/ momentum'`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":941,"value":"viewof learningRate = slider({\n message:'Learning rate for SGD', \n min: 0.0001, max:0.1, start:0.03, step:0.001,\n precision: 4, int:false\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1677,"value":"viewof momentum = slider({\n message:'Momentum amount for SGD', \n min: 0, max:0.99, start:0.4, step:0.01,\n precision: 3, int:false\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1980,"value":"act_func = {\n switch(activation_type){\n case 'relu':\n return dl.relu;\n case 'sigmoid':\n return dl.sigmoid;\n case 'tanh':\n return dl.tanh;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1231,"value":"md `Here is the code to train the model.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1467,"value":"extractObs = (tensor, i) => tensor.slice([i],[1])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2245,"value":"dl.zeros([1,1]).as1D().data()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2150,"value":"$32","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1215,"value":"md `### Acknowledgements\n\nThe fact that this is even remotely possible in javascript is an amazing testimate to the work of the developers working on [deeplearnjs](https://deeplearnjs.org). It's fantastically exciting to get to play around in javascript with something as powerful as deep learning/ automatic gradients.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2072,"value":"function slider({\n message = 'Set your value', \n min = 0, max = 1, start = 0.5, step = 0.1,\n precision = 3, int = false,\n}){\n \n min = min/step;\n max = max/step;\n start = start/step;\n \n const div = html`\n\t${message}\n \n ${start*step}\n `;\n const range = div.querySelector(\"[type=range]\");\n const value_disp = div.querySelector('span');\n div.value = value_disp.innerHTML = start*step;\n\n \n range.addEventListener(\"input\", () => {\n const newValue = range.valueAsNumber*step; \n div.value = newValue;\n value_disp.innerHTML = int ? Math.round(newValue) : newValue.toPrecision(precision); \n });\n return div;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1848,"value":"figHeight = 400","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":119,"value":"dl = require('deeplearn@0.5.0')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":191,"value":"vegalite = require(\"@observablehq/vega-lite@0.1\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1762,"value":"d3 = require('d3')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2224,"value":"modelTracker = ({count: 0})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W33","stdlib":"1"}} 2f:"$2e:body"