2a:null 2b:["$","title",null,{"children":"Hand-Drawn Classification with Neural Networks | Nick Strayer | Observable"}] 30:T6c5,md`# Hand-Drawn Classification with Neural Networks In [my last notebook](https://beta.observablehq.com/@nstrayer/exploring-the-universal-approximation-theorem) I explored the [universal approximation theorem](https://en.wikipedia.org/wiki/Universal_approximation_theorem) of neural networks. This stating that a neural network with an adequately large hidden layer can approximate any function. While the theorem is true the notebook went to show that, while it's possible, in real scenarios it's hard to train a network to actually model the function well in a reasonable number of epochs. Also a 1D to 1D regression model is not exactly the best usecase of a neural network (better to use regression with splines or something easier to train.) ## What we're doing In this notebook I am going to explore something that is a little more up a neural networks alley: classification. Specifically two class classification with a 2D input space. The network is again a very simple one: one hidden layer of variable size, and classifying in a simple 0-1 output using a sigmoid activation function on the final layer. ## Okay, let's actually train a model Below you can draw some data for the model to classify. Switch the class you are drawing with the dropdown and then tune the neural network hyperparameters. When you are ready to train the model hit the train now button and watch it go! Some questions you could ask when playing: - Can you fool the network? - How does the size of the hidden layer effect the shape of its decision boundaries? - Is there a difference in training speed between a relu and sigmoid hidden activation? What may cause this? - See if you can add another layer in the code. ## Draw your data! `31:T697,viewof dataDrawer = { wipe_drawing; const height = figHeight; const context = DOM.context2d(width, height); const xScale = d3.scaleLinear().range([0,1]).domain([0,width]) const yScale = d3.scaleLinear().range([0,1]).domain([figHeight, 0]); let currentStroke = []; mutable drawnData = null; // Create a new empty stroke at the start of a drag gesture. function dragsubject() { return currentStroke; } context.fillStyle="#e0e0e0"; context.fillRect(0,0,width, height); // Add to the stroke when dragging, render, and report the new value. function dragged() { const stroke = d3.event.subject; stroke.push({x:xScale(d3.event.x), y: yScale(d3.event.y), class: (mutable currentClass)}); context.clearRect(0, 0, width, height); context.fillStyle="#e0e0e0"; context.fillRect(0,0,width, height); for (let i = 1; i < stroke.length; ++i) { context.strokeStyle = stroke[i].class !== 'a' ? groupOneColor: groupTwoColor; context.beginPath(); context.arc(xScale.invert(stroke[i].x), yScale.invert(stroke[i].y), 2, 0, 2 * Math.PI, true); context.stroke(); } currentStroke = stroke; } function sendDragData(){ const drawnPoints = dl.tidy(() => { const groups = dl.tensor(currentStroke.map(d => d.class == 'a' ? 1: 0)), inputs = dl.tensor(currentStroke.map(d => [d.x, d.y])); return {inputs, groups}; }) mutable drawnData = drawnPoints; drawnPoints.dispose } d3.select(context.canvas).call(d3.drag() .container(context.canvas) .subject(dragsubject) .on("start drag", dragged) .on("end", sendDragData)); return context.canvas; }32:T5d8,{ const nhidden = 5, nepochs = 25; const div = html`

Model Parameters

Size of hidden layer: ${nhidden}

Hidden layer activation function:

Number of epochs: ${nepochs}

`; const numNodes = div.querySelector("input.hiddenSelector"); const hidden_disp = div.querySelector('span.hiddenSize'); mutable hidden_nodes = nhidden; const numEpochs = div.querySelector("input.epochSelector"); const epoch_disp = div.querySelector('span.epochNum'); mutable number_epochs = nepochs const actSelect = div.querySelector("select") mutable act_type = actSelect.value numEpochs.addEventListener("input", () => { const newValue = numEpochs.valueAsNumber; mutable number_epochs = newValue epoch_disp.innerHTML = newValue }) actSelect.addEventListener("input", () => { mutable act_type = actSelect.value }) numNodes.addEventListener("input", () => { const newValue = numNodes.valueAsNumber; mutable hidden_nodes = newValue hidden_disp.innerHTML = newValue; }); return div; }33:Tb41,chart = { if (!inputData) return html`Press the 'Train Model!' button above to watch your model learn the data you've drawn.` const w = width/2; const h = width/2; // get the drawn data to plot const numObs = inputData.inputs.shape[0] const xvals = await inputData.inputs.slice([0,0],[numObs, 1]).data(); const yvals = await inputData.inputs.slice([0,1], [numObs,1]).data(); const classes = await inputData.groups.data(); //scales const colorScale = d3.scaleLinear() .domain(thresholds) .range(['#ca0020','#f4a582','#f7f7f7','#92c5de','#0571b0']), x = d3.scaleLinear().domain([0, 1]).range([0, w]), y = d3.scaleLinear().domain([0, 1]).range([h, 0]); const svg = d3.select(DOM.svg(w, h)) .style("display", "block") .style("margin", "0") .style("width", "calc(100%)"); // legend const legendW = 80, legendH = w/2; const legend = svg.append('g') .attr('transform', `translate(${w + 20},${legendH})`); legend.append('text') .text('Model Probs') .attr('text-anchor', 'middle') .attr('y', -5) .attr('x', legendW/2) legend.selectAll('.key') .data(thresholds).enter() .append('rect') .attr('class', d => `key_${Math.round(d*10)}`) .attr('y', (d,i) => i*(legendH/thresholds.length)) .attr('height', legendH/thresholds.length) .attr('width', legendW) .attr('fill', (d,i) => colorScale(d)) .attr('fill-opacity', 0.75) legend.selectAll('.key') .data(thresholds).enter() .append('text') .attr('text-anchor', 'middle') .attr('x', legendW/2) .attr('y', (d,i) => (i+1)*(legendH/thresholds.length) - 15) .text((d,i) => `[${d.toPrecision(1)}, ${(thresholds[i + 1] || 1).toPrecision(1)})`) svg.append("g") .attr("fill", "none") .attr("stroke", "#fff") .attr("stroke-opacity", 0.5) .selectAll("path") .data(contours) .enter().append("path") .attr("d", d3.geoPath( d3.geoIdentity().scale(w/gridRes) .reflectY(true).translate([0,w]) )) .attr('fill', d => colorScale(d.value)) .attr('fill-opacity', 0.75) .on('mouseover',highlightLegend) .on('mouseout', resetLegend); function highlightLegend(d){ svg.select(`.key_${Math.round(d.value*10)}`) .transition() .duration(200) .attr('width', legendW*1.2) } function resetLegend(d){ svg.select(`.key_${Math.round(d.value*10)}`) .transition() .duration(100) .attr('width', legendW) } svg.selectAll('.userPoints') .data(classes).enter() .append('circle') .attr('cx', (d,i) => x(xvals[i])) .attr('cy', (d,i) => y(yvals[i])) .attr('r', 5) .attr('stroke', 'white') .attr('stroke-opacity', 0.5) .attr('fill', d => d ? groupTwoColor: groupOneColor) .attr('fill-opacity', 0.5); return svg.node(); }34:Ta03,model_predictions = { ++modelTracker.count if (!inputData) return {preds:[], losses: []} const numObs = inputData.inputs.shape[0]; const w1_init_var = 2/(2 + hiddenNodes); const W1 = dl.variable(dl.randomNormal([2,hiddenNodes], 0, Math.sqrt(w1_init_var))); const b1 = dl.variable(dl.zeros([1,hiddenNodes])); const w2_init_var = 2/(hiddenNodes + 2); const W2 = dl.variable(dl.randomNormal([hiddenNodes,1], 0, Math.sqrt(w2_init_var))); const b2 = dl.variable(dl.zeros([1,1])); let hidden_act_func; switch(activation_type){ case 'relu': hidden_act_func = dl.relu; break; case 'sigmoid': hidden_act_func = dl.sigmoid; break; case 'tanh': hidden_act_func = dl.tanh; break; } const classifier = xs => dl.tidy(() => { const first_layer = hidden_act_func(xs.matMul(W1).add(b1)) const second_layer = first_layer.matMul(W2).add(b2).sigmoid() return second_layer.asScalar() }) const one = dl.scalar(1); const negativeOne = dl.scalar(-1); const loss = (prediction, truth) => dl.tidy(() => truth.mul(dl.log(prediction)) .add( one.sub(truth).mul(dl.log(one.sub(prediction))) ).mul(negativeOne)) const optimizer = dl.train.adam(); // Train the model. async function train(inputs, classes){ return dl.tidy(()=>{ let loss_sum = dl.scalar(0); for(let i=0; i loss( classifier(twoDSlice(inputs, i)), oneDSlice(classes, i) ).asScalar(), true) ) } return loss_sum.data() }) } const losses = [], runNumAtStart = modelTracker.count, modelOutput = new Array(gridRes*gridRes); let currentPred, currentLoss, xInp, yInp; for (let iter = 0; iter < numberEpochs; iter++) { if(runNumAtStart !== modelTracker.count) break; // checks to see if we have reinstantiated new run currentLoss = await train(inputData.inputs, inputData.groups); losses.push( currentLoss[0]); await dl.nextFrame(); for(let i=0; i classifier(dl.tensor([xInp, yInp], [1,2])).data()) modelOutput[j*gridRes + i] = currentPred[0] } } yield {preds:modelOutput, losses} } // cleaning up old tensors W1.dispose(); b1.dispose(); W2.dispose(); b2.dispose(); one.dispose(); negativeOne.dispose(); }35:[] 2e:{"id":"34c18e0f57d2c307","slug":"hand-drawn-classification-with-neural-networks","title":"Hand-Drawn Classification with Neural Networks","thumbnail":"c6edc7818aa43dd94ee2e2c944b62505667aae8abf2508f1b9442570ac51ba24","access_level":"public","create_time":"$D2018-02-24T18:03:51.989Z","update_time":"$D2018-02-27T15:22:41.803Z","view_time":null,"first_public_time":"$D2018-02-27T15:31:53.081Z","has_unpublished_edits":null,"fork":false,"stars":1,"starred":false,"owner":{"type":"team","id":"c7ceaaf194433370","login":"nstrayer","name":"Nick Strayer","avatar_url":"https://avatars-next.observableusercontent.com/56ce9e43c9ae8d967575f2641fd299251ce5e022874e0c8fe8d04becea060719","tier":"starter_2024"},"version":992,"publish_version":992,"role":null,"authored":false,"authors":[{"type":"individual","id":"e253fe2c017ea3f1","login":"nstrayer","name":"Nick Strayer","avatar_url":"https://avatars-next.observableusercontent.com/56ce9e43c9ae8d967575f2641fd299251ce5e022874e0c8fe8d04becea060719","tier":null}],"members":[{"type":"individual","id":"e253fe2c017ea3f1","login":"nstrayer","name":"Nick Strayer","avatar_url":"https://avatars-next.observableusercontent.com/56ce9e43c9ae8d967575f2641fd299251ce5e022874e0c8fe8d04becea060719","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Hand-Drawn Classification with Neural Networks","cells":[{"id":0,"value":"$30","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"$31","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":890,"value":"pickedClass = {\n const div = html`\n\tCurrently drawing class: \n \n `;\n const dropdown = div.querySelector(\"select\");\n \n div.value = mutable currentClass;\n\n \n dropdown.addEventListener(\"input\", () => {\n const newValue = dropdown.value;\n mutable currentClass = dropdown.value\n });\n\n return div\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":701,"value":"viewof wipe_drawing = {\n const resetButton = html``;\n resetButton.style = \"border-radius:5px; width: 100px; height: 30px; color: white; background-color: orangered;\";\n return resetButton\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":809,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":766,"value":"viewof sendToModel = {\n const div = html``\n div.style = \"border-radius:5px; width: 150px; height: 30px; color: white; background-color: orangered;\";\n return div\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":510,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":423,"value":"{\n if (!inputData) return html`

`\n\n return vegalite({\n data: {values : model_predictions.losses.map((d,i) => ({x:i, y:d}))},\n width: width,\n height: figHeight/1.5,\n mark: \"line\",\n title: \"Training Loss\",\n encoding: {\n x: {\n field: \"x\", \n type: \"ordinal\",\n axis: {title: \"Epoch\", labelAngle:0}\n },\n y: {\n field: \"y\", \n type: \"quantitative\",\n axis: {title: \"Total Epoch Loss\"},\n scale: {domain: d3.extent(model_predictions.losses) }}\n }\n})\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":989,"value":"md`\n## How we're doing it\nThe model is made with the new api for [deeplearnjs](https://deeplearnjs.org/) which makes rapidly prototyping models super easy due to its eager execution setup. You simply chain together your model like you would expect and it will automatically calculate gradients for you. It's wonderful and I hope this notebook will help you get a better feel of using it.\n\nThe contours for the model predictions are calculated with the new [d3-contour](https://github.com/d3/d3-contour) module in the latest d3 release. \n\nTake a look at the code below to see how the model was put together. The beauty of these observable notebooks is you can tinker with it right here. Add another layer, add batchnorm, anything you can think of.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":50,"value":"$34","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":473,"value":"gridRes = 25","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":418,"value":"inputData = {\n sendToModel;\n return mutable drawnData\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":357,"value":"mutable drawnData = {shape: [1,2]}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":337,"value":"mutable currentClass = 'a'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":798,"value":"mutable hidden_nodes = 5","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":803,"value":"hiddenNodes = {\n sendToModel;\n return mutable hidden_nodes\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":800,"value":"mutable number_epochs = 10","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":806,"value":"numberEpochs = {\n sendToModel;\n return mutable number_epochs\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":734,"value":"mutable act_type = 'relu'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":792,"value":"activation_type = {\n sendToModel;\n return mutable act_type\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":685,"value":"grid_ranges = ({xmin:0, xmax:1, xwidth: 1, ymin:0, ymax:1, ywidth: 1})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":688,"value":"xside = d3.range(grid_ranges.xmin,grid_ranges.xmax, grid_ranges.xwidth/gridRes)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":693,"value":"yside = d3.range(grid_ranges.ymin,grid_ranges.ymax, (grid_ranges.ywidth)/gridRes)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":563,"value":"thresholds = d3.range(0,1,.2)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":567,"value":"contours = inputData\n ? d3.contours()\n .size([gridRes, gridRes])\n .thresholds(thresholds)\n (model_predictions.preds)\n : []","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":290,"value":"extractObs = (tensor, i) => tensor.slice([i],[1])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":331,"value":"twoDSlice = (tensor, i) => tensor.slice([i,0],[1,2])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":334,"value":"oneDSlice = (tensor, i) => tensor.slice([i],[1])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"figHeight = 400","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":959,"value":"groupOneColor = '#ca0020'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":961,"value":"groupTwoColor = '#0571b0'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":36,"value":"dl = require('deeplearn')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":9,"value":"d3 = require(\"https://d3js.org/d3.v5.min.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":240,"value":"vegalite = require(\"@observablehq/vega-lite@0.1\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":683,"value":"modelTracker = ({count: 0})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W35","stdlib":"1"}} 2f:"$2e:body"