2c:null 2d:["$","title",null,{"children":"Interactive Convolutional Neural Network (CNN) | Mario Delgado | Observable"}] 32:T441,### Biological Neural Networks The neural network's primary component is the [neuron](http://en.wikipedia.org/wiki/Neuron). It operates under a [stimulus threshold dynamic](http://www.pnas.org/content/105/9/3599.full); '[firing](http://psychology.about.com/od/aindex/g/actionpot.htm)' in response to stimuli that meet its firing threshold; a [non-linear dynamic](https://en.wikipedia.org/wiki/Nonlinear_system). The notebook [***Leaky integrate-and-fire (LIF) neuron with balanced excitation and inhibition***](https://observablehq.com/@patrickmineault/balanced-e-i-networks) models this dynamic allowing you to visualize neuronal thresholds of [excitatory](https://en.wikipedia.org/wiki/Excitatory_postsynaptic_potential) and [inhibitory](https://en.wikipedia.org/wiki/Inhibitory_postsynaptic_potential) inputs: 33:T264c,function createVis() { const dpi = DPI const canvas = document.createElement('canvas') const context = canvas.getContext('2d') const canvasWidth = Math.max(width, 830) const height = canvasWidth * 0.5 canvas.width = canvasWidth * dpi canvas.height = height * dpi canvas.style.width = canvasWidth + 'px' context.scale(dpi, dpi) const div = document.createElement('div') div.style.width = '100%' div.style.overflow = 'auto' div.appendChild(canvas) const blockDpi = 2 const link = d3.linkHorizontal() .context(context) .source(d => d[0]) .target(d => d[1]) let layersWithCords = [] function drawTopology(topology) { context.clearRect(0, 0, canvasWidth, height) const sizeScale = d3.scaleSqrt() .domain([1, d3.max(topology, d => d.outSize)]) .range([4, 40]) let xCursor = 0 const opacityScale = d3.scaleLinear() .domain([-1, 1]) .range([0.001, 0.04]) const piyg = d3.scaleSequential(d3.interpolatePiYG) .domain([-0.15, 0.15]) layersWithCords = topology.map((layer, i) => { const nodeIndexes = d3.range(layer.nodes) xCursor += 20 const yScale = d3.scalePoint() .domain(nodeIndexes) .rangeRound([0, height]) .padding(1) const size = sizeScale(layer.outSize) let weights if (layer.weights) { if (layer.weights[0].length > 1) { xCursor += 30 } const baseWX = xCursor weights = layer.weights.map((nodeWeights, i) => { const y = yScale(i) const blockSize = Math.sqrt(nodeWeights[0].length) const actualSize = blockSize * blockDpi let wx = baseWX let wy = y - actualSize / 2 const pad = 3 return nodeWeights.map(weights => { const block = { baseWX, size: blockSize, data: weights, cx: wx + actualSize / 2, cy: wy + actualSize / 2, x: wx, y: wy } wx += blockSize * blockDpi + pad return block }) }) xCursor = d3.max(weights.reduce( (all, blocks) => all.concat(blocks.map(block => block.x + block.size * blockDpi)), [] )) + 20 } xCursor += size / 2 const x = xCursor const nodes = nodeIndexes.map((node, i) => { const y = yScale(node) return { x, y } }) xCursor += size + 20 return { data: layer, x, size, nodes, weights } }) layersWithCords.forEach((layer, i, all) => { const prev = all[i - 1] if (layer.data.bias) { context.globalAlpha = 1 layer.nodes.forEach(({ y }, i) => { context.fillStyle = piyg(layer.data.bias[i]) context.fillRect(layer.x + layer.size / 2, y - layer.size / 2, 4, layer.size) }) } if (prev && !HIDDEN_CONNECTIONS.includes(layer.data.name)) { const sourceX = prev.x if (layer.weights) { context.strokeStyle = '#000' layer.weights.forEach((nodeWeights, i) => { const node = layer.nodes[i] nodeWeights.forEach((block, i) => { const prevNode = prev.nodes[i] const targetX = block.x const targetY = block.y const blockCenter = [block.cx, block.cy] context.globalAlpha = nodeWeights.length > 1 ? 0.07 : 0.2 context.beginPath() link([ [sourceX, prevNode.y], blockCenter ]) context.stroke() context.globalAlpha = nodeWeights.length > 1 ? 0.1 : 0.2 context.beginPath() link([ blockCenter, [layer.x - layer.size / 2, node.y] ]) context.stroke() }) }) } else { context.globalAlpha = 0.2 if (layer.nodes.length === prev.nodes.length) { layer.nodes.forEach((node, i) => { const prevNode = prev.nodes[i] context.beginPath() link([ [sourceX + prev.size / 2 + (prev.data.bias ? 4 : 0), prevNode.y], [layer.x, node.y] ]) context.stroke() }) } else { prev.nodes.forEach((prevNode) => { layer.nodes.forEach((node) => { context.beginPath() link([ [sourceX + prev.size / 2 + (prev.data.bias ? 4 : 0), prevNode.y], // layer.size / 4 looks best [layer.x - layer.size / 4, node.y] ]) context.stroke() }) }) } } } }) context.globalAlpha = 1 layersWithCords.forEach((layer, i, all) => { if (layer.weights) { const isConv2D = layer.data.name.match(/^conv2d/) layer.weights.forEach((nodeWeights, i) => { nodeWeights.forEach((block, i) => { context.translate(block.x, block.y) if (isConv2D) { // context.save() // context.rotate(10 * Math.PI / 180) } block.data.forEach((d, i) => { const color = piyg(d) const row = Math.floor(i / block.size) context.fillStyle = color context.fillRect((i % block.size) * blockDpi, row * blockDpi, blockDpi, blockDpi) }) if (isConv2D) { // context.restore() } context.translate(-block.x, -block.y) }) }) } }) context.globalAlpha = 1 } async function drawResults(results) { for (let i = 0; i < layersWithCords.length; i++) { const layer = layersWithCords[i] const { x, size } = layer const xEnd = ( x + size / 2 + (layer.data.weights ? Math.sqrt(layer.data.weights[0].length) * blockDpi : 0) + 1 + // border 1 // padding ) if (i === layersWithCords.length - 1) { context.clearRect(xEnd, 0, 100, height) } const result = results && results[i] if (result) { const dataPerNode = tf.tidy(() => { // console.log('---') // console.log(layer.data.name) const activation = results[i].squeeze([0]) // activation.print(true) const perNode = tf.split(activation, layer.data.nodes, layer.data.outShape.length - 1) .map(t => t.squeeze().clipByValue(0, 1)) .map(t => { if (t.rank === 0) { return t.expandDims().expandDims() } if (t.rank === 1) { const s = Math.sqrt(t.shape[0]) return t.reshape([s, s]) } return t }) .map(t => t.mul(tf.scalar(255)).cast('int32')) // perNode.map(n => n.print(true)) return perNode }) for (let i = 0; i < dataPerNode.length; i++) { if (dataPerNode[i].rank === 2) { const canvas = document.createElement('canvas') await tf.toPixels(dataPerNode[i], canvas) const y = layer.nodes[i].y context.drawImage(canvas, x - size / 2, y - size / 2, size, size) } } if (i === layersWithCords.length - 1) { // console.log('res', result.dataSync()) if (DRAW_RESULT) { const firstLayer = layersWithCords[0] const y = height / 2 const digit = result.argMax(1).dataSync() const rNode = layer.nodes[digit] context.globalAlpha = 0.2 context.beginPath() link([ [xEnd, rNode.y], [x + 62, height / 2] ]) context.stroke() context.globalAlpha = 1 context.beginPath() context.rect( x + 62, y - firstLayer.size / 2, firstLayer.size, firstLayer.size ) context.fillStyle = '#fff' context.fill() context.strokeStyle = '#000' context.stroke() context.font = `${firstLayer.size}px sans-serif` context.textAlign = 'center' context.textBaseline = 'middle' context.fillStyle = '#000' context.fillText(digit, x + 62 + firstLayer.size / 2, height / 2) } } } else { layer.nodes.forEach(({ y }) => { context.clearRect(x - size / 2, y - size / 2, size, size) }) } layer.nodes.forEach(({ y }) => { context.strokeStyle = '#000' context.strokeRect(x - size / 2, y - size / 2, size + (layer.data.bias ? 4 : 0), size) }) } } // let raf const drawResultsOnNAF = (results) => { // raf && window.cancelAnimationFrame(raf) window.requestAnimationFrame(() => { drawResults(results) }) } const drawResultsWhenEv = debounce(drawResultsOnNAF, 50) // throttle or debounce? leading: false, maxWait: 1000 let lastTopology function draw(topology, results) { if (lastTopology !== topology) { drawTopology(topology) drawResults(results) lastTopology = topology } else { drawResultsWhenEv(results) } return div } return { canvas, node: div, // drawTopology, // drawResults: throttle(drawResults, 100), draw } }34:T904,function readTopology({ model }) { const input = model.inputs[0] let inputNodes = 1 let size = IMG_SIZE * IMG_SIZE const layers = [{ nodes: inputNodes, outSize: size, outShape: input.shape.slice(1), name: 'input', }] for (let tfLayer of model.layers) { const name = tfLayer.name const nodes = tfLayer.filters || tfLayer.units || (name.match(/flatten/) ? 1 : inputNodes) const layer = { nodes, inSize: size } // don't know why but the output of conv2d layers seem to always loose 4px size if (name.match(/^conv2d/)) { size = Math.pow(Math.sqrt(size) - 4, 2) } // max sample half (not supporting other configurations) if (name.match(/^max_pooling2d/)) { size = Math.pow(Math.sqrt(size) / 2, 2) } if (name.match(/^flatten/)) { size = inputNodes * size } if (name.match(/^dense/)) { size = 1 } layer.outSize = size layer.outShape = tfLayer.output.shape.slice(1) layer.name = name if (tfLayer.useBias) { layer.bias = tfLayer.bias.val.dataSync() } if (tfLayer.kernel) { const kernelVal = tfLayer.kernel.val // expect weights per node if (nodes > 1 && kernelVal.shape[kernelVal.shape.length - 1] !== nodes) { console.warn('unexpected kernel val shape, ignoring weights') return } const weightsPerNode = nodes > 1 ? tf.split(kernelVal, nodes, kernelVal.shape.length - 1).map(val => val.squeeze()) : [tfLayer.kernel.val] layer.weights = weightsPerNode .map(perNode => { if (inputNodes > 1) { // expect weights per input node if (inputNodes > 1 && perNode.shape[perNode.shape.length - 1] !== inputNodes) { console.warn('unexpected kernel val shape, ignoring weights') return } return tf.split(perNode, inputNodes, perNode.shape.length - 1).map(val => val.squeeze()) } return [perNode] }) .map(vals => vals.map(val => val.dataSync())) // layer.weightExtent = layer.weights.map(n => n.map(i => d3.extent(i))) // layer.weightSums = layer.weights.map(n => n.map(i => d3.sum(i))) } layers.push(layer) inputNodes = nodes } return layers }35:Tedb,class MnistData { constructor() { this.shuffledTrainIndex = 0; this.shuffledTestIndex = 0; } async load() { // Make a request for the MNIST sprited image. const img = new Image(); const canvas = document.createElement('canvas'); const ctx = canvas.getContext('2d'); const imgRequest = new Promise((resolve, reject) => { img.crossOrigin = ''; img.onload = () => { img.width = img.naturalWidth; img.height = img.naturalHeight; const datasetBytesBuffer = new ArrayBuffer(NUM_DATASET_ELEMENTS * IMAGE_SIZE * 4); const chunkSize = 5000; canvas.width = img.width; canvas.height = chunkSize; for (let i = 0; i < NUM_DATASET_ELEMENTS / chunkSize; i++) { const datasetBytesView = new Float32Array( datasetBytesBuffer, i * IMAGE_SIZE * chunkSize * 4, IMAGE_SIZE * chunkSize); ctx.drawImage( img, 0, i * chunkSize, img.width, chunkSize, 0, 0, img.width, chunkSize); const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height); for (let j = 0; j < imageData.data.length / 4; j++) { // All channels hold an equal value since the image is grayscale, so // just read the red channel. datasetBytesView[j] = imageData.data[j * 4] / 255; } } this.datasetImages = new Float32Array(datasetBytesBuffer); resolve(); }; img.src = MNIST_IMAGES_SPRITE_PATH; }); const labelsRequest = fetch(MNIST_LABELS_PATH); const [imgResponse, labelsResponse] = await Promise.all([imgRequest, labelsRequest]); this.datasetLabels = new Uint8Array(await labelsResponse.arrayBuffer()); // Create shuffled indices into the train/test set for when we select a // random dataset element for training / validation. this.trainIndices = tf.util.createShuffledIndices(NUM_TRAIN_ELEMENTS); this.testIndices = tf.util.createShuffledIndices(NUM_TEST_ELEMENTS); // Slice the the images and labels into train and test sets. this.trainImages = this.datasetImages.slice(0, IMAGE_SIZE * NUM_TRAIN_ELEMENTS); this.testImages = this.datasetImages.slice(IMAGE_SIZE * NUM_TRAIN_ELEMENTS); this.trainLabels = this.datasetLabels.slice(0, NUM_CLASSES * NUM_TRAIN_ELEMENTS); this.testLabels = this.datasetLabels.slice(NUM_CLASSES * NUM_TRAIN_ELEMENTS); } nextTrainBatch(batchSize) { return this.nextBatch( batchSize, [this.trainImages, this.trainLabels], () => { this.shuffledTrainIndex = (this.shuffledTrainIndex + 1) % this.trainIndices.length; return this.trainIndices[this.shuffledTrainIndex]; }); } nextTestBatch(batchSize) { return this.nextBatch(batchSize, [this.testImages, this.testLabels], () => { this.shuffledTestIndex = (this.shuffledTestIndex + 1) % this.testIndices.length; return this.testIndices[this.shuffledTestIndex]; }); } nextBatch(batchSize, data, index) { const batchImagesArray = new Float32Array(batchSize * IMAGE_SIZE); const batchLabelsArray = new Uint8Array(batchSize * NUM_CLASSES); for (let i = 0; i < batchSize; i++) { const idx = index(); const image = data[0].slice(idx * IMAGE_SIZE, idx * IMAGE_SIZE + IMAGE_SIZE); batchImagesArray.set(image, i * IMAGE_SIZE); const label = data[1].slice(idx * NUM_CLASSES, idx * NUM_CLASSES + NUM_CLASSES); batchLabelsArray.set(label, i * NUM_CLASSES); } const xs = tf.tensor2d(batchImagesArray, [batchSize, IMAGE_SIZE]); const labels = tf.tensor2d(batchLabelsArray, [batchSize, NUM_CLASSES]); return {xs, labels}; } }36:T5d6,async function train(model, data) { const lossValues = [] const accuracyValues = [] for (let i = 0; i < TRAIN_BATCHES; i++) { const batch = data.nextTrainBatch(BATCH_SIZE) let testBatch let validationData // Every few batches test the accuracy of the mode. if (i % TEST_ITERATION_FREQUENCY === 0) { testBatch = data.nextTestBatch(TEST_BATCH_SIZE) validationData = [ testBatch.xs.reshape([TEST_BATCH_SIZE, 28, 28, 1]), testBatch.labels ] } // The entire dataset doesn't fit into memory so we call fit repeatedly // with batches. const history = await model.fit( batch.xs.reshape([BATCH_SIZE, 28, 28, 1]), batch.labels, { batchSize: BATCH_SIZE, validationData, epochs: 1, callbacks: { onBatchEnd() { mutable topology = readTopology({model}) } } } ) const loss = history.history.loss[0] const accuracy = history.history.acc[0] // Plot loss / accuracy. lossValues.push({'batch': i, 'loss': loss, 'set': 'train'}) mutable loss = loss if (testBatch != null) { accuracyValues.push({'batch': i, 'accuracy': accuracy, 'set': 'train'}) mutable accuracy = accuracy } mutable canvasWithInput = canvasWithInput batch.xs.dispose() batch.labels.dispose() if (testBatch != null) { testBatch.xs.dispose() testBatch.labels.dispose() } await tf.nextFrame() } }37:T421,function getBounds(canvas) { const { width: w, height: h } = canvas const context = canvas.getContext('2d') const imageData = context.getImageData(0, 0, w, h) const buffer = imageData.data let x1 = w let y1 = h let x2 = 0 let y2 = 0 // get left edge for (let y = 0; y < h; y++) { for (let x = 0; x < w; x++) { if (buffer[(x + y * w) * 4 + 3] > 0) { if (x < x1) x1 = x } } } // get right edge for (let y = 0; y < h; y++) { for (let x = w; x >= 0; x--) { if (buffer[(x + y * w) * 4 + 3] > 0) { if (x > x2) x2 = x } } } // get top edge for (let x = 0; x < w; x++) { for (let y = 0; y < h; y++) { if (buffer[(x + y * w) * 4 + 3] > 0) { if (y < y1) y1 = y } } } // get bottom edge for (let x = 0; x < w; x++) { for (let y = h; y >= 0; y--) { if (buffer[(x + y * w) * 4 + 3] > 0) { if (y > y2) y2 = y } } } const width = x2 - x1 const height = y2 - y1 return {x1, y1, x2, y2, width, height} }38:T775,function createResult({model}) { const canvas = document.createElement('canvas') const context = canvas.getContext('2d') const size = 28 canvas.width = IMG_SIZE canvas.height = IMG_SIZE // canvas.style.border = '1px solid black' // const resultText = document.createTextNode(' ') // const div = document.createElement('div') // div.appendChild(canvas) // div.appendChild(document.createTextNode(' ')) // div.appendChild(resultText) // const numberFormat = d3.format('.1f') const models = model.layers.map(layer => tf.model({name: `inspect-${layer.name}`, inputs: model.inputs, outputs: layer.output})) return { read(inputCanvas) { context.clearRect(0, 0, size, size) context.beginPath() context.rect(0, 0, size, size) context.fillStyle = '#fff' context.fill() // center draw let {x1: sx, y1: sy, width: sw, height: sh} = getBounds(inputCanvas) if (sw > sh) { sy -= (sw - sh) / 2 sh = sw } if (sh > sw) { sx -= (sh - sw) / 2 sw = sh } context.drawImage( inputCanvas, sx, sy, sw, sh, 4, 4, size - 8, size - 8 ) const imgData = context.getImageData(0, 0, size, size) const data = imgData.data // invert for(let i = 0; i < imgData.data.length; i += 4) { imgData.data[i] = 255 - imgData.data[i] imgData.data[i + 1] = 255 - imgData.data[i + 1] imgData.data[i + 2] = 255 - imgData.data[i + 2] } return tf.tidy(() => { const image = tf.fromPixels(imgData, 1) const reshaped = image.reshape([1, 28, 28, 1]) const casted = tf.cast(reshaped, 'float32') // const output = model.predict(casted) return [casted].concat(models.map(m => { return m.predict(casted) })) }) } } }39:T5ba,function createInput({ onDraw, onClear }) { const dpi = window.devicePixelRatio const canvas = document.createElement('canvas') const context = canvas.getContext('2d') const size = 200 canvas.width = size * dpi canvas.height = size * dpi canvas.style.width = size + 'px' canvas.style.border = '1px solid black' context.scale(dpi, dpi) let paths = [] const line = d3.line() .curve(d3.curveBasis) .context(context) function draw() { context.clearRect(0, 0, size, size) context.beginPath() context.lineCap = 'round' paths.forEach(path => { line(path) }) context.lineWidth = 10 context.stroke() } function dragstarted() { const d = d3.event.subject let x0 = d3.event.x let y0 = d3.event.y paths.push(d) d3.event.on('drag', function() { var x1 = d3.event.x, y1 = d3.event.y, dx = x1 - x0, dy = y1 - y0; if (dx * dx + dy * dy > 100) d.push([x0 = x1, y0 = y1]) else d[d.length - 1] = [x1, y1] draw() onDraw && onDraw(canvas) }) } d3.select(canvas).call(d3.drag() .container(function() { return this }) .subject(function() { var p = [d3.event.x, d3.event.y]; return [p, p]; }) .on("start", dragstarted) ) return { canvas, clear() { paths = [] draw() onClear() }, read() { return context.getImageData(0, 0, size * dpi, size * dpi) } } }3a:[{"id":"ebc8c7e45960b6e53cba387d7d8c52546fde473cd9277a303c180efbbb48f809d82d485cdedf561e116cb28dc88ffbd6f3433afe22eeeee33a8d9a37e9ecae4d","href":"https://static.observableusercontent.com/files/ebc8c7e45960b6e53cba387d7d8c52546fde473cd9277a303c180efbbb48f809d82d485cdedf561e116cb28dc88ffbd6f3433afe22eeeee33a8d9a37e9ecae4d","name":"image.png","type":"image/png","lastModified":1684077927269.667,"size":186955},{"id":"0c2cf50fdd3f7263ae25c8ce9c10120f388aca0104dc404c96035f8e6f577ac93f0fac3cf732f4b86fd4a9997845080d95abdd7ba7512ee32d8eb077dbeb5d57","href":"https://static.observableusercontent.com/files/0c2cf50fdd3f7263ae25c8ce9c10120f388aca0104dc404c96035f8e6f577ac93f0fac3cf732f4b86fd4a9997845080d95abdd7ba7512ee32d8eb077dbeb5d57","name":"image@1.png","type":"image/png","lastModified":1684085271103.281,"size":31061},{"id":"7b41010d33f98c2a0a0adf3e4d42628c8424ac07d4b163c33f03cf496497eb8c96267ad1b615dc45b4e144d963e0cfdf6e58d6675b63c235e8de7336e62c7b9e","href":"https://static.observableusercontent.com/files/7b41010d33f98c2a0a0adf3e4d42628c8424ac07d4b163c33f03cf496497eb8c96267ad1b615dc45b4e144d963e0cfdf6e58d6675b63c235e8de7336e62c7b9e","name":"image@3.png","type":"image/png","lastModified":1684085589167.852,"size":138692}] 30:{"id":"93293bee8c8e191a","slug":null,"title":"Interactive Convolutional Neural Network (CNN)","thumbnail":"8071cd3a354775ba2a4d6cad1e6edf4876d66ba5bb052eda8926cbf09a22e9f7","access_level":"public","create_time":"$D2023-04-21T20:51:04.211Z","update_time":"$D2023-06-07T20:55:22.568Z","view_time":null,"first_public_time":"$D2023-04-26T15:02:36.211Z","has_unpublished_edits":null,"fork":{"id":"7084b41c752b126d","slug":"interactive-convolutional-neural-network","title":"Interactive Convolutional Neural Network","owner":{"id":"37829517efb06ab9","login":"tpreusse","name":"Thomas Preusse"}},"stars":1,"starred":false,"owner":{"type":"team","id":"f57b9f8a19d28778","login":"mariodelgadosr","name":"Mario Delgado ","avatar_url":"https://avatars-next.observableusercontent.com/2d2982ad09c4ca9dfd6570810ae00301e0db6ef798c4f7025ea03d4325d4807d","tier":"starter_2024"},"version":1941,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"1d3988d482a8ec0f","login":"mariodelgadosr","name":"Mario Delgado ","avatar_url":"https://avatars-next.observableusercontent.com/2d2982ad09c4ca9dfd6570810ae00301e0db6ef798c4f7025ea03d4325d4807d","tier":null}],"members":[{"type":"individual","id":"1d3988d482a8ec0f","login":"mariodelgadosr","name":"Mario Delgado ","avatar_url":"https://avatars-next.observableusercontent.com/2d2982ad09c4ca9dfd6570810ae00301e0db6ef798c4f7025ea03d4325d4807d","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":{"id":"0dd4b0da06a9764f","slug":"youre-never-too-old-to-code","title":"You're Never Too Old To Code","owner":{"id":"f57b9f8a19d28778","login":"mariodelgadosr","name":"Mario Delgado "}},"type":"notebook","body":{"title":"Interactive Convolutional Neural Network (CNN)","cells":[{"id":0,"value":"md`# Interactive [Convolutional Neural Network (CNN)](https://en.wikipedia.org/wiki/Convolutional_neural_network)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1833,"value":"toc(\"h2,h3,h4\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1850,"value":"## Background\n### Pixel Perfect Picker","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1815,"value":"viewof color = Inputs.color({\n label: \"Pick a Color for a Pixel\",\n value: d3.rgb(255, 255, 255).hex()\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1905,"value":"colorSelected = ({\n red: d3.rgb(color).r,\n green: d3.rgb(color).g,\n blue: d3.rgb(color).b,\n decimalValue: d3.format(\",\")(parseInt(color.substring(1), 16))\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1795,"value":"md`\n\n### Video: Concolutional Neural Networks\n\n### [Convolution Visualizer](https://ezyang.github.io/convolution-visualizer/)\n![image1.png](${await FileAttachment(\"image@1.png\").url()})\n -[Google’s MobileNets on the iPhone](https://machinethink.net/blog/googles-mobile-net-architecture-on-iphone/) \n\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1924,"value":"$32","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1927,"value":"eiChart","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1929,"value":"viewof alpha = Inputs.range([0, 1], {\n value: 0.5,\n step: 0.1,\n label: \"Alpha Scaling\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1931,"value":"viewof beta = Inputs.range([0, 0.2], {\n value: 0.1,\n step: 0.01,\n label: \"Beta Decay\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1933,"value":"viewof excRate = Inputs.range([0, 20], {\n value: 12,\n step: 0.1,\n label: \"Excitation Rate\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1938,"value":"viewof inhRate = Inputs.range([0.1, 20], {\n value: 12,\n step: 0.1,\n label: \"Inhibition Rate\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1920,"value":"### [Neural Network Playground](http://playground.tensorflow.org) \n[Understanding neural networks with TensorFlow Playground](https://cloud.google.com/blog/products/ai-machine-learning/understanding-neural-networks-with-tensorflow-playground)\n[![image3.png](${await FileAttachment(\n \"image@3.png\"\n).url()})](http://playground.tensorflow.org)\n\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1922,"value":"### Advanced\n* [Image Tensors](https://observablehq.com/@jeyabbalas/image-tensors?collection=@jeyabbalas/tensorflow-js)\n* [Pretrained models (image classification)](https://observablehq.com/@jeyabbalas/pretrained-models?collection=@jeyabbalas/tensorflow-js)\n* [Advanced pretrained models (object detection)](https://observablehq.com/@jeyabbalas/object-detection?collection=@jeyabbalas/tensorflow-js)\n* [Peering Inside the Black Box](https://www.republik.ch/2018/06/26/peering-inside-the-black-box)","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1837,"value":"## The [Modified National Institute of Standards and Technology (MINST)](https://en.wikipedia.org/wiki/MNIST_database) Model \n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1844,"value":"### [Visualizing the Minst Model](https://projector.tensorflow.org/)\n[Usage](https://www.tensorflow.org/tensorboard/tensorboard_projector_plugin)\n\n[

](https://projector.tensorflow.org/)","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":436,"value":"md`### MINST Model in Action\nDraw a digit below`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":84,"value":"input.canvas","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":77,"value":"{\n const clearButton = html``\n clearButton.onclick = (e) => {\n e.preventDefault()\n input.clear()\n }\n\n return clearButton\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1663,"value":"md`The drawing is centered, inverted and fed into the network:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1353,"value":"vis.draw(topology, results)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1666,"value":"md`Legend: \nGreen: positiv weight or bias \nPurple: negative weight or bias \nSquare: weight \nRectangle: bias `","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":789,"value":"{\n const div = document.createElement('div')\n\n const newModelButton = html``\n newModelButton.onclick = (e) => {\n e.preventDefault()\n mutable model = createModel()\n mutable accuracy = undefined\n mutable loss = undefined\n }\n div.appendChild(newModelButton)\n div.appendChild(document.createTextNode(' '))\n \n const loadPretrainedButton = html``\n loadPretrainedButton.onclick = (e) => {\n e.preventDefault()\n mutable model = loadPretrained()\n mutable accuracy = undefined\n mutable loss = undefined\n }\n div.appendChild(loadPretrainedButton)\n \n return div\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":453,"value":"{\n const div = document.createElement('div')\n \n if (!model._built) {\n return div\n }\n \n if (!data) {\n const loadDataButton = html``\n loadDataButton.onclick = (e) => {\n e.preventDefault()\n mutable data = loadData()\n }\n div.appendChild(loadDataButton)\n div.appendChild(document.createTextNode(' '))\n }\n \n const trainButton = data && html``\n if (trainButton) { \n trainButton.onclick = (e) => {\n e.preventDefault()\n train(model, data)\n }\n div.appendChild(trainButton)\n }\n \n\n const testButton = data && html``\n if (testButton) {\n testButton.onclick = (e) => {\n e.preventDefault()\n logPredictionTest(model, data)\n }\n div.appendChild(document.createTextNode(' '))\n div.appendChild(testButton)\n }\n \n return div\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":438,"value":"md`Model Info:\n${accuracy ? `\n**accuracy:** ${accuracy} \n**loss:** ${loss}\n` : model._built ? 'untrained' : 'pretrained'}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1874,"value":"## Notebook Objects","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1718,"value":"mutable download = null","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1694,"value":"DRAW_RESULT = true","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1697,"value":"HIDDEN_CONNECTIONS = [\n // \"max_pooling2d_MaxPooling2D1\",\n // \"dense_Dense1\",\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1699,"value":"DPI = 4 // window.devicePixelRatio","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":784,"value":"mutable model = loadPretrained()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1111,"value":"mutable results = {\n return readResults(canvasWithInput, this)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":830,"value":"mutable topology = readTopology({model})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1348,"value":"vis = createVis()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":922,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"async function createModel() {\n const model = tf.sequential()\n \n model.add(tf.layers.conv2d({\n inputShape: [28, 28, 1],\n kernelSize: 5,\n filters: 8,\n strides: 1,\n activation: 'relu',\n kernelInitializer: 'VarianceScaling'\n }))\n model.add(tf.layers.maxPooling2d({\n poolSize: [2, 2],\n strides: [2, 2]\n }))\n model.add(tf.layers.conv2d({\n kernelSize: 5,\n filters: 16,\n strides: 1,\n activation: 'relu',\n kernelInitializer: 'VarianceScaling'\n }))\n model.add(tf.layers.maxPooling2d({\n poolSize: [2, 2],\n strides: [2, 2]\n }))\n model.add(tf.layers.flatten())\n model.add(tf.layers.dense({\n units: 10,\n kernelInitializer: 'VarianceScaling',\n activation: 'softmax',\n // useBias: false\n }))\n \n const optimizer = tf.train.sgd(LEARNING_RATE)\n \n await model.compile({\n optimizer: optimizer,\n loss: 'categoricalCrossentropy',\n metrics: ['accuracy']\n })\n \n return model\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":819,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1128,"value":"mutable canvasWithInput = null","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":312,"value":"input = createInput({\n onDraw(canvas) {\n mutable canvasWithInput = canvas\n },\n onClear() {\n mutable canvasWithInput = null\n }\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":336,"value":"result = createResult({model})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1288,"value":"function readResults(canvas, prev) {\n prev && prev.forEach(result => result.dispose())\n if (!canvas) {\n return null\n }\n return result.read(canvas)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":318,"value":"IMG_SIZE = 28","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":21,"value":"LEARNING_RATE = 0.15","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":25,"value":"BATCH_SIZE = 64","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":27,"value":"TRAIN_BATCHES = 100","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":29,"value":"TEST_BATCH_SIZE = 1000","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":31,"value":"TEST_ITERATION_FREQUENCY = 5","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":33,"value":"IMAGE_SIZE = 784","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":35,"value":"NUM_CLASSES = 10","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":37,"value":"NUM_DATASET_ELEMENTS = 65000","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":39,"value":"NUM_TRAIN_ELEMENTS = 55000","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":41,"value":"NUM_TEST_ELEMENTS = NUM_DATASET_ELEMENTS - NUM_TRAIN_ELEMENTS","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":43,"value":"MNIST_IMAGES_SPRITE_PATH =\n 'https://cdn.republik.space/s3/republik-assets/dynamic-components/talk-to-the-machine/neural-network/mnist_images.png'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":45,"value":"MNIST_LABELS_PATH =\n 'https://cdn.republik.space/s3/republik-assets/dynamic-components/talk-to-the-machine/neural-network/mnist_labels_uint8'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":680,"value":"async function loadPretrained() {\n const model = await tf.loadModel('https://cdn.republik.space/s3/republik-assets/dynamic-components/talk-to-the-machine/neural-network/the98one.json')\n return model\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1420,"value":"async function loadData() {\n const data = new MnistData()\n await data.load()\n return data\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":47,"value":"$35","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1430,"value":"mutable data = null","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":54,"value":"mutable accuracy = 0","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":59,"value":"mutable loss = 0","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":52,"value":"$36","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":182,"value":"async function logPredictionTest(model, data) {\n const testExamples = 100\n const batch = data.nextTestBatch(testExamples)\n\n tf.tidy(() => {\n const output = model.predict(batch.xs.reshape([-1, 28, 28, 1]))\n\n const axis = 1\n const labels = Array.from(batch.labels.argMax(axis).dataSync())\n const predictions = Array.from(output.argMax(axis).dataSync())\n\n let right = 0\n const wrongs = d3.range(10).map(() => 0)\n predictions.map((v, i) => {\n if (labels[i] === v) {\n right++\n } else {\n // console.log('wrong', labels[i], v)\n wrongs[labels[i]]++\n }\n })\n console.log(right + '% correct, wrong histogram:', wrongs)\n })\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":798,"value":"$37","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":316,"value":"$38","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":70,"value":"$39","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":353,"value":"function drawData(tensor, context) {\n const imageData = new ImageData(IMG_SIZE, IMG_SIZE);\n const data = tensor.dataSync();\n for (let i = 0; i < IMG_SIZE * IMG_SIZE; ++i) {\n const j = i * 4;\n imageData.data[j + 0] = data[i] * 255\n imageData.data[j + 1] = data[i] * 255\n imageData.data[j + 2] = data[i] * 255\n imageData.data[j + 3] = 255\n }\n context.putImageData(imageData, 0, 0)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1465,"value":"throttle = _.throttle","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1493,"value":"debounce = _.debounce","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1453,"value":"_ = require('lodash')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"tf = require('@tensorflow/tfjs@0.11.6')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":68,"value":"d3 = require('d3@5')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1830,"value":"import { toc } from \"@sugi2000/toc\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1915,"value":"import { eiChart, set } with {\n alpha as alpha,\n beta as beta,\n excRate as excRate,\n inhRate as inhRate\n} from \"@patrickmineault/balanced-e-i-networks\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W3a","stdlib":"1","titling":"auto"}} 31:"$30:body"