34:["$","title",null,{"children":"Transfer learning on MobileNetV2, including visualisation of activations, with TensorFlow.js | Andy Smy | Observable"}] 3c:T618,collageFn = { return pixels => { return tf.tidy(() => { // Feed the input through the model. let activation = models[layerId].predict(pixels.expandDims(0)).squeeze(); // Normalize within each channel if the checkbox is checked. // Global pool so we compute normalization params for each channel. const strides = 1; // Doesn't matter const pad = 0; const maxpool = activation.maxPool([activation.shape[0], activation.shape[1]], strides, pad); const minpool = activation.neg().maxPool( [activation.shape[0], activation.shape[1]], strides, pad).neg() // To avoid dividing by zero. .add(tf.scalar(.0000001)); activation = activation.sub(minpool).div(maxpool.sub(minpool)); // Transpose the channels to the outer most dimension and then reshape the activation so we can show // it in a rectangular collage. const reshaped = activation.transpose([2, 0, 1]).reshape( [collageDims[layerId][0], collageDims[layerId][1], activation.shape[0], activation.shape[1]]); // Split channels between rows and columns. [16, width, height] becomes [4, width, 4, height] const transposed = reshaped.transpose([0, 2, 1, 3]); // Combine channels. [4, height, 4, height] becomes [4 * height, 4 * height] let output = transposed.reshape( [transposed.shape[0] * transposed.shape[1], transposed.shape[2] * transposed.shape[3]]); return output.maximum(tf.scalar(0)).minimum(tf.scalar(1)); }); } }3d:T434,tensor2img = (tensor, classIndex) => { // halves size of image to show on page, not used in model const [width, height] = [IMAGE_SIZE, IMAGE_SIZE]; //[IMAGE_SIZE / 2, IMAGE_SIZE / 2]; var c = document.getElementById("canvasClass" + classIndex); var ctx = c.getContext("2d"); //const ctx = document.getElementById('canvas0').getContext('2d'); const imageData = new ImageData(width, height); const data = tf.tidy(() => tf.image.resizeBilinear(tensor, [width, height]).dataSync() ); for (let i = 0; i < height * width; ++i) { const j = i * 4; imageData.data[j + 0] = (data[i * 3 + 0] + 1) * 127; // R imageData.data[j + 1] = (data[i * 3 + 1] + 1) * 127; // G imageData.data[j + 2] = (data[i * 3 + 2] + 1) * 127; // B imageData.data[j + 3] = 255; } var font = "16px sans-serif"; ctx.font = font; ctx.textBaseline = "top"; var textWidth = ctx.measureText(classIndex).width; var textHeight = parseInt(font, 10); // base 10 ctx.fillStyle = "#00FFFF"; ctx.putImageData(imageData, 0, 0); ctx.fillText(classIndex, 3, 3); }3e:[] 38:{"id":"ae4d6c8a0d5ca827","slug":"transfer-learning-on-mobilenetv2-including-visualisatio","title":"Transfer learning on MobileNetV2, including visualisation of activations, with TensorFlow.js","thumbnail":"06db7c896e3be9b7d08f111fd3963297d6919ad90e0e7c99ec673ff160d58db6","access_level":"public","create_time":"$D2019-02-15T16:01:34.483Z","update_time":"$D2022-07-29T20:49:54.004Z","view_time":null,"first_public_time":"$D2019-02-17T17:26:54.875Z","has_unpublished_edits":null,"fork":{"id":"889586f08bc75f5d","slug":"visualizing-activations-of-mobilenet-with-tensorflow-js-w","title":"Visualizing activations of MobileNet with TensorFlow.js (webcam only)","owner":{"id":"8009a012d73ccc24","login":"andysmee","name":"Andy Smy"}},"stars":0,"starred":false,"owner":{"type":"team","id":"8009a012d73ccc24","login":"andysmee","name":"Andy Smy","avatar_url":"https://avatars-next.observableusercontent.com/b97842f8f180a0a1c3373eb40effd86beb9906c1075f408f2613dedd5115da1c","tier":"starter_2024","bio":"","followed_by_you":false,"follows_you":false},"version":972,"publish_version":972,"role":null,"authored":false,"authors":[{"type":"individual","id":"8af36cf1436b1198","login":"andysmee","name":"Andy Smy","avatar_url":"https://avatars-next.observableusercontent.com/b97842f8f180a0a1c3373eb40effd86beb9906c1075f408f2613dedd5115da1c","tier":null,"bio":"","followed_by_you":false,"follows_you":false}],"members":[{"type":"individual","id":"8af36cf1436b1198","login":"andysmee","name":"Andy Smy","avatar_url":"https://avatars-next.observableusercontent.com/b97842f8f180a0a1c3373eb40effd86beb9906c1075f408f2613dedd5115da1c","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":5,"collection":{"id":"a320cbd1597add2c","slug":"machine-learning","title":"Machine Learning","owner":{"id":"8009a012d73ccc24","login":"andysmee","name":"Andy Smy"}},"type":"notebook","body":{"titling":"auto","title":"Transfer learning on MobileNetV2, including visualisation of activations, with TensorFlow.js","cells":[{"id":0,"value":"md`# Transfer learning on MobileNetV2, including visualisation of activations, with TensorFlow.js\n\nWebcam only forks of...\n\nVisualizing activations of MobileNet with TensorFlow.js by @nsthoratNikhil Thorat (https://beta.observablehq.com/@nsthorat/visualizing-activations-of-mobilenet-with-tensorflow-js), simplified to webcam only, and Machine Learning in JS with TensorFlow.js (Part III) by @tvirot\"Ta\" Virot C (https://beta.observablehq.com/@tvirot/machine-learning-in-javascript-with-tensorflow-js-part-iii).\n\n1. Activate webcam and look at visualisations of layers of pre-loaded model\n2. Add examples for new class A, B & C into new model, using truncated pre-loaded model\n3. Activate training of new model\n4. Stop training and activate predictions from new model`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":502,"value":"md`***Check the box below to see webcam activations visualized in real time.***`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":337,"value":"viewof webcam = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":376,"value":"md`Visual scaling factor of collage: **${scale}**`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":253,"value":"viewof scale = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":214,"value":"viewof layerId = {\n const el = html``;\n for (let i = 0; i < layers.length; i++) {\n const layer = layers[i];\n const option = document.createElement('option');\n option.value = i;\n option.innerText = layer.name;\n el.appendChild(option);\n }\n return el;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":44,"value":"{ \n // This isn't working on iOS.\n // if (source.complete === false) {\n // await new Promise(resolve => {\n // source.onload = resolve;\n // });\n // }\n if (webcam) {\n const getCollage = () => {\n return tf.tidy(() => { \n const pixels = tf.browser.fromPixels(input).toFloat().sub(offset).div(offset);\n return collageFn(pixels);\n });\n };\n\n const init = getCollage();\n\n const [height, width] = init.shape;\n const c = DOM.context2d(width, height);\n\n c.canvas.width = width;\n c.canvas.height = height;\n c.canvas.style.width = scale * width + 'px';\n c.canvas.style.height = scale * height + 'px';\n\n while(true) {\n const collage = getCollage();\n await tf.browser.toPixels(collage, c.canvas);\n\n yield c.canvas;\n collage.dispose();\n }\n } else {\n yield html`\n

(source not active)

\n `\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":348,"value":"input = {\n if (webcam) {\n const vid = html`

`;\n navigator.mediaDevices.getUserMedia({ \n video: {facingMode: \"environment\", width: 224, height: 224}, \n //{ deviceId: { exact: '978cac91d8237273fea5309b96450b162a0f40920c1d285760470de9c8bb80a1' } }, // my Logitech \n //{ deviceId: { exact: '80bcef3f37bd17206e40a747b89c07b63a07bfeb6ed7a87de2088ff7b59c3ecf' } }, // my Trust \n audio: false\n }).then((stream) => {\n vid.srcObject = stream;\n vid.onloadedmetadata = () => vid.play();\n });\n return vid;\n } else {\n const vid = document.querySelector('#vid');\n if (vid && vid.srcObject) {\n vid.srcObject.getTracks().forEach(t => { t.stop(); });\n }\n return html`\n

(webcam not active)

\n `\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":901,"value":"canvases = {\n var canvasouterHTML = ``;\n var i;\n for (i = 0; i < NUM_CLASSES; i++) {\n canvasouterHTML += `

`;\n }\n const canvases = html`${canvasouterHTML}`;\n console.log(canvases);\n return canvases;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":802,"value":"controller = {\n //if (isPredicting) return html``;\n\n var bluebuttons = ``;\n var i;\n for (i = 0; i < NUM_CLASSES; i++) {\n bluebuttons += ` `;\n }\n\n //const controller = html`\n //

\n//`;\n console.log(html`${bluebuttons}`);\n const controller = html`${bluebuttons}`;\n\n const buttons = controller.querySelectorAll('button');\n [...buttons].forEach((d,i) => {\n d.onclick = (e) => {\n e.preventDefault();\n tf.tidy(() => {\n const example = capture();\n addExample(mobilenet_truncated.predict(example), i);\n });\n tensor2img(capture(), i);\n }\n });\n \n return controller;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":806,"value":"md`***Check the box below to see predictions in real time.***`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":803,"value":"viewof isPredicting = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":879,"value":"predictedClass = {\n while (isPredicting) {\n const predictedClass = tf.tidy(() => {\n const img = capture();\n const activation = mobilenet_truncated.predict(img);\n const predictions = model.predict(activation);\n return predictions.as1D().argMax();\n });\n await tf.nextFrame();\n // alternatively, highlight the example that matches\n var i\n for (i = 0; i < NUM_CLASSES; i++) {\n document.getElementById(\"canvasClass\" + i).style.border = '1px solid #000000';\n }\n document.getElementById(\"canvasClass\" + [predictedClass.dataSync()[0]]).style.border = '3px solid #FF0000';\n\n yield md`Predicted Class: ${['A', 'B', 'C'][predictedClass.dataSync()[0]]}`;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":886,"value":"md`***Check the box to run training:***`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":841,"value":"viewof isTraining = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":845,"value":"trainingInfo = {\n let i = 0;\n while (isTraining) {\n const history = await model.fit(xs, ys, {\n batchSize: 8,\n epochs: 5\n });\n const loss = history.history.loss[0];\n console.log('doing this');\n yield {i: i++, loss};\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":837,"value":"md`## Training the Model`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":782,"value":"capture = () => {\n return tf.tidy(() => {\n const webcamImage = tf.browser.fromPixels(input).toFloat();\n const croppedImage = cropImage(webcamImage);\n const batchedImage = croppedImage.expandDims(0);\n const captureOffset = tf.scalar(127);\n return batchedImage.sub(captureOffset).div(captureOffset);\n });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":795,"value":"mutable xs = null","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":797,"value":"mutable ys = null","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":793,"value":"addExample = (example, label) => {\n // One-hot encode the label.\n const y = tf.tidy(() => tf.oneHot(tf.tensor1d([label], 'int32'), NUM_CLASSES));\n\n if (xs == null) {\n mutable xs = tf.keep(example);\n mutable ys = tf.keep(y);\n } else {\n const oldX = xs;\n mutable xs = tf.keep(oldX.concat(example, 0));\n\n const oldY = ys;\n mutable ys = tf.keep(oldY.concat(y, 0));\n\n oldX.dispose();\n oldY.dispose();\n y.dispose();\n }\n }","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":839,"value":"model = {\n const m = tf.sequential({\n layers: [\n // Flattens the input to a vector so we can use it in a dense layer. While\n // technically a layer, this only performs a reshape (and has no training\n // parameters).\n tf.layers.flatten({inputShape: [7, 7, 256]}),\n tf.layers.dense({\n units: 100,\n activation: 'relu',\n kernelInitializer: 'varianceScaling',\n useBias: true\n }),\n // The number of units of the last layer should correspond\n // to the number of classes we want to predict.\n tf.layers.dense({\n units: NUM_CLASSES,\n kernelInitializer: 'varianceScaling',\n useBias: false,\n activation: 'softmax'\n })\n ]\n });\n \n const optimizer = tf.train.adam();\n m.compile({optimizer: optimizer, loss: 'categoricalCrossentropy'});\n \n return m;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":91,"value":"md`### Layers / models`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":409,"value":"md`To construct truncated versions of a model and share layer parameters, we can construct a new model whose inputs are the inputs of MobileNet and the output is an intermediate layer.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":568,"value":"mobilenet.layers.map(l => l.name)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":70,"value":"layers = mobilenet.layers.filter((l) => l.name.startsWith('conv') && !l.name.endsWith('preds'))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":76,"value":"models = layers.map(layer => tf.model({inputs: mobilenet.inputs, outputs: layer.output}))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":82,"value":"shapes = models.map(model => model.outputs[0].shape.slice(1))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":19,"value":"mobilenet = await tf.loadModel('https://storage.googleapis.com/tfjs-models/tfjs/mobilenet_v1_0.25_224/model.json')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":773,"value":"mobilenet_truncated = {\n const layer = mobilenet.getLayer('conv_pw_13_relu');\n \n return tf.model({inputs: mobilenet.inputs, outputs: layer.output});\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":95,"value":"md`### Utils`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":121,"value":"toSquareish = {\n return x => {\n let sqrt = Math.sqrt(x);\n if (Number.isInteger(sqrt)) {\n return [sqrt, sqrt];\n }\n sqrt = Math.floor(sqrt);\n // Who cares that this is slow.\n while (!Number.isInteger(x / sqrt)) {\n sqrt--;\n }\n return [sqrt, x / sqrt];\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":289,"value":"$3c","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":130,"value":"collageDims = shapes.map(shape => toSquareish(shape[2]))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":245,"value":"activationShape = models[layerId].output.shape.slice(1)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":41,"value":"offset = tf.scalar(127)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":791,"value":"$3d","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":784,"value":"cropImage = (img) => {\n// const size = Math.min(img.shape[0], img.shape[1]); // find the shortest dimension based on (height, width)\n const size = IMAGE_SIZE; // for me, I'm setting the size to 224 \n const centerHeight = img.shape[0] / 2; // find the vertical centre\n const beginHeight = centerHeight - (size / 2); // cropping to the shortest dimension square, around the centre\n const centerWidth = img.shape[1] / 2; // find the horizontal centre\n const beginWidth = centerWidth - (size / 2); // cropping to the shortest dimension square, around the centre\n return img.slice([beginHeight, beginWidth, 0], [size, size, 3]); // perform the crop\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":787,"value":"IMAGE_SIZE = 224","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":789,"value":"NUM_CLASSES = 4","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"tf = require('@tensorflow/tfjs@0.15.1') // 0.10.3","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W3e","stdlib":"1"}} 33:null