2c:null 2d:["$","title",null,{"children":"Visualizing activations of MobileNet with TensorFlow.js (webcam only) | Andy Smy | Observable"}] 32:T41f,source = { if (webcam) { const vid = html`

`; navigator.mediaDevices.getUserMedia({ video: {facingMode: "environment", width: 224, height: 224}, //your default //{ deviceId: { exact: '978cac91d8237273fea5309b96450b162a0f40920c1d285760470de9c8bb80a1' } }, // my Logitech //{ deviceId: { exact: '80bcef3f37bd17206e40a747b89c07b63a07bfeb6ed7a87de2088ff7b59c3ecf' } }, // my Trust audio: false }).then((stream) => { vid.srcObject = stream; vid.onloadedmetadata = () => vid.play(); //const used_devices = stream.getTracks() // .map( (track) => track.getSettings().deviceId ); //use these lines to find your webcam deviceId //console.log(used_devices); }); return vid; } else { const vid = document.querySelector('#vid'); if (vid && vid.srcObject) { vid.srcObject.getTracks().forEach(t => { t.stop(); }); } return html`

(webcam not active)

` } }33:T618,collageFn = { return pixels => { return tf.tidy(() => { // Feed the input through the model. let activation = models[layerId].predict(pixels.expandDims(0)).squeeze(); // Normalize within each channel if the checkbox is checked. // Global pool so we compute normalization params for each channel. const strides = 1; // Doesn't matter const pad = 0; const maxpool = activation.maxPool([activation.shape[0], activation.shape[1]], strides, pad); const minpool = activation.neg().maxPool( [activation.shape[0], activation.shape[1]], strides, pad).neg() // To avoid dividing by zero. .add(tf.scalar(.0000001)); activation = activation.sub(minpool).div(maxpool.sub(minpool)); // Transpose the channels to the outer most dimension and then reshape the activation so we can show // it in a rectangular collage. const reshaped = activation.transpose([2, 0, 1]).reshape( [collageDims[layerId][0], collageDims[layerId][1], activation.shape[0], activation.shape[1]]); // Split channels between rows and columns. [16, width, height] becomes [4, width, 4, height] const transposed = reshaped.transpose([0, 2, 1, 3]); // Combine channels. [4, height, 4, height] becomes [4 * height, 4 * height] let output = transposed.reshape( [transposed.shape[0] * transposed.shape[1], transposed.shape[2] * transposed.shape[3]]); return output.maximum(tf.scalar(0)).minimum(tf.scalar(1)); }); } }34:[] 30:{"id":"889586f08bc75f5d","slug":"visualizing-activations-of-mobilenet-with-tensorflow-js-w","title":"Visualizing activations of MobileNet with TensorFlow.js (webcam only)","thumbnail":"8cc376ac099258bcec3de04bfd376a454809ae470560ecf8240a57aaf25befbc","access_level":"public","create_time":"$D2019-02-15T14:37:13.882Z","update_time":"$D2022-07-12T10:13:28.105Z","view_time":null,"first_public_time":"$D2019-02-15T15:27:29.352Z","fork":{"id":"4e3769c81c2b8634","slug":"visualizing-activations-of-mobilenet-with-tensorflow-js","title":"Visualizing activations of MobileNet with TensorFlow.js","owner":{"id":"a9fc173995ab08d5","login":"nsthorat","name":"Nikhil Thorat"}},"stars":1,"starred":false,"owner":{"type":"team","id":"8009a012d73ccc24","login":"andysmee","name":"Andy Smy","avatar_url":"https://avatars-next.observableusercontent.com/b97842f8f180a0a1c3373eb40effd86beb9906c1075f408f2613dedd5115da1c","tier":"starter_2024"},"version":784,"publish_version":784,"role":null,"authored":false,"authors":[{"type":"individual","id":"8af36cf1436b1198","login":"andysmee","name":"Andy Smy","avatar_url":"https://avatars-next.observableusercontent.com/b97842f8f180a0a1c3373eb40effd86beb9906c1075f408f2613dedd5115da1c","tier":null}],"members":[{"type":"individual","id":"8af36cf1436b1198","login":"andysmee","name":"Andy Smy","avatar_url":"https://avatars-next.observableusercontent.com/b97842f8f180a0a1c3373eb40effd86beb9906c1075f408f2613dedd5115da1c","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":{"id":"a320cbd1597add2c","slug":"machine-learning","title":"Machine Learning","owner":{"id":"8009a012d73ccc24","login":"andysmee","name":"Andy Smy"}},"type":"notebook","body":{"title":"Visualizing activations of MobileNet with TensorFlow.js (webcam only)","cells":[{"id":0,"value":"md`# Visualizing activations of MobileNet with TensorFlow.js (webcam only)\n\nFork of...\n\nVisualizing activations of MobileNet with TensorFlow.js by @nsthoratNikhil Thorat, simplified to webcam only`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":372,"value":"md`\nThis notebook demonstrates how to visualize internal activations from MobileNet using [TensorFlow.js](https://js.tensorflow.org).\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":502,"value":"md`***Check the box below to see webcam activations visualized in real time.***`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":337,"value":"viewof webcam = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":348,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":214,"value":"viewof layerId = {\n const el = html``;\n for (let i = 0; i < layers.length; i++) {\n const layer = layers[i];\n const option = document.createElement('option');\n option.value = i;\n option.innerText = layer.name;\n el.appendChild(option);\n }\n return el;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":44,"value":"{ \n // This isn't working on iOS.\n // if (source.complete === false) {\n // await new Promise(resolve => {\n // source.onload = resolve;\n // });\n // }\n if (webcam) {\n const getCollage = () => {\n return tf.tidy(() => { \n const pixels = tf.fromPixels(source).toFloat().sub(offset).div(offset);\n return collageFn(pixels);\n });\n };\n\n const init = getCollage();\n\n const [height, width] = init.shape;\n const c = DOM.context2d(width, height);\n\n c.canvas.width = width;\n c.canvas.height = height;\n c.canvas.style.width = scale * width + 'px';\n c.canvas.style.height = scale * height + 'px';\n\n while(true) {\n const collage = getCollage();\n await tf.toPixels(collage, c.canvas);\n\n yield c.canvas;\n collage.dispose();\n }\n } else {\n yield html`\n

(source not active)

\n `\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":245,"value":"activationShape = models[layerId].output.shape.slice(1)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":376,"value":"md`Visual scaling factor of collage: **${scale}**`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":253,"value":"viewof scale = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":399,"value":"md`The following cell is where the collage tensor is computed.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":289,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":530,"value":"md`### Observations`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":533,"value":"md`\nYou'll notice that some of the channel activations are zeros after the first conv. I'm not quite sure whether I have a bug in the collage code, or whether the activations are actually dead. For any motivated readers: this is worth investigating.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":91,"value":"md`### Layers / models`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":409,"value":"md`To construct truncated versions of a model and share layer parameters, we can construct a new model whose inputs are the inputs of MobileNet and the output is an intermediate layer.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":568,"value":"mobilenet.layers.map(l => l.name)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":70,"value":"layers = mobilenet.layers.filter((l) => l.name.startsWith('conv') && !l.name.endsWith('preds'))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":76,"value":"models = layers.map(layer => tf.model({inputs: mobilenet.inputs, outputs: layer.output}))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":82,"value":"shapes = models.map(model => model.outputs[0].shape.slice(1))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":130,"value":"collageDims = shapes.map(shape => toSquareish(shape[2]))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":95,"value":"md`### Utils`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":121,"value":"toSquareish = {\n return x => {\n let sqrt = Math.sqrt(x);\n if (Number.isInteger(sqrt)) {\n return [sqrt, sqrt];\n }\n sqrt = Math.floor(sqrt);\n // Who cares that this is slow.\n while (!Number.isInteger(x / sqrt)) {\n sqrt--;\n }\n return [sqrt, x / sqrt];\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":19,"value":"mobilenet = await tf.loadModel('https://storage.googleapis.com/tfjs-models/tfjs/mobilenet_v1_0.25_224/model.json')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":41,"value":"offset = tf.scalar(127.5)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"tf = require('@tensorflow/tfjs@0.10.3')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W34","stdlib":"1","titling":"auto"}} 31:"$30:body"