2d:null 2e:["$","title",null,{"children":"Machine Learning in JS with TensorFlow.js (Part III) | \"Ta\" Virot C. | Observable"}] 33:[] 31:{"id":"7ae62ae9482d3001","slug":"machine-learning-in-javascript-with-tensorflow-js-part-iii","title":"Machine Learning in JS with TensorFlow.js (Part III)","thumbnail":"85110b96510764707a7a24fc85e0d75e4317a4fcc8001ca03159730fe717bf4a","access_level":"public","create_time":"$D2018-04-20T11:25:22.047Z","update_time":"$D2019-03-20T16:16:13.454Z","view_time":null,"first_public_time":"$D2018-04-21T01:06:30.214Z","has_unpublished_edits":null,"fork":false,"stars":14,"starred":false,"owner":{"type":"team","id":"05fe87ce00c24684","login":"tvirot","name":"\"Ta\" Virot C.","avatar_url":"https://avatars-next.observableusercontent.com/a2b53d057ac064ea2a178f61376d79ebffd11284cadbef50588e520a74a730b2","tier":"starter_2024"},"version":398,"publish_version":398,"role":null,"authored":false,"authors":[{"type":"individual","id":"679d00b8a8620131","login":"tvirot","name":"\"Ta\" Virot C.","avatar_url":"https://avatars-next.observableusercontent.com/a2b53d057ac064ea2a178f61376d79ebffd11284cadbef50588e520a74a730b2","tier":null}],"members":[{"type":"individual","id":"679d00b8a8620131","login":"tvirot","name":"\"Ta\" Virot C.","avatar_url":"https://avatars-next.observableusercontent.com/a2b53d057ac064ea2a178f61376d79ebffd11284cadbef50588e520a74a730b2","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":25,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Machine Learning in JS with TensorFlow.js (Part III)","cells":[{"id":0,"value":"md`# Machine Learning in JS with TensorFlow.js (Part III)\n\nIn the previous notebooks ([Part I](https://beta.observablehq.com/@tvirot/machine-learning-in-javascript-with-tensorflow-js), [Part II](https://beta.observablehq.com/@tvirot/machine-learning-in-javascript-with-tensorflow-js-part-ii)), we discuss two use cases of TensorFlow.js:\n1. training a new model from scratch, and \n2. importing a pre-trained model for inference. \n\nNow we are going to do something in between. We will train (or retrain, to be exact) a model from a pre-trained model, using an internal activation (the output from an internal layer) as input to our new model.\n\n## Truncated MobileNet model\n\nFirst, we will load MobileNet model and truncate it to output an internal activation, \\`conv_pw_13_relu\\`. Note that this model won't be trained after being loaded into the browser.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":11,"value":"mobilenet_truncated = {\n const mobilenet = await tf.loadModel('https://storage.googleapis.com/tfjs-models/tfjs/mobilenet_v1_0.25_224/model.json');\n // Return a model that outputs an internal activation.\n const layer = mobilenet.getLayer('conv_pw_13_relu');\n \n return tf.model({inputs: mobilenet.inputs, outputs: layer.output});\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":386,"value":"md`Check the box below to turn on your camera`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":36,"value":"viewof webcam = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":38,"value":"input = {\n if (webcam) {\n // TO-DO(tvirot): Make it work on mobile\n const vid = html`

`;\n navigator.mediaDevices.getUserMedia({\n video: {facingMode: \"environment\", width: 224, height: 224}, \n audio: false\n }).then((stream) => {\n vid.srcObject = stream;\n vid.onloadedmetadata = () => vid.play();\n });\n return vid;\n } else {\n const vid = document.querySelector('#vid');\n if (vid && vid.srcObject) {\n vid.srcObject.getTracks().forEach(t => { t.stop(); });\n }\n return html`\n

Turn on webcam to get started! Adding the first example may take a while.

\n `\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":44,"value":"controller = {\n if (isPredicting) return html``;\n const controller = html`\n

\n`;\n\n const buttons = controller.querySelectorAll('button');\n const textboxes = controller.querySelectorAll('input');\n [...buttons].forEach((d,i) => {\n d.onclick = (e) => {\n e.preventDefault();\n tf.tidy(() => {\n const example = capture();\n addExample(mobilenet_truncated.predict(example), i);\n });\n tensor2img(capture(), capturedImg);\n }\n });\n \n return controller;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":137,"value":"capturedImg = {\n if (isPredicting) return html``;\n const canvas = document.createElement('canvas');\n canvas.width = IMAGE_SIZE / 2;\n canvas.height = IMAGE_SIZE / 2;\n return canvas\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":388,"value":"md`Check this box to start making predictions (but make sure you scroll down to train a model first!). \nUncheck to collect more data.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":287,"value":"viewof isPredicting = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":349,"value":"predictedClass = {\n while (isPredicting) {\n const predictedClass = tf.tidy(() => {\n const img = capture();\n const activation = mobilenet_truncated.predict(img);\n const predictions = model.predict(activation);\n return predictions.as1D().argMax();\n });\n\n await tf.nextFrame();\n yield md`Predicted Class: ${['A', 'B', 'C'][predictedClass.dataSync()[0]]}`;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":82,"value":"capture = () => {\n return tf.tidy(() => {\n const webcamImage = tf.fromPixels(input).toFloat();\n const croppedImage = cropImage(webcamImage);\n const batchedImage = croppedImage.expandDims(0);\n const offset = tf.scalar(127);\n return batchedImage.sub(offset).div(offset);\n });\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":141,"value":"addExample = (example, label) => {\n // One-hot encode the label.\n const y = tf.tidy(() => tf.oneHot(tf.tensor1d([label]), NUM_CLASSES));\n\n if (xs == null) {\n mutable xs = tf.keep(example);\n mutable ys = tf.keep(y);\n } else {\n const oldX = xs;\n mutable xs = tf.keep(oldX.concat(example, 0));\n\n const oldY = ys;\n mutable ys = tf.keep(oldY.concat(y, 0));\n\n oldX.dispose();\n oldY.dispose();\n y.dispose();\n }\n }","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":206,"value":"mutable xs = null","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":209,"value":"mutable ys = null","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":317,"value":"md`## Training the Model`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":321,"value":"model = {\n const m = tf.sequential({\n layers: [\n // Flattens the input to a vector so we can use it in a dense layer. While\n // technically a layer, this only performs a reshape (and has no training\n // parameters).\n tf.layers.flatten({inputShape: [7, 7, 256]}),\n tf.layers.dense({\n units: 100,\n activation: 'relu',\n kernelInitializer: 'varianceScaling',\n useBias: true\n }),\n // The number of units of the last layer should correspond\n // to the number of classes we want to predict.\n tf.layers.dense({\n units: NUM_CLASSES,\n kernelInitializer: 'varianceScaling',\n useBias: false,\n activation: 'softmax'\n })\n ]\n });\n \n const optimizer = tf.train.adam();\n m.compile({optimizer: optimizer, loss: 'categoricalCrossentropy'});\n \n return m;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":396,"value":"md`Check the box to start training the model in your browser. Uncheck to stop.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":330,"value":"viewof isTraining = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":326,"value":"trainingInfo = {\n let i = 0;\n while (isTraining) {\n const history = await model.fit(xs, ys, {\n batchSize: 8,\n epochs: 5\n });\n const loss = history.history.loss[0];\n console.log('doing this');\n yield {i: i++, loss};\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":97,"value":"tensor2img = (tensor, canvas) => {\n const [width, height] = [IMAGE_SIZE / 2, IMAGE_SIZE / 2];\n const ctx = canvas.getContext('2d');\n const imageData = new ImageData(width, height);\n const data = tf.tidy(() => \n tf.image.resizeBilinear(tensor, [width, height]).dataSync()\n );\n for (let i = 0; i < height * width; ++i) {\n const j = i * 4;\n imageData.data[j + 0] = (data[i * 3 + 0] + 1) * 127; // R\n imageData.data[j + 1] = (data[i * 3 + 1] + 1) * 127; // G\n imageData.data[j + 2] = (data[i * 3 + 2] + 1) * 127; // B\n imageData.data[j + 3] = 255;\n }\n ctx.putImageData(imageData, 0, 0);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":128,"value":"cropImage = (img) => {\n const size = Math.min(img.shape[0], img.shape[1]);\n const centerHeight = img.shape[0] / 2;\n const beginHeight = centerHeight - (size / 2);\n const centerWidth = img.shape[1] / 2;\n const beginWidth = centerWidth - (size / 2);\n return img.slice([beginHeight, beginWidth, 0], [size, size, 3]);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":109,"value":"IMAGE_SIZE = 224","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":123,"value":"NUM_CLASSES = 3","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":13,"value":"tf = require('@tensorflow/tfjs@0.9.0')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":9,"value":"html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":361,"value":"md`Follow us on [Facebook](//www.facebook.com/skooldio) and [Medium](//www.medium.com/skooldio).\n\n![skooldio](https://cdn-images-1.medium.com/max/2000/1*JjMVAbfDVuVdN7DoGP5X-g@2x.png)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W33","stdlib":"1"}} 32:"$31:body"