3e:null 3f:["$","title",null,{"children":"Applied Machine Learning Days: Hands-on deep learning with TensorFlow.js | Joey Yang | Observable"}] 45:T7e2,md`# Applied Machine Learning Days: Hands-on deep learning with TensorFlow.js By the end of this part of the workshop you will: * Know how to import a TF model into the web browser, and use it to make predictions on image data. * How to navigate and write code in an Observable JavaScript notebook. * How to visualize your predictions using the D3 charting library. ## A quick note about writing code inside an Observable notebook: If you're used developing JavaScript apps, this isn't exactly the same. Notebooks use "cells" --this block of text is a cell all of it's own, in fact. Cells can contain functions, blocks of markdown text, "views", and other features. If you're having trouble following the structure of the code, then I advise first looking at the [Observable documentation on GitHub](https://github.com/observablehq/stdlib). We'll be building an interface the recognizes hand-drawn "doodles" that a user can draw inside of a canvas using either a mouse on their PC, or by hand on a smartphone or tablet. ## Fork this notebook You don't have to, of course. And I won't make you sign-up for another service that you don't want to use. You can edit this notebook, but you won't be able to save your edits. But if you fork the notebook, you'll be able to edit freely, and be able to take your code home with you. --- ## Create a drawable canvas element We'll do this inside of a cell that we can reference with the name "theCanvasContext". The "context" of a canvas element refers to the data object of the canvas (pixel information, etc.), rather than the canvas element itself. In this case, we're creating a 300 x 300 pixel canvas element with a white background. But most of the logic that you see below has to do with the "drawable" feature of this canvas. We want to be able to click our mouse down on the canvas and drag it to another point and draw a black line from point-to-point. See the cell below "theCanvas" if you want to find out how exactly the black line gets drawn on the canvas.`46:T44b,theCanvas = { const canvas = DOM.canvas(300, 300); // Creates a canvas element const context = canvas.getContext("2d"); context.fillStyle = "#ffffff"; context.fillRect(0, 0, 300, 300); let mouseDown = false; let lastX; let lastY; const handleMousedown = () => { mouseDown = true; }; const handleMouseup = () => { mouseDown = false; [lastX, lastY] = [undefined, undefined]; }; const handleMousemove = (e) => { const rect = e.target.getBoundingClientRect(); const x = e.clientX - rect.left; const y = e.clientY - rect.top; if (mouseDown) { [lastX, lastY] = drawLine(e.target, x, y, lastX, lastY); } }; // The code below is what makes the canvas "drawable" and gives it a nice black border. canvas.addEventListener("mousedown", handleMousedown); canvas.addEventListener("mouseup", handleMouseup); canvas.addEventListener("mousemove", handleMousemove); canvas.style.boxShadow = "-1px -1px 0 black, 1px -1px 0 black, -1px 1px 0 black, 1px 1px 0 black"; canvas.style.margin = "20px"; return canvas; }47:[] 43:{"id":"247f00fe1198fc67","slug":null,"title":"Applied Machine Learning Days: Hands-on deep learning with TensorFlow.js","thumbnail":"88d56e90e6757dc07c746f349582aebea67c2f6886bc263bf698457440fc7102","access_level":"public","create_time":"$D2023-02-01T14:45:25.176Z","update_time":"$D2023-02-01T14:45:25.176Z","view_time":null,"first_public_time":"$D2023-02-01T14:45:25.176Z","has_unpublished_edits":null,"fork":{"id":"01f5a90d714d62e4","slug":"applied-machine-learning-days-hands-on-deep-learning-with-t","title":"Applied Machine Learning Days: Hands-on deep learning with TensorFlow.js","owner":{"id":"b5e9cbf453adc77f","login":"albertowar","name":"Alberto"}},"stars":0,"starred":false,"owner":{"type":"team","id":"33d52a29319889a2","login":"code2life","name":"Joey Yang","avatar_url":"https://avatars-next.observableusercontent.com/268e8d2b0c701e905e13925a6951c59243e50168133cbfefdb93f5c504404dc4","tier":"starter_2024","bio":"Amateur bug maker","following":false},"version":1111,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"97adbe196add972c","login":"code2life","name":"Joey Yang","avatar_url":"https://avatars-next.observableusercontent.com/268e8d2b0c701e905e13925a6951c59243e50168133cbfefdb93f5c504404dc4","tier":null,"bio":"Amateur bug maker","following":false}],"members":[{"type":"individual","id":"97adbe196add972c","login":"code2life","name":"Joey Yang","avatar_url":"https://avatars-next.observableusercontent.com/268e8d2b0c701e905e13925a6951c59243e50168133cbfefdb93f5c504404dc4","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Applied Machine Learning Days: Hands-on deep learning with TensorFlow.js","cells":[{"id":0,"value":"$45","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":509,"value":"$46","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":762,"value":"drawLine = (canvas, x, y, lastX, lastY) => {\n let ctx = canvas.getContext(\"2d\");\n\n ctx.strokeStyle = \"#000000\";\n ctx.lineWidth = 12;\n ctx.lineJoin = \"round\";\n\n ctx.beginPath();\n ctx.moveTo(lastX, lastY);\n ctx.lineTo(x, y);\n ctx.closePath();\n ctx.stroke();\n\n return [x, y];\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":534,"value":"md `#### Go ahead --draw something in the box to test it out.\nBut let's also create a \"clear canvas\" button to go with it, so that we can clear our drawings without needing to reload the page.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":537,"value":"clearCanvas = {\n const clearButton = html ``;\n \n clearButton.onclick = () => {\n theCanvas.getContext(\"2d\").fillRect(0, 0, 300, 300);\n };\n \n return clearButton;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":783,"value":"md `## Import TensorFlow.js\nIncluding external libraries in Observable is done via the \"require\" method.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":786,"value":"tf = require('@tensorflow/tfjs');","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":543,"value":"md `## Loading a model\nTo make this easier to follow, we'll create a function that when called, returns a promise of our desired model. This is an asynchronous function because some models are heavier than others, and async functions won't block-up our program.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":501,"value":"model = await tf\n .loadModel(\"https://gist.githubusercontent.com/albertowar/d34b5fe1f5b463ddcdc46a14159cf2cd/raw/e8757ba1b4457c25865c56b0016f2d67c4a508a0/model.json\");","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":547,"value":"md `The model is now available for us to use. But first we'll have to convert the canvas data into something that we can use with TensorFlow.js. The cell below converts the canvas data into a \"tensor\", which is a data type that we can use to make our prediction.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":740,"value":"tensor = preprocessCanvas(theCanvas);","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":799,"value":"preprocessCanvas = (canvas) => {\n // Preprocess image for the network\n let tensor = tf\n .fromPixels(canvas) // Shape: (300, 300, 3) - RGB image\n .resizeNearestNeighbor([28, 28]) // Shape: (28, 28, 3) - RGB image\n .mean(2) // Shape: (28, 28) - grayscale\n .expandDims(2) // Shape: (28, 28, 1) - network expects 3d values with channels in the last dimension\n .expandDims() // Shape: (1, 28, 28, 1) - network makes predictions for \"batches\" of images\n .toFloat(); // Network works with floating points inputs\n return tensor.div(255.0); // Normalize [0..255] values into [0..1] range\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":752,"value":"md`Now that we have our tensor, we can plug it into TFJS's model.predict function to return an array of predictions. Remember that there are ten labels in our data-set of drawings. The predict function checks the image in our canvas against each of these labels, and gives us the likelihood that each label fits our drawing.` ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":742,"value":"prediction = await model.predict(tensor).data();","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":754,"value":"md`We can quicly return the maximum value of the prediction array using the tf.argMax() function below.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":744,"value":"result = tf.argMax(prediction).data();","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":857,"value":"labels = ['cookie', 'airplane', 'plane', 'strawberry', 'flower', 'axe', 'broom', 'eye', 'bird', 'dog', 'fork', 'ship', 'book', 'candle', 'sheep', 'banana', 'car', 'cat', 'chair', 'donut'];","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":938,"value":"predictedLabel = labels[result];","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":733,"value":"md`## Converting our prediction into a more flashy display\nNow that we have a working prediction, let's turn our data into something more presentable... After all; much of the point of working with TFJS is the ability to take advantage of all of the creative tools that the browser gives us.\n\nBelow is an SVG image that is generated by looking up the actual lable of the drawn canvas image, given the prediction value, and then looping through the letters of the lable to randomly assign a color to them --just for fun!\n\nJust before the SVG though, notice the \"empty\"-looking cell. This cell actually contains some HTML that we are using to import the font that we use in the SVG, \"Leckerli One\". This is a handy trick that can be used to make Observable notebooks a little more customized.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":768,"value":"html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":935,"value":"colors = [\"#f0c24f\", \"#37bbe4\", \"#ec6b2d\", \"#3dbd5d\", \"#f77e5e\", \"#2677bb\", \"#c7943e\", \"#f43530\", \"#0098d8\", \"#dbb700\"];","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":557,"value":"{\n const letters = predictedLabel.toUpperCase().split('').map((letter, i) => {\n const rand = Math.floor(Math.random() * Math.floor(colors.length - 1));\n const fill = colors[rand];\n \n return `${letter}`;\n });\n \n return svg``;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":780,"value":"md`---\n# Tasks:\n* Fix the table below so that it reflects the label and percentage value for each class.\n * Bonus: Color each label using the \"colors\" array that was used in the SVG above.\n* Import your new model, and re-run your prediction. Re-run the prediction a few times to make sure it works well.\n* Render a bar-chart of the \"prediction\" array that is modeled on this one:\n [beta.observablehq.com/@mbostock/d3-bar-chart](https://beta.observablehq.com/@mbostock/d3-bar-chart)\n * Make sure to include the prediciton labels in the histogram.\n * Hint: You just need to do a little data-wrangling to make your data fit the data in Bostock's example.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":814,"value":"md`---\n# Answers`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":874,"value":"html`\n \n ${Array.from(prediction).map((value, i) => html`\n \n \n `)}\n \n

${labels[i]}

${Number((value*100).toFixed(2))}%

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":881,"value":"md`## D3 Bar Chart\nmodeled on [d3-bar-chart by @mbostock]((https://beta.observablehq.com/@mbostock/d3-bar-chart))`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":895,"value":"d3 = require(\"d3@5\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":911,"value":"data = Array.from(prediction).map((value, i) => { return {name: labels[i], value: value, color: colors[Math.floor(Math.random() * Math.floor(colors.length - 1))]}});","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":883,"value":"margin = ({top: 20, right: 0, bottom: 30, left: 40})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":885,"value":"height = 500","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":893,"value":"y = d3.scaleLinear()\n .domain([0, d3.max(data, d => d.value)]).nice()\n .range([height - margin.bottom, margin.top])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":887,"value":"yAxis = g => g\n .style(\"font-family\", \"Leckerli One\") \n .style('font-size', '12px')\n .attr(\"transform\", `translate(${margin.left},0)`)\n .call(d3.axisLeft(y))\n .call(g => g.select(\".domain\").remove())","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":901,"value":"x = d3.scaleBand()\n .domain(data.map(d => d.name))\n .range([margin.left, width - margin.right])\n .padding(0.1)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":904,"value":"xAxis = g => g\n .style(\"font-size\", \"16px\")\n .style(\"font-family\", \"Righteous\")\n .style(\"text-transform\", \"Capitalize\")\n .attr(\"transform\", `translate(0,${height - margin.bottom})`)\n .call(d3.axisBottom(x)\n .tickSizeOuter(0))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":906,"value":"chart = {\n const svg = d3.select(DOM.svg(width, height));\n \n svg.append(\"g\")\n .selectAll(\"rect\").data(data).enter().append(\"rect\")\n .attr(\"fill\", d => d.color) \n .attr(\"x\", d => x(d.name))\n .attr(\"y\", d => y(d.value))\n .attr(\"height\", d => y(0) - y(d.value))\n .attr(\"width\", x.bandwidth());\n \n svg.append(\"g\")\n .call(xAxis);\n \n svg.append(\"g\")\n .call(yAxis);\n \n return svg.node();\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":985,"value":"md`This workshop was given at the [2019 Applied Machine Learning Days](https://www.appliedmldays.org/workshops/hands-on-deep-learning-with-tensorflow-js) in Lausanne, Switzerland.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W47","stdlib":"1"}} 44:"$43:body"