Notebooks 2.0 is here.
viewof datasetFile = Inputs.file({ label: "KNN Dataset File", accept: ".json", required: false })
p5(sketch => {
let features
let knn
let video
let leftButton
let rightButton
let upButton
let downButton
let stopButton
let saveButton
let loadButton
let keyboardButton
let label = "waiting for data"
let classificationTask
let showKeyboard = false
let sentence = "____________"
let characterSet = "_abcdefghijklmnopqrstuvwxyz"
let force = sketch.createVector(0, 0)
let velocity = sketch.createVector(0, 0)
let position = sketch.createVector(0, 0)
let mass = 0.5
let damping = 0.95
function drawChar(char, row, col, bgFill = "#000000", fgFill = "#FFFFFF") {
const x = col * 50 + 10
const y = row * 60
sketch.fill(bgFill)
sketch.rect(x, y, 40, 50)
sketch.fill(fgFill)
sketch.textSize(40)
sketch.text(char.toUpperCase(), x + 5, y + 40)
}
function drawKeyboard() {
const selectedCol = Math.trunc(position.x)
const selectedChar = sentence[selectedCol]
const sentenceYOffset = 5
const charsetYOffset = -characterSet.indexOf(selectedChar)
// Draw character set if moving vertically
for (let row = 0; row < characterSet.length; row++) {
const char = characterSet[row]
drawChar(char, row + sentenceYOffset + charsetYOffset, selectedCol, "rgba(0,0,0,0)", "#FFFFFF")
}
// Draw sentence
for (let col = 0; col < sentence.length; col++) {
const bgFill = selectedCol === col ? "#FF0000" : "#000000"
const char = sentence[col]
drawChar(char, sentenceYOffset, col, bgFill)
}
}
function moveCursor() {
const prevX = Math.trunc(position.x)
const prevY = Math.trunc(position.y)
force.set(0, 0)
if (label === 'right')
force.add(1, 0)
if (label === 'left')
force.add(-1, 0)
if (label === 'up')
force.add(0, -1)
if (label === 'down')
force.add(0, 1)
if (label === 'stop')
velocity.set(0, 0)
force.div(mass)
velocity.add(force)
velocity.mult(1 - damping)
position.add(velocity)
position.x = sketch.constrain(position.x, 0, sentence.length - 1)
position.y = sketch.constrain(position.y, 0, characterSet.length - 1)
const newX = Math.trunc(position.x)
const newY = Math.trunc(position.y)
// Switched columns: clear any vertical velocity
if (prevX !== newX) {
velocity.y = 0
}
// Switched rows: clear horizontal velocity and update the sentence
if (prevY !== newY) {
velocity.x = 0
sentence = sentence.substring(0, newX) + characterSet[newY] + sentence.substring(newX + 1);
}
}
function extractLabelFromResults(results) {
// In an ideal world I could simple use `results.label` as the value of the label.
// Unfortunately, when you save + load a KNN model the labels are a bit fudged so
// you need to do some manual sort (like we are doing below) to retrieve the label
// with the highest confidence level.
const pairs = Object.keys(results.confidencesByLabel).map(k => [k, results.confidencesByLabel[k]])
pairs.sort((a, b) => b[1] - a[1])
return pairs[0][0]
}
function beginClassifying() {
if (classificationTask) return
classificationTask = new Promise((resolve, reject) => {
const logits = features.infer(video)
knn.classify(logits, (error, results) => {
if (error) reject(error)
else resolve(results)
})
}).then(results => {
label = extractLabelFromResults(results)
}).catch(error => {
console.warn(error)
}).finally(() => {
classificationTask = null
})
}
sketch.setup = function() {
sketch.createCanvas(640, 480)
sketch.background(0)
video = sketch.createCapture(sketch.VIDEO)
video.hide()
features = ml5.featureExtractor('MobileNet')
knn = ml5.KNNClassifier()
leftButton = sketch.createButton("⬅️");
leftButton.mousePressed(() => {
const logits = features.infer(video)
knn.addExample(logits, 'left')
})
rightButton = sketch.createButton("➡️");
rightButton.mousePressed(() => {
const logits = features.infer(video)
knn.addExample(logits, 'right')
})
upButton = sketch.createButton("⬆️");
upButton.mousePressed(() => {
const logits = features.infer(video)
knn.addExample(logits, 'up')
})
downButton = sketch.createButton("⬇️");
downButton.mousePressed(() => {
const logits = features.infer(video)
knn.addExample(logits, 'down')
})
stopButton = sketch.createButton("⏹️");
stopButton.mousePressed(() => {
const logits = features.infer(video)
knn.addExample(logits, 'stop')
})
saveButton = sketch.createButton("Save")
saveButton.mousePressed(() => knn.save())
loadButton = sketch.createButton("Load")
loadButton.mousePressed(() => {
datasetFile.url().then(url => knn.load(url))
})
keyboardButton = sketch.createButton("⌨️ ")
keyboardButton.mousePressed(() => { showKeyboard = !showKeyboard })
};
sketch.draw = function() {
// Classify video
if (knn.getNumLabels() > 0)
beginClassifying()
// Draw video
sketch.push()
sketch.translate(sketch.width, 0)
sketch.scale(-1,1)
sketch.image(video, 0, 0)
sketch.pop()
// Draw keyboard
if (showKeyboard) {
moveCursor()
drawKeyboard()
}
// Draw label
sketch.fill(0)
sketch.rect(0, 0, 640, 30)
sketch.fill(255)
sketch.textSize(24)
sketch.text("Label: " + label, 10, 24)
}
})
import { p5 } from "@bberak/p5"
ml5 = require("https://unpkg.com/ml5@0.12.2/dist/ml5.min.js")
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
