class Neuron {
// `nin` = number of inputs to a neuron
constructor(nin) {
// create a weight for each input
this.w = util.array(nin, () => new Value(util.randomUniform(-1, 1)).set({ label: 'weight', color: Colors.weights }));
// bias which controls overall "trigger happiness" for this neuron
this.b = new Value(util.randomUniform(-1, 1)).set({ label: 'bias', color: Colors.bias });
}
// we could implement a callable "class" like the video in JS, but this
// is simpler
_call = x => {
/**
* `w * x + b`, where `w * x` is a dot-product (multiply each weight
* by each input, pairwise).
*
* WARNING: we need to ensure that we're using the Value class methods
* for arithmetic operations bc those methods produce internal mutations
* which build the internal graph
*/
const activation = util.zip(this.w, x).reduce((sum, [wi, _xi]) => {
// only converting to a Value here so we can add label & color
const xi = typeof _xi === 'number'
? new Value(_xi).set({ label: 'input', color: Colors.input })
: _xi;
return sum.add(wi.mult(xi));
}, this.b);
// pass through non-linearity tanh
const out = activation.tanh().set({ label: 'act', color: Colors.activation });
return out;
}
// added for `Minimizing Loss` section below
parameters = () => [...this.w, this.b]
}
class Layer {
// `nin` = number of inputs per neuron
// `nout` = number of output neurons in this layer
constructor(nin, nout) {
this.neurons = util.array(nout, () => new Neuron(nin))
}
_call = x => {
const outs = this.neurons.map(n => n._call(x));
return outs.length === 1 ? outs[0] : outs;
}
// added for `Minimizing Loss` section below
parameters = () => this.neurons.flatMap(n => n.parameters())
}
class MLP {
// `nouts` = list of output neuron lengths; i.e., the sizes of the layers in the MLP
// where each element of the array represents a layer, the number representing
// the number of neurons in the layer
constructor(nin, nouts) {
const sz = [nin, ...nouts];
this.layers = util.array(nouts.length, i => new Layer(sz[i], sz[i + 1]));
}
_call = x => {
// call each layer sequentially
return this.layers.reduce((_x, layer) => layer._call(_x), x);
}
// added for `Minimizing Loss` section below
parameters = () => this.layers.flatMap(layer => layer.parameters())
}
// test mlp
{
const x = [2.0, 3.0, -1.0];
const mlp = new MLP(3, [4, 4, 1]);
const out = mlp._call(x);
return digraph(util.trace(out));
}
loss = {
const mlp = new MLP(3, [4, 4, 1]);
// four examples; four possible inputs into the neural net
const xs = [
[2.0, 3.0, -1.0],
[3.0, -1.0, 0.5],
[0.5, 1.0, 1.0],
[1.0, 1.0, -1.0]
];
// desired target; one NN output per input example
const ys = [1.0, -1.0, -1.0, 1.0];
// these are the actual outputs, which differ from the desired targets:
const yPredictions = xs.map(x => mlp._call(x));
// how do we tune the NN/weights above to better predict the desired targets?
// in deep learning, we calculate a single number to measure the total performance
// of your NN: the "loss" (we'll do "mean-squared" loss).
// instead of squaring, could also do abs value (need to get rid of the sign)
// "mean-square loss" = (groundTruth - output) ** 2
// greater loss = further away from desired target
const losses = util.zip(ys, yPredictions).map(([gt, output]) => output.sub(gt).pow(2));
const sumLoss = losses.reduce((sum, loss) => sum.add(loss), new Value(0.0));
return { mlp, xs, ys, yPredictions, losses, sumLoss };
}
// four examples; four possible inputs into the neural net
xs = [
[2.0, 3.0, -1.0], // in plots below, this is 0
[3.0, -1.0, 0.5], // 1
[0.5, 1.0, 1.0], // 2
[1.0, 1.0, -1.0] // 3
]
// desired target; one NN output per input example
ys = [1.0, -1.0, -1.0, 1.0]
trainingLoop = (mlp, stepSize = 0.05) => {
// forward pass
const yPredictions = xs.map(x => mlp._call(x))
const losses = util.zip(ys, yPredictions).map(([gt, output]) => output.sub(gt).pow(2));
const sumLoss = losses.reduce((sum, loss) => sum.add(loss), new Value(0.0));
// backward pass
for (const p of mlp.parameters()) {
// reset grad to zero to prevent building up grad values
p.grad = 0.0;
}
sumLoss.backward();
// we do not want to modify the inputs (`xs`) or the targets (`ys`) bc they are fixed.
// instead, we want to modify the weights and biases.
for (const p of mlp.parameters()) {
// change param datum slightly according to gradient info; tiny update in gradient
// descent scheme. think of gradient as a vector pointing in direction of increased
// loss.
// modify datum by small step size in the direction of the gradient. we want to
// minimize the loss, not maximize it.
p.datum += -stepSize * p.grad;
}
return { predictions: yPredictions, sum: sumLoss, losses };
}
// this shows us the effect of the first weight in the first neuron in the first
// layer of the MLP. for example, if the gradient is negative, that means its
// influence on loss is negative (increasing this weight would make the loss go down).
// WARNING uncommenting and running this cell will cause weird behavior in the notebook.
// {
// loss.sumLoss.backward();
// return Inputs.table([
// {
// datum: loss.mlp.layers[0].neurons[0].w[0].datum.toFixed(10),
// grad: loss.mlp.layers[0].neurons[0].w[0].grad.toFixed(10)
// }
// ]);
// }
util = ({
..._util,
// not a uniform distribution but may fix later
randomUniform: (min, max) => Math.random() * (max - min) + min,
zip: (a, b) => a.map((x, i) => [x, b[i]]),
array: (size, callback) => Array(size).fill(null).map((_, i) => callback(i)),
sum: (arr, accessor, start = 0) => arr.reduce((sum, item, i) => sum + (accessor ? accessor(item, i, arr) : item), start)
})
Colors = ({
weights: 'darkorange',
bias: 'greenyellow',
activation: 'plum',
input: 'deepskyblue'
})
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
