{
// let status = [];
for (let k = 0; k < iterations; k++) {
// forward
const { totalLoss, accuracy } = loss();
// backward
model.zeroGrad();
totalLoss.backward();
// update (sgd)
const learningRate = 1.0 - 0.9 * k / iterations;
for (let p of model.parameters()) {
p.datum -= learningRate * p.grad;
}
if (k % 1 === 0) {
yield `step ${k + 1}/${iterations} loss ${totalLoss.datum}, accuracy ${accuracy}`;
// status.push({ iteration: k, loss: totalLoss.datum, accuracy });
// yield Plot.plot({
// x: { domain: [0, iterations] },
// marks: [
// Plot.line(status, { x: 'iteration', y: 'loss' })
// ]
// })
}
}
mutable done = true;
}
mutable model = new MLP(2, [16, 16, 1])
loss = () => {
const inputs = data.map(([x, y]) => [new Value(x), new Value(y)]);
const scores = inputs.map(input => model._call(input));
const yb = data.map(([,,y]) => y);
const losses = util.zip(yb, scores).map(([yi, scorei]) => scorei.mult(-yi).add(1).relu());
const dataLoss = losses.reduce((sum, v) => sum.add(v), new Value(0.0)).mult(1.0 / losses.length);
const alpha = 1e-4;
const regLoss = model.parameters().reduce((sum, p) => sum.add(p.mult(p)), new Value(0.0)).mult(alpha);
const totalLoss = dataLoss.add(regLoss);
const accuracy = util.zip(yb, scores).map(([yi, scorei]) => +(yi > 0 === scorei.datum > 0));
return { totalLoss, accuracy: accuracy.reduce((sum, v) => sum + v, 0) / accuracy.length }
}
class Neuron extends Module {
constructor(nin, nonlin = true) {
super();
this.w = util.array(nin, () => new Value(util.randomUniform(-1, 1)));
this.b = new Value(0.0);
this.nonlin = nonlin;
}
_call = x => {
const activation = util.zip(this.w, x).reduce((sum, [wi, xi]) => sum.add(wi.mult(xi)), this.b);
//return this.nonlin ? activation.relu() : activation;
if (this.nonlin) {
return activation.relu()
}
return activation;
}
parameters = () => [...this.w, this.b]
}
class Layer extends Module {
constructor(nin, nout, nonlin) {
super();
this.neurons = util.array(nout, () => new Neuron(nin, nonlin))
}
_call = x => {
const outs = this.neurons.map(n => n._call(x));
return outs.length === 1 ? outs[0] : outs;
}
parameters = () => this.neurons.flatMap(n => n.parameters())
}
class MLP extends Module {
constructor(nin, nouts) {
super();
const sz = [nin, ...nouts];
this.layers = util.array(
nouts.length,
// last neuron is the output and isn't passed through the activation function
i => new Layer(sz[i], sz[i + 1], i !== (nouts.length - 1))
);
}
_call = x => {
return this.layers.reduce((_x, layer) => layer._call(_x), x);
}
parameters = () => this.layers.flatMap(layer => layer.parameters());
}
class Module {
zeroGrad = () => {
for (let p of this.parameters()) {
p.grad = 0.0;
}
}
parameters = () => []
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
