Public
Edited
May 5, 2023
1 fork
1 star
class Value {
constructor(value, label = "", operator = "", children = [], exponent = 1) {
this.value = value;
this.label = label;
this.operator = operator;
this.gradient = 0;
this.exponent = exponent;
this.children = children;
}
add(other) {
if (typeof other === "number") other = new Value(other);
const newValue = this.value + other.value;
return new Value(newValue, "", "+", [this, other]);
}
sub(other) {
if (typeof other === "number") other = new Value(other);
return this.add(other.mul(-1));
}
mul(other) {
if (typeof other === "number") other = new Value(other);
const newValue = this.value * other.value;
return new Value(newValue, "", "*", [this, other]);
}
div(other) {
if (typeof other === "number") other = new Value(other);
return this.mul(other.pow(-1));
}
neg() {
return this.mul(-1);
}
pow(x) {
const newValue = Math.pow(this.value, x);
this.exponent = x;
return new Value(newValue, "", "^", [this], x);
}
exp() {
const newValue = Math.exp(this.value);
return new Value(newValue, "", "exp", [this]);
}
tanh() {
const newValue = Math.tanh(this.value);
return new Value(newValue, "", "tanh", [this]);
}
backward() {
this.gradient = 1;
let topo = [];
let visited = new Set();
const buildTopo = (v) => {
if (!visited.has(v)) {
visited.add(v);
for (let child of v.children) {
buildTopo(child);
}
topo.push(v);
}
};
buildTopo(this);
for (let node of topo.reverse()) {
node._setChildGradients();
}
}
_setChildGradients() {
const saved = this.children.map(c => ({ v: c.value, g: c.gradient, op: c.operator }));
switch (this.operator) {
case "+": {
const [left, right] = this.children;
left.gradient += this.gradient;
right.gradient += this.gradient;
break;
}
case "*": {
const [left, right] = this.children;
left.gradient += this.gradient * right.value;
right.gradient += this.gradient * left.value;
break;
}
case "^": {
const [c] = this.children;
c.gradient += this.exponent * (Math.pow(c.value, this.exponent - 1)) * this.gradient;
break;
}
case "tanh": {
const [c] = this.children;
c.gradient += this.gradient * (1 - Math.pow(this.value, 2));
break;
}
case "exp": {
const [c] = this.children;
c.gradient += this.gradient * this.value;
break;
}
case "":
break;
default:
throw new Error(`Operator '${this.operator}' not implemented!`);
break;
}
}
}
function preventInfinity(x) {
if (isFinite(x)) return x;
if (x > 0) return Number.MAX_VALUE;
return Number.MIN_VALUE;
}
function visualize(value) {
const children = value.children.map(c => visualize(c)).reduce((curr, prev) => html`${prev}${curr}`, html``);
return html`
<div class="tree">
<div><b>${value.label}</b>(${value.value}, grad = ${value.gradient})</div>
<div class="tree-branch-wrapper">
<div class="operator">${value.operator === "exp" ? "eˣ" : value.operator}${value.exponent !== 1 ? value.exponent : ""}</div>
<div class="tree-branch">
${children}
</div>
</div>
</div>
`;
}
{
const a = new Value(2, "a");
const b = new Value(3, "b");
const c = a.add(b); c.label = "c";
const d = new Value(5, "d");
const e = d.mul(b); e.label = "e";
const f = c.mul(e); f.label = "f";
f.backward();
return visualize(f)
}
class Neuron {
constructor(nin) {
this.w = [];
for (let i = 0; i < nin; i++)
this.w.push(new Value((Math.random() * 2) - 1))
this.b = new Value((Math.random() * 2) - 1);
}
call(x) {
const z = x.map((x, i) => this.w[i].mul(x)).reduce((sum, p) => sum.add(p), new Value(0)).add(this.b);
return z.tanh();
}
parameters() {
return [...this.w, this.b];
}
}
class Layer {
constructor(nin, nout) {
this.neurons = [];
for (let i = 0; i < nout; i++) {
this.neurons.push(new Neuron(nin));
}
}
call(x) {
const outs = this.neurons.map(n => n.call(x));
return outs.length === 1 ? outs[0] : outs;
}
parameters() {
return this.neurons.reduce((p, n) => p.concat(n.parameters()), []);
}
}
class MLP {
constructor(nin, nouts) {
const sizes = [nin].concat(nouts);
this.layers = [];
for (let i = 0; i < nouts.length; i++) {
this.layers.push(new Layer(sizes[i], sizes[i + 1]));
}
}
call(x) {
for (const layer of this.layers) {
x = layer.call(x);
}
return x;
}
parameters() {
return this.layers.reduce((p, n) => p.concat(n.parameters()), []);
}
}
{
const n = new MLP(3, [4, 4, 1]);
const xs = [
[2.0, 3.0, -1.0],
[3.0, -1.0, 0.5],
[0.5, 1.0, 1.0],
[1.0, 1.0, -1.0],
];
const ys = [1.0, -1.0, -1.0, 1.0];
let ypred;
let loss;
const iterations = 1000;
const learningRate = 0.1;
for (let k = 0; k < iterations; k++) {
// forward pass
ypred = xs.map(x => n.call(x));
loss = ys.reduce((acc, ygt, index) => {
const yout = ypred[index];
return yout.sub(ygt).pow(2).add(acc);
}, 0);
// backward pass
for (const p of n.parameters()) {
p.gradient = 0;
}
loss.backward();
// learning
for (const p of n.parameters()) {
p.value -= p.gradient * learningRate;
}
}
return { loss, ypred };
}
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.