i do science and love art
Published
Edited
Jul 27, 2019
Importers
import {zeros,Mat,fillRandn,assert} from "@steveoni2/utility-function"
g = new Mat(5,6)
md`Let create a Tensor obbject`
Tensor = {
function Tensor(arr,require_grad){
this.item = arr;
this.require_grad = require_grad;
this.gradv = 0;
this.name = "<Tensor>"
}
Tensor.prototype = {
grad: function(g){
this.gradv = g;
}
}
return Tensor
}
md`### Addition Object`
add = {
function add(x,y){
this.x = x;
this.y = y;
this.require_grad=true;
this.item = x.item + y.item
this.gradv = 0;
this.name = "<Add>"
}
add.prototype = {
backward: function(){
if(this.x.require_grad){
this.x.grad(1*this.gradv);
if("backward" in this.x){
this.x.backward()
}
}
if(this.y.require_grad){
this.y.grad(1*this.gradv);
if("backward" in this.y){
this.y.backward()
}
}
},
grad: function(g){
this.gradv = g;
}
}
return add
}
multi = {
function multi(x, y) {
this.item = x.item * y.item;
this.x = x;
this.y = y;
this.gradv = 0;
this.require_grad = true;
this.name = "<Multi>"
}
multi.prototype = {
backward: function () {
if (this.x.require_grad) {
this.x.grad(this.y.item * this.gradv);
if ("backward" in this.x) {
// console.log("True")
this.x.backward()
}
}
if (this.y.require_grad) {
this.y.grad(this.x.item * this.gradv);
if ("backward" in this.y) {
this.y.backward()
}
}
},
grad: function (g) {
this.gradv = g;
}
}
return multi
}
md`Now let implement the goal we want to achieve`
x = new Tensor(-2,true); //click on the object above to see the prop
y = new Tensor(5,true);
z = new Tensor(-4,true);
q = new add(x,y);
f = new multi(q,z);
f.item //this the output
f.grad(1)
f.backward()
f
viewof input1 = html`<input type=range />`
viewof input2 = html`<input type=range />`
viewof input3 = html`<input type=range />`
x2 = new Tensor(input1,true);
y2 = new Tensor(input2,true);
z2 = new Tensor(input3,true);
q2 = new add(x2,y2);
f2 = new multi(q2,z2);
f2.item
viewof grad = html`<input type=range />` //assuming we are recieving gradient flow into the Multid object
{
//vary the grad bar to see the gradient change
f2.grad(grad)
f2.backward()
return f2
}
TensorD = {
function TensorD (n, d, require_grad) {
this.n = n;
this.d = d;
this.out = zeros(n * d);
this.dout = zeros(n * d);
this.require_grad = require_grad;
this.func_name = "<Tensor>"
}
TensorD.prototype = {
get: function (row, col) {
var ix = (this.d * row) + col;
assert(ix >= 0 && ix < this.out.length);
return this.out[ix];
},
set: function (row, col, v) {
var ix = (this.d * row) + col;
assert(ix >= 0 && ix < this.out.length);
this.out[ix] = v;
},
setFrom: function (arr) {
for (var i = 0, n = arr.length; i < n; i++) {
this.out[i] = arr[i];
}
},
randn: function (mu, std) {
fillRandn(this.out, mu, std);
return this;
},
grad: function (grad) {
this.dout = grad;
}
}
return TensorD
}
addD = {
function addD(x, y) {
assert(x.out.length === y.out.length);
this.items = new Mat(1,x.out.length);
for (var i = 0; i < x.out.length; i++) {
this.items.out[i] = x.out[i] + y.out[i];
}
this.x = x;
this.y = y;
this.require_grad = true;
this.out = this.items.out;
this.dout = this.items.dout;
this.n = this.items.n;
this.d = this.items.d;
this.func_name = '<Add>'
}
addD.prototype = {
backward: function () {
if (this.x.require_grad) {
this.x.grad(this.dout);
if ("backward" in this.x) {
this.x.backward()
}
}
if (this.y.require_grad) {
this.y.grad(this.dout);
if ("backward" in this.y) {
this.y.backward()
}
}
},
grad: function (g) {
assert(this.items.dout.length === g.length);
this.dout = g;
}
}
return addD
}
MultiD = {
function MultiD(x, y) {
assert(x.d === y.n, "matmul dimension misaligned");
this.n = x.n;
this.d = y.d;
this.x = x;
this.y = y;
this.require_grad = true;
this.items = new Mat(this.n, this.d);
this.out = this.items.out;
this.dout = this.items.dout
this.func_name = "<Multiply>"
for (var i = 0; i < x.n; i++) {
for (var j = 0; j < y.d; j++) {
var dot = 0.0;
for (var k = 0; k < x.d; k++) {
dot += this.x.out[x.d * i + k] * this.y.out[y.d * k + j];
}
this.out[this.d * i + j] = dot;
}
}
}
MultiD.prototype = {
backward: function () {
if (this.x.require_grad) {
for(var i = 0;i< this.x.n;i++){
for(var j=0;j<this.y.d;j++){
for(var k =0;k<this.x.d;k++){
var b = this.dout[this.y.d*i+j];
// console.log(b);
this.x.dout[this.x.d*i+k] += this.y.out[this.y.d*k+j] * b;
}
}
}
if ("backward" in this.x) {
this.x.backward()
}
}
if (this.y.require_grad) {
for(var i = 0;i< this.x.n;i++){
for(var j=0;j<this.y.d;j++){
for(var k =0;k<this.x.d;k++){
var b = this.dout[this.y.d*i+j];
this.y.dout[this.y.d*k+j] += this.x.out[this.x.d*i+k] * b;
}
}
}
if ("backward" in this.y) {
this.y.backward()
}
}
},
grad: function (g) {
assert(this.dout.length === g.length);
this.dout = g;
}
}
return MultiD
}
a = new TensorD(2,3,true)
a.setFrom([2,3,4,5,6,7])
a.out
b = new TensorD(3,4,true)
b.setFrom([4,5,6,7,8,9,1,2,4,7,1,4])
b.out
mult = new MultiD(a,b)
mult.out //output
`the number of rows for mult is ${mult.n} and column is ${mult.d} and the total length of flatten matrix is ${mult.n * mult.d}`
//let set the grad to one-array with size 8
gradm = new TensorD(2,4,false)
gradm.setFrom([1,1,1,1,1,1,1,1])
{
mult.grad(gradm.out)
mult.backward()
return mult
}
import {Linear as Linear1,ReLU as ReLU1,Softmax as Softmax1} from "@steveoni2/basicautograd" //you can click the link @steveoni2/basicautograd to see d function
md` check if softmax is working`
s = new TensorD(1,6,true)
s.setFrom([2,10,2,4,0.2,7])
softmax = new Softmax1(s)
softmax.out //the output must sum to 1 and the highest value has the highest probability
md`Let backprop through softmax and let see the gradient of s`
softmax.backward(4) //the backward takes in the desired target value, let say ours is index 4
l1 = new Linear1(s,6,10)
l1_r = new ReLU1(l1)
l2 = new Linear1(l1_r,10,2)
pred = new Softmax1(l2)
pred.out
{
pred.backward(1)
return pred
}
import {Linear ,ReLU ,Sequential,Softmax,OptimSGD,Loss} from "@steveoni2/pytorchlike"
model = new Sequential([
new Linear(4,3),
new ReLU(),
new Linear(3,2),
new Softmax()
]);
inp = new TensorD(1,4,true)
inp.setFrom([2,4,5,6])
m = model.forward(inp)
m.out
loss = new Loss(1,m.out)
loss.backward()
m.models[0].dout // the gradient flowing to the first layer
optim = new OptimSGD(model,0.001)
m.models[0].W.out
optim.step()
m.models[0].W.out //the weight is now updated a little bit :)
m.models[0].W.dout
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
