2d:null 2e:["$","title",null,{"children":"Neural Net | delph123 | Observable"}] 33:T475,backprop = (net, delta_net, v, cat) => { let h = [] let x = [v] let o = x; // Feed-forward for (let l = 0; l < net.w.length; l++) { o = compute(x => x, net.w[l], net.b[l], o) h.push(o[0]) o = [o[0].map(net.act)] x.push(o[0]) } // Compute delta from last layer and back let d0 = [] for (let j = 0; j < net.w[net.w.length-1].length; j++) { let s0 = net.cost_deriv(x[net.w.length][j], (j === cat ? 1 : 0)) s0 *= net.act_deriv(h[net.w.length-1][j]) d0.push(s0) } let delta = [d0] for (let l = net.w.length-2; l >= 0; l--) { let d = [] for (let j = 0; j < net.w[l].length; j++) { let s = 0 for (let k = 0; k < net.w[l+1].length; k++) { s += delta[0][k] * net.w[l+1][k][j] } s *= net.act_deriv(h[l][j]) d.push(s) } delta.unshift(d) } // Compute error derivative for (let l = 0; l < net.w.length; l++) { for (let i = 0; i < net.w[l].length; i++) { for (let j = 0; j < net.w[l][i].length; j++) { delta_net.w[l][i][j] += delta[l][i] * x[l][j] } delta_net.b[l][i] += delta[l][i] } } return delta_net }34:T604,COLORS = [ "#000000", "#FFFF00", "#1CE6FF", "#FF34FF", "#FF4A46", "#008941", "#006FA6", "#A30059", "#FFDBE5", "#7A4900", "#0000A6", "#63FFAC", "#B79762", "#004D43", "#8FB0FF", "#997D87", "#5A0007", "#809693", "#FEFFE6", "#1B4400", "#4FC601", "#3B5DFF", "#4A3B53", "#FF2F80", "#61615A", "#BA0900", "#6B7900", "#00C2A0", "#FFAA92", "#FF90C9", "#B903AA", "#D16100", "#DDEFFF", "#000035", "#7B4F4B", "#A1C299", "#300018", "#0AA6D8", "#013349", "#00846F", "#372101", "#FFB500", "#C2FFED", "#A079BF", "#CC0744", "#C0B9B2", "#C2FF99", "#001E09", "#00489C", "#6F0062", "#0CBD66", "#EEC3FF", "#456D75", "#B77B68", "#7A87A1", "#788D66", "#885578", "#FAD09F", "#FF8A9A", "#D157A0", "#BEC459", "#456648", "#0086ED", "#886F4C", "#34362D", "#B4A8BD", "#00A6AA", "#452C2C", "#636375", "#A3C8C9", "#FF913F", "#938A81", "#575329", "#00FECF", "#B05B6F", "#8CD0FF", "#3B9700", "#04F757", "#C8A1A1", "#1E6E00", "#7900D7", "#A77500", "#6367A9", "#A05837", "#6B002C", "#772600", "#D790FF", "#9B9700", "#549E79", "#FFF69F", "#201625", "#72418F", "#BC23FF", "#99ADC0", "#3A2465", "#922329", "#5B4534", "#FDE8DC", "#404E55", "#0089A3", "#CB7E98", "#A4E804", "#324E72", "#6A3A4C", "#83AB58", "#001C1E", "#D1F7CE", "#004B28", "#C8D0F6", "#A3A489", "#806C66", "#222800", "#BF5650", "#E83000", "#66796D", "#DA007C", "#FF1A59", "#8ADBB4", "#1E0200", "#5B4E51", "#C895C5", "#320033", "#FF6832", "#66E1D3", "#CFCDAC", "#D0AC94", "#7ED379", "#012C58" ]35:[] 31:{"id":"fb10bfef056ca067","slug":"neural-net","title":"Neural Net","thumbnail":"2be7206fe32805e569a2b9c8c019b27faf9d156b16c4426071e2995390168df5","access_level":"public","create_time":"$D2018-07-12T17:28:46.683Z","update_time":"$D2019-06-30T22:54:02.320Z","view_time":null,"first_public_time":"$D2018-07-18T11:32:25.500Z","has_unpublished_edits":null,"fork":false,"stars":2,"starred":false,"owner":{"type":"team","id":"c1d5e3641d24c575","login":"delph123","name":"","avatar_url":"https://avatars-next.observableusercontent.com/7238c2c5717e5dd55a9d9f97fa11bc2b688e49b8ceb3c0237060d4680db6b502","tier":"starter_2024"},"version":911,"publish_version":911,"role":null,"authored":false,"authors":[{"type":"individual","id":"eacd402ac07bf979","login":"delph123","name":"","avatar_url":"https://avatars-next.observableusercontent.com/7238c2c5717e5dd55a9d9f97fa11bc2b688e49b8ceb3c0237060d4680db6b502","tier":null}],"members":[{"type":"individual","id":"eacd402ac07bf979","login":"delph123","name":"","avatar_url":"https://avatars-next.observableusercontent.com/7238c2c5717e5dd55a9d9f97fa11bc2b688e49b8ceb3c0237060d4680db6b502","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"Neural Net","cells":[{"id":0,"value":"md`# Neural Net`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":120,"value":"html`\n \n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":68,"value":"DISKS = gen(3, Disk)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"dispatch = refSample.map( x => x.cat ).reduce((a,b) => { a[b] = (a[b] || 0) + 1; return a }, {})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":698,"value":"err = error(feedForward(brain, refSample.map(flatten)), refSample)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":487,"value":"RATE = 1","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":299,"value":"BATCH = 100","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":641,"value":"successfullyCategorized = 100 * correct(refSample) / refSample.length","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":263,"value":"LAYERS = [8,8]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":866,"value":"LOOPS = 1000000","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":290,"value":"brain = {\n let net = makeNet(LAYERS, randWeight, sigmoid, sigmoid_deriv, error_deriv)\n let delta_net = makeNet(LAYERS, () => 0)\n for (let i = 0; i < LOOPS; i++) {\n let points = gen(BATCH, PointCat)\n points.forEach(p => backprop(net, delta_net, flatten(p), p.cat))\n for (let a = 0; a < net.w.length; a++) {\n for (let b = 0; b < net.w[a].length; b++) {\n for (let c = 0; c < net.w[a][b].length; c++) {\n net.w[a][b][c] -= RATE * delta_net.w[a][b][c] / points.length\n delta_net.w[a][b][c] = 0\n }\n net.b[a][b] -= RATE * delta_net.b[a][b] / points.length\n delta_net.b[a][b] = 0\n }\n }\n if (i % 1000 === 0) yield net\n //await pause(200)\n }\n yield net\n return net\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":172,"value":"refSample = gen(5000, PointCat)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":381,"value":"exact = points => points.map(diskCategory)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":183,"value":"guess = points => feedForward(brain, points.map(flatten)).map(argMax)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":638,"value":"correct = points => guess(points).map((c,i) => (c === points[i].cat ? 1 : 0)).reduce((a,b) => a+b, 0)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":244,"value":"compute = (act, w, b, v) => {\n let out = []\n for (let k = 0; k < v.length; k++) {\n let o = []\n for (let i = 0; i < w.length; i++) {\n let s = 0;\n for (let j = 0; j < v[k].length; j++) {\n s += w[i][j] * v[k][j]\n }\n o.push(act(s + b[i]))\n }\n out.push(o)\n }\n return out\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":261,"value":"feedForward = (net, v) => {\n let o = v;\n for (let i = 0; i < net.w.length; i++) {\n o = compute(net.act, net.w[i], net.b[i], o)\n }\n return o;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":489,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":423,"value":"argMax = v => v.reduce((r, s, i) => {\n if (s > r.max) {\n r.max = s\n r.idx = i\n }\n return r\n }, {max: -Infinity, idx: -1}).idx","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":265,"value":"makeNet = (layers, init, act, act_deriv, cost_deriv) => {\n let l = [ flatten(new Point()).length ]\n l.push(...layers)\n l.push(2 ** DISKS.length)\n let net = {\n act: act,\n act_deriv: act_deriv,\n cost_deriv: cost_deriv,\n w: [],\n b: []\n }\n for (let i = 1; i < l.length; i++) {\n let lw = [], lb = []\n for (let j = 0; j < l[i]; j++) {\n let llw = []\n for (let k = 0; k < l[i-1]; k++) {\n llw.push(init())\n }\n lw.push(llw)\n lb.push(init())\n }\n net.w.push(lw)\n net.b.push(lb)\n }\n return net;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":302,"value":"flatten = p => ([p.x/X_MAX, p.y/Y_MAX, p.x * p.y / (X_MAX * Y_MAX), p.x ** 2 / (X_MAX **2), p.y **2 / (Y_MAX ** 2), Math.sin(p.x), Math.sin(p.y)])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":660,"value":"sigmoid = (x) => 1 / (1 + Math.exp(-x))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":671,"value":"sigmoid_deriv = (x) => sigmoid(x) * (1 - sigmoid(x))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":457,"value":"ReLU = x => (x > 0 ? x : 0)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":459,"value":"ReLU_deriv = x => (x > 0 ? 1 : 0)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":445,"value":"error = (actual, expected) => actual.map((v,i) => v.map((s,j) => ((s - (j === expected[i].cat ? 1 : 0)) ** 2))).reduce((sum, v) => sum.map((s,i) => (s + v[i])), actual[0].map(x => 0)).map(s => s/(2 * actual.length))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":465,"value":"error_deriv = (x,y) => x - y","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":15,"value":"function category(disks, p, i) {\n i = i || 0\n if (i < disks.length) {\n let d = dist(disks[i], p)\n return (d < disks[i].r ? 1 : 0) + 2 * category(disks, p, i+1)\n } else {\n return 0\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":404,"value":"diskCategory = p => category(DISKS, p)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":185,"value":"color = cat => COLORS[cat]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":62,"value":"function gen(n, Obj) {\n let a = [];\n for (let i = 0; i < n; i++) {\n a.push(new Obj())\n }\n return a;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":28,"value":"dist = (a, b) => Math.sqrt((a.x - b.x) ** 2 + (a.y - b.y) ** 2)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":50,"value":"rand = (max) => Math.floor(max * Math.random())","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":473,"value":"randWeight = () => (2 * Math.random() - 1)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"Point = function () {\n this.x = rand(X_MAX)\n this.y = rand(Y_MAX)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"Disk = function () {\n Point.apply(this)\n this.r = (dist(ORIGIN, XY_MAX) / 6) + (1 * dist(ORIGIN, XY_MAX) * Math.random() / 6)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":13,"value":"PointCat = function() {\n Point.apply(this)\n this.cat = category(DISKS, this)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":23,"value":"X_MAX = 600","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":26,"value":"Y_MAX = 400","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":41,"value":"XY_MAX = ({x: X_MAX, y: Y_MAX})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":36,"value":"ORIGIN = ({ x: 0, y: 0 })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":230,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":337,"value":"pause = delay => (new Promise(resolve => setTimeout(resolve, delay)))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W35","stdlib":"1","titling":"auto"}} 32:"$31:body"