2d:null 2e:["$","title",null,{"children":"Vanishing Gradients | Kris Sankaran | Observable"}] 33:T177a,vanishing_grads = { const svg = d3.select(DOM.svg(width, f3.height)); var scales = f3.generate_scales(), x = math.zeros([0, 2]); f3.setup_background(svg, scales); var w = math.matrix([[1, 1], [-1, 1]]), u = math.matrix([[1, 1], [-1, 1]]), z = math.zeros([0, 2]), r = math.zeros([0, 2]), all_data = f3.generate_data(u, w, x, z, r); // draw the points svg.select("#points") .selectAll("circle") .data(all_data["points"]).enter() .append("circle") .attrs({ "cx": (d) => scales.h0(d.h._data[0]), "cy": (d) => scales.h1(d.h._data[1]), "fill": (d) => scales.step(d.step), "fill-opacity": (d) => scales.step_alpha(x.length - d.step), "r": 3 }); svg.append("rect") .attrs({ "x": 0, "y": 0, "width": 200, "height": 200, "fill-opacity": 0 }) .on("click", reset); // input x's background and selector svg.select("#x_background") .append("rect") .attrs({ "x": scales.x0.range()[0], "y": scales.x1.range()[1], "width": scales.x0.range()[1] - scales.x0.range()[0], "height": scales.x1.range()[0] - scales.x1.range()[1], "fill": "#F7F7F7" }) .on("click", add_input); svg.select("#weights_select") .selectAll("circle") .data(f3.rows(w, u)).enter() .append("circle") .attrs({ "cx": (d) => scales.wselect_x(d.value._data[0][0]) + scales.wtype(d.weight), "cy": (d) => scales.wselect_y(d.value._data[0][1]), "r": 7, "fill": (d) => scales.weight_fill(d.weight) }) .call(d3.drag().on("drag", (d) => update_weights(d))); redraw(); function add_input() { var pos = d3.mouse(this) x.push([scales.x0.invert(pos[0]), scales.x1.invert(pos[1])]) z.push([.9, .9]); r.push([.95, .95]); redraw() } function reset() { x = math.zeros([0, 2]), z = math.zeros([0, 2]), r = math.zeros([0, 2]); redraw(); } function update_weights(d) { var cur = [scales.wselect_x.invert(d3.event.x - scales.wtype(d.weight)), scales.wselect_y.invert(d3.event.y)]; if (cur[0] < scales.wselect_x.domain()[0] || cur[0] > scales.wselect_x.domain()[1] || cur[1] < scales.wselect_y.domain()[0] || cur[1] > scales.wselect_y.domain()[1]) { return; } if (d.weight == "w") { w.subset(math.index(d.row, [0, 1]), cur); } else { u.subset(math.index(d.row, [0, 1]), cur); } redraw() } function update_gates(d) { var pos = [d3.event.x, d3.event.y], pos = [scales.zx.invert(pos[0]), scales.zy.invert(pos[1])], new_pos = [Math.round(pos[0] * 50) / 50, Math.round(pos[1] * 50) / 50]; // snap to grid if (new_pos == pos) return; pos = new_pos if (pos[0] < scales.zx.domain()[0] || pos[0] > scales.zx.domain()[1] || pos[1] < scales.zy.domain()[0] || pos[1] > scales.zy.domain()[1]) { return; } if (d.weight == "z") { z[d.step] = pos } else { r[d.step] = pos } redraw() } function update_figure(points, links) { svg.select("#links") .selectAll("path") .data(links).enter() .append("path"); svg.select("#links") .selectAll("path") .attrs({"d": f3.line_fun()}); svg.select("#points") .selectAll("circle") .data(points).enter() .append("circle"); svg.select("#points") .selectAll("circle") .data(points).exit() .remove() svg.select("#points") .selectAll("circle") .data(points) .attrs({ "cx": (d) => scales.h0(d.h._data[0]), "cy": (d) => scales.h1(d.h._data[1]), "fill": (d) => scales.step(d.step), "fill-opacity": (d) => scales.step_alpha(x.length - d.step), "r": 3 }); svg.select("#x_input") .selectAll("circle") .data(x).enter() .append("circle") .call(d3.drag().on("drag", (d, i) => { x = f3.update_x(d, i, x); redraw(); })); svg.select("#x_input") .selectAll("circle") .data(x).exit() .remove(); svg.select("#x_links") .select("path") .data([x]) .attrs({ "d": d3.line().x((d) => scales.x0(d[0])).y((d) => scales.x1(d[1])), "stroke-width": 0.3, "fill": "none" }) svg.select("#x_input") .selectAll("circle") .attrs({ "cx": (d) => scales.x0(d[0]), "cy": (d) => scales.x1(d[1]), "r": 4 }); svg.select("#weights_select") .selectAll("circle") .data(f3.rows(w, u)) .attrs({ "cx": (d) => scales.wselect_x(d.value._data[0][0]) + scales.wtype(d.weight), "cy": (d) => scales.wselect_y(d.value._data[0][1]) }); // draw the gates var pts = f3.gates(z, r) svg.select("#gates") .selectAll("circle") .data(pts["points"]).enter() .append("circle") .call(d3.drag().on("drag", (d) => update_gates(d))); svg.select("#gates") .selectAll("circle") .data(pts["points"]).exit() .remove(); svg.select("#gates") .selectAll("circle") .attrs({ "cx": (d) => scales.zx(d.value[0]), "cy": (d) => scales.zy(d.value[1]), "fill": (d) => scales.ztype(d.weight), "r": 7 }); svg.select("#gates #r") .select("path") .data([pts["links"]["r"]]) .attrs({ "d": d3.line().x((d) => scales.zx(d[0])).y((d) => scales.zy(d[1])) }); svg.select("#gates #z") .select("path") .data([pts["links"]["z"]]) .attrs({ "d": d3.line().x((d) => scales.zx(d[0])).y((d) => scales.zy(d[1])), "fill": "none" }); } function redraw() { var all_data = f3.generate_data(u, w, x, z, r); update_figure(all_data["points"], all_data["links"]); } return svg.node() }34:T1a9a,f3 = { var height = 400, scales = generate_scales(); function generate_data(w, u, x, z, r) { var f = rnn_gen(u, w, math.zeros(2)), seq = d3.range(-1, 1, .39), h = []; for (var i = 0; i < seq.length; i++) { for (var j = 0; j < seq.length; j++) { var cur_h = [math.matrix([seq[i], seq[j]])]; for (var step = 0; step < x.length; step++) { cur_h.push(f(x[step], cur_h[cur_h.length - 1], z[step], r[step])); } h.push(cur_h) } } var points = []; for (var i = 0; i < h.length; i++) { for (var step = 0; step < h[i].length; step++) { points.push({"h": h[i][step], "step": step}); } } return {"links": h, "points": points} } function generate_scales() { var width0 = 0.85 * width return { "h0": d3.scaleLinear().domain([-1, 1]).range([0, .95 * 3 * width0 / 4]), "h1": d3.scaleLinear().domain([-1, 1]).range([height, 0]), "x0": d3.scaleLinear().domain([-1.2, 1.2]).range([3 * width0 / 4, width0]), "x1": d3.scaleLinear().domain([-1.2, 1.2]).range([height / 2, 0]), "zx": d3.scaleLinear().domain([0, 1]).range([3 * width0 / 4, width0]), "zy": d3.scaleLinear().domain([0, 1]).range([.95 * height, 3 * height / 4]), "step": d3.scaleLinear().domain([0, 5]).range(["#ff9966 ", "#9955bb"]), "step_alpha": d3.scaleLinear().domain([0, 5, 1000]).range([1, 0.4, 0.2]), "wselect_x": d3.scaleLinear().domain([-3, 3]).range([0, width0 / 8]), "wselect_y": d3.scaleLinear().domain([-3, 3]).range([.95 * 3 * height / 4, 1.05 * height / 2]), "weight_opacity": d3.scaleLinear().domain([0, 3]).range([0.4, 1]), "row_offset": d3.scaleOrdinal().domain([0, 1]).range([3 * height / 4, 7 * height / 8]), "col_offset": d3.scaleOrdinal().domain([0, 1]).range([0, width0 / 16]), "weight_fill": d3.scaleOrdinal().domain(["w", "u", "x", "z", "r"]).range(["#298eab", "#29ab87", "#ab294d", "#ffba00", "#ffa6c9"]), "wtype": d3.scaleOrdinal().domain(["w", "u"]).range([3 * width0 / 4, 7 * width0 / 8]), "ztype": d3.scaleOrdinal().domain(["z", "r"]).range(["#ffba00", "#ffa6c9"]) }; } function rnn_gen(w, u, b) { function rnn(x, h, z, r) { var h_tilde = (math.add(math.multiply(w, x), math.multiply(u, math.dotMultiply(r, h)))).map(Math.tanh) return math.add(math.dotMultiply(math.subtract(1, z), h), math.dotMultiply(z, h_tilde)) } return rnn } function rows(w, u) { var weight_rows = []; for (var i = 0; i < w._size[0]; i++) { weight_rows.push({"value": math.row(w, i), "weight": "w", "row": i}) weight_rows.push({"value": math.row(u, i), "weight": "u", "row": i}) } return weight_rows; } function gates(z, r) { var points = [], links = {"z": [], "r": []}; for (var step = 0; step < z.length; step++) { points.push({"value": z[step], "weight": "z", "step": step}); points.push({"value": r[step], "weight": "r", "step": step}); links["z"].push(z[step]); links["r"].push(r[step]); } return {"points": points, "links": links}; } function line_fun(scale) { return d3.line() .x((d) => scales.h0(d._data[0])) .y((d) => scales.h1(d._data[1])); } function update_x(d, i, x) { if (d3.event.x > width || d3.event.x < scales.x0.range()[0] || d3.event.y > scales.x1.range()[0] || d3.event.y < 0) { return; } var pos = [d3.event.x, d3.event.y], pos = [scales.x0.invert(pos[0]), scales.x1.invert(pos[1])], new_pos = [Math.round(pos[0] * 50) / 50, Math.round(pos[1] * 50) / 50]; // snap to grid if (new_pos == pos) return; pos = new_pos x[i] = pos; return x; } function setup_background(svg, scales) { svg.append("g").attr("id", "x_background"); var g_elems = ["links", "points", "weights_select", "gates", "x_background", "x_links", "x_input", "labels"]; draw_axes(svg, scales); svg.selectAll("g") .data(g_elems, (d) => d).enter() .append("g") .attr("id", (d) => d); svg.select("#x_links").append("path") svg.select("#gates").append("g").attr("id", "r") svg.select("#gates").append("g").attr("id", "z") svg.select("#gates #r").append("path") svg.select("#gates #z").append("path") // add some text labels svg.select("#labels") .selectAll("text") .data([{"text": "W", "weight": "w"}, {"text": "U", "weight": "u"}, {"text": "x[t]", "weight": "x"}, {"text": "z[t]", "weight": "z"}, {"text": "r[t],", "weight": "r"}, ]).enter() .append("text") .attrs({ "x": (d) => scales.wtype(d.weight) + 10, "y": (d) => d.weight == "x" ? 20 : (d.weight == "w" || d.weight == "u" ? scales.wselect_y.range()[1] + 10 : scales.zy.range()[1] + 10), "fill": (d) => scales.weight_fill(d.weight) }) .text((d) => d.text); } function draw_axes(svg, scales) { svg.append("g") .attr("transform", "translate(" + scales.h0(0) + ",0)") .call(d3.axisLeft().scale(scales.h1)) svg.append("g") .attr("transform", "translate(0," + scales.h1(0) + ")") .call(d3.axisBottom().scale(scales.h0)) svg.append("g") .attr("transform", "translate(" + scales.x0(0) + ",0)") .call(d3.axisLeft().scale(scales.x1).ticks(4)) svg.append("g") .attr("transform", "translate(0," + scales.x1(0) + ")") .call(d3.axisBottom().scale(scales.x0).ticks(4)) svg.append("g") .attr("transform", "translate(" + (scales.wtype("u") + scales.wselect_x(0)) + ",0)") .call(d3.axisLeft().scale(scales.wselect_y).ticks(4)) svg.append("g") .attr("transform", "translate(" + scales.wtype("u") + "," + scales.wselect_y(0) + ")") .call(d3.axisBottom().scale(scales.wselect_x).ticks(4)) svg.append("g") .attr("transform", "translate(" + (scales.wtype("w") + scales.wselect_x(0)) + ",0)") .call(d3.axisLeft().scale(scales.wselect_y).ticks(4)) svg.append("g") .attr("transform", "translate(" + scales.wtype("w") + "," + scales.wselect_y(0) + ")") .call(d3.axisBottom().scale(scales.wselect_x).ticks(4)) svg.append("g") .attr("transform", "translate(" + scales.zx(0) + ",0)") .call(d3.axisLeft().scale(scales.zy).ticks(4)) svg.append("g") .attr("transform", "translate(0," + scales.zy(0) + ")") .call(d3.axisBottom().scale(scales.zx).ticks(4)) } return { "height": height, "generate_scales": generate_scales, "setup_background": setup_background, "generate_data": generate_data, "rows": rows, "gates": gates, "line_fun": line_fun, "update_x": update_x } }35:[] 31:{"id":"7315ee562cd03844","slug":"vanishing-gradients","title":"Vanishing Gradients","thumbnail":"87e16fc98c22c30605e5fed00468bfc3cf4f103b7e439fc8a485dc050f77aa0b","access_level":"public","create_time":"$D2019-08-23T16:06:21.069Z","update_time":"$D2019-10-21T01:32:53.848Z","view_time":null,"first_public_time":"$D2019-08-23T19:05:40.376Z","fork":true,"stars":1,"starred":false,"owner":{"type":"team","id":"58c635fe0c6a930c","login":"krisrs1128","name":"Kris Sankaran","avatar_url":"https://avatars-next.observableusercontent.com/384a4d4ac6e6ad0ce3c5b1bc69c03d2c693a0b2859424bd8395dfef0e462967e","tier":"starter_2024"},"version":492,"publish_version":492,"role":null,"authored":false,"authors":[{"type":"individual","id":"544fdf34bf62b87c","login":"krisrs1128","name":"Kris Sankaran","avatar_url":"https://avatars-next.observableusercontent.com/384a4d4ac6e6ad0ce3c5b1bc69c03d2c693a0b2859424bd8395dfef0e462967e","tier":null}],"members":[{"type":"individual","id":"544fdf34bf62b87c","login":"krisrs1128","name":"Kris Sankaran","avatar_url":"https://avatars-next.observableusercontent.com/384a4d4ac6e6ad0ce3c5b1bc69c03d2c693a0b2859424bd8395dfef0e462967e","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"Vanishing Gradients","cells":[{"id":118,"value":"md`# Vanishing Gradients`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":435,"value":"md`Click on the grey square to introduce new inputs. Control the amount of gatings using the orange and pink points in the bottom right. Notice how the summaries h[t] quickly saturate without some amount of gating. Click near the top left to reset.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":0,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":312,"value":"html`\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"math = require(\"https://cdnjs.cloudflare.com/ajax/libs/mathjs/6.0.3/math.min.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":6,"value":"d3 = require(\"d3-selection\", \"d3-selection-multi\", \"d3-scale\", \"d3-array\", \"d3-fetch\", \"d3-axis\", \"d3-drag\", \"d3-shape\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":8,"value":"import {slider} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W35","stdlib":"1","titling":"auto"}} 32:"$31:body"