23:null 24:["$","title",null,{"children":"Untitled | Kris Sankaran | Observable"}] 29:T4c1,// Create a slide: title = { const svg = d3.select(DOM.svg(width, width / 2)); svg.append("rect") .attrs({ "id": "title_rect", "height": width / 2, "width": width, "fill": "none" }); svg.append("text") .attrs({ "transform": "translate(" + .23 * width + "," + .1 * width + ")", "font-family": "Muli", "font-size": "32px", "fill": "#191919" }) .text("Attention and Memory in Deep Learning") svg.append("text") .attrs({ "transform": "translate(" + .23 * width + "," + .15 * width + ")", "font-family": "Red Hat Display", "font-size": "28px", "fill": "#5BAFC1" }) .text("Foundations, Examples, and Frontiers") svg.append("text") .attrs({ "transform": "translate(" + .23 * width + "," + .48 * width + ")", "font-size": "23px" }) .text("Kris Sankaran & Aidan Gomez") svg.append("image") .attrs({ "xlink:href": "https://pbs.twimg.com/profile_images/1025731558458761216/fEbRJ1YE_400x400.jpg", "transform": "translate(" + .4 * width + "," + .18 * width + ")", "width": width / 4, "height": width / 4 }) return svg.node(); }2a:T42b,slide`#### References [1] Deisenroth, M. P., Faisal, A. A., & Ong, C. S. (2018). [Mathematics for Machine Learning](https://www.doc.ic.ac.uk/~mpd37/teaching/2017/496/notes.pdf). [2] Graves, Alex. Supervised sequence labelling. In : Supervised sequence labelling with recurrent neural networks. Springer, Berlin, Heidelberg, 2012. p. 5-13. [3] Chung, J., Gulcehre, C., Cho, K., & Bengio, Y. (2014). Empirical evaluation of gated recurrent neural networks on sequence modeling. arXiv preprint arXiv:1412.3555. [4] Katharopoulos, A., & Fleuret, F. (2019). Processing Megapixel Images with Deep Attention-Sampling Models. arXiv preprint arXiv:1905.03711. [5] Mnih, V., Heess, N., & Graves, A. (2014). Recurrent models of visual attention. In Advances in neural information processing systems (pp. 2204-2212). [6] Mohamed, Shakir. (2018) [Planting the Seeds of Probabilistic Thinking](https://www.shakirm.com/slides/MLSS2018-Madrid-ProbThinking.pdf). [7] Tallec, C., & Ollivier, Y. (2018). Can recurrent neural networks warp time?. arXiv preprint arXiv:1804.11188. `2b:T4f4,fig1 = function() { const svg = d3.select(DOM.svg(width, f1.height)); svg.selectAll("g") .data(["background", "counter", "range", "series"]).enter() .append("g") .attr("id", (d) => d); var counts = f1.entry_counts(f1.x), scales = f1.generate_scales(counts); svg.select("#background") .append("rect") .classed("active", true) .attrs({ "width": scales.t.range()[1], "height": 200, "fill": "#F7F7F7" }) .on("mousemove", function() { f1.click_fun(svg, this) }); svg.select("#range") .append("rect") .attrs({ "x": 0, "y": scales.x(1), "width": scales.t.range()[1], "height": scales.x(0) - scales.x(1), "fill": "#e3e3e3" }) svg.select("#series") .selectAll("circle") .data(f1.x).enter() .append("circle") .attrs({ "cx": (d) => scales.t(d.t), "cy": (d) => scales.x(d.x), "fill": (d) => scales.fill(d.in_range), "r": 2 }); svg.select("#counter") .selectAll("circle") .data(counts).enter() .append("circle") .attrs({ "cx": (d) => scales.t(d.t), "cy": (d) => scales.counter(d.count), "fill": (d) => scales.h(d.count), "r": 3, "opacity": 0.8 }); return svg.node() }2c:T504,function fig2() { const svg = d3.select(DOM.svg(width, f1.height)); svg.selectAll("g") .data(["background", "counter", "range", "series", "states"]).enter() .append("g") .attr("id", (d) => d); var h = f2.hidden_states(f2.x), scales = f2.generate_scales(h); svg.select("#background") .append("rect") .classed("active", true) .attrs({ "width": width, "height": 200, "fill": "#F7F7F7" }) .on("mousemove", function() { f2.click_fun(svg, this) }); svg.select("#range") .append("rect") .attrs({ "x": 0, "y": scales.x(1), "width": width, "height": scales.x(0) - scales.x(1), "fill": "#e3e3e3" }) svg.select("#series") .selectAll("circle") .data(f2.x).enter() .append("circle") .attrs({ "cx": (d) => scales.t(d.t), "cy": (d) => scales.x(d.x), "fill": (d) => scales.fill(d.in_range), "r": 2 }); svg.select("#states") .selectAll("rect") .data(h).enter() .append("rect") .attrs({ "x": (d) => scales.t(d.t), "y": (d) => scales.state(d.coord), "fill": (d) => scales.h(d.h), "height": scales.state(1) - scales.state(0), "width": scales.t(1 / 200) - scales.t(0) }); return svg.node() }2d:T114d,function fig3() { const svg = d3.select(DOM.svg(width, f3.height)); var scales = f3.generate_scales(), x = math.zeros(2); f3.draw_axes(svg, scales); svg.selectAll("g") .data(["links", "points", "labels", "weights_select", "weights", "weights_text", "x_input"], (d) => d).enter() .append("g") .attr("id", (d) => d); var w = math.zeros(2, 2), u = math.matrix([[1, 1], [-1, 1]]), all_data = f3.generate_data(u, w, x); // draw the links svg.select("#links") .selectAll("path") .data(all_data["links"]).enter() .append("path") .attrs({ "d": (d) => "M" + scales.h0(d["pre"]._data[0]) + "," + scales.h1(d["pre"]._data[1]) + "L" + scales.h0(d["post"]._data[0]) + "," + scales.h1(d["post"]._data[1]), "stroke": "#a4a4a4", "stroke-width": 0.3 }); svg.select("#points") .selectAll("circle") .data(all_data["points"]).enter() .append("circle") .attrs({ "cx": (d) => scales.h0(d.h._data[0]), "cy": (d) => scales.h1(d.h._data[1]), "r": 3, "fill-opacity": .6, "fill": (d) => scales.type(d.type) }); svg.select("#x_input") .selectAll("circle") .data([x]).enter() .append("circle") .attrs({ "cx": (d) => scales.x0(d._data[0]), "cy": (d) => scales.x1(d._data[1]), "r": 10, "fill": "#ab294d" }) .call(d3.drag().on("drag", update_x)) svg.select("#weights_select") .selectAll("circle") .data(f3.rows(w, u)).enter() .append("circle") .attrs({ "cx": (d) => scales.wselect_x(d.value._data[0][0]) + scales.wtype(d.weight), "cy": (d) => scales.wselect_y(d.value._data[0][1]), "r": 7, "fill": (d) => scales.weight_fill(d.weight) }) .call(d3.drag().on("drag", (d) => update_weights(d))); svg.select("#weights") .selectAll("rect") .data(f3.entries(w, u)).enter() .append("rect") .attrs({ "x": (d) => scales.wtype(d.weight) + scales.col_offset(d.col), "y": (d) => scales.row_offset(d.row), "width": scales.col_offset(1) - scales.col_offset(0), "height": scales.row_offset(1) - scales.row_offset(0), "fill": (d) => scales.weight_fill(d.weight), "fill-opacity": (d) => .5 * scales.weight_opacity(Math.abs(d.value)), }) svg.select("#weights_text") .selectAll("text") .data(f3.entries(w, u)).enter() .append("text") .attrs({ "x": (d) => scales.wtype(d.weight) + scales.col_offset(d.col), "y": (d) => scales.row_offset(d.row), "fill": (d) => scales.weight_fill(d.weight), "fill-opacity": (d) => scales.weight_fill(Math.abs(d.value)), "font-size": 15, "alignment-baseline": "hanging" }) .text((d) => Number(d.value).toFixed(3)) svg.select("#labels") .selectAll("text") .data([{"text": "W", "weight": "w"}, {"text": "U", "weight": "u"}, {"text": "x[t]", "weight": "x"}]).enter() .append("text") .attrs({ "x": (d) => scales.wtype(d.weight), "y": (d) => d.weight == "x" ? 20 : scales.wselect_y.range()[1] + 10, "fill": (d) => scales.weight_fill(d.weight) }) .text((d) => d.text); function update_x() { if (d3.event.x > width || d3.event.x < scales.x0.range()[0] + 10 || d3.event.y > scales.x1.range()[0] - 10 || d3.event.y < 0) { return; } x = math.matrix( [scales.x0.invert(d3.event.x), scales.x1.invert(d3.event.y)]); all_data = f3.generate_data(u, w, x); f3.update_figure(svg, all_data["points"], all_data["links"], x, w, u) } function update_weights(d) { var cur = [scales.wselect_x.invert(d3.event.x - scales.wtype(d.weight)), scales.wselect_y.invert(d3.event.y)]; if (cur[0] < scales.wselect_x.domain()[0] || cur[0] > scales.wselect_x.domain()[1] || cur[1] < scales.wselect_y.domain()[0] || cur[1] > scales.wselect_y.domain()[1]) { return; } if (d.weight == "w") { w.subset(math.index(d.row, [0, 1]), cur); } else { u.subset(math.index(d.row, [0, 1]), cur); } all_data = f3.generate_data(u, w, x); f3.update_figure(svg, all_data["points"], all_data["links"], x, w, u) } return svg.node() }2e:T81f,reinforce_chart = { const svg = d3.select(DOM.svg(width, rf.height)); var scales = rf.generate_scales() svg.selectAll("g") .data(["background", "pdf", "probe", "score"]).enter() .append("g") .attr("id", (d) => d); svg.select("#background") .append("rect") .attrs({ "width": width, "height": rf.height, "fill": "#F7F7F7" }) .on("click", reset) .on("mousemove", update); function reset() { theta = -4; f_samples = []; pdfs = []; svg.selectAll("path").remove(); svg.selectAll("circle").remove(); update(); } var pdf_line = d3.line() .x((d) => scales.x(d.x)) .y((d) => scales.y(10 * d.pdf)); var grid = d3.range(-6, 6, 0.1); var theta = -4 var i = 0; var f_samples = [], pdfs = []; function update() { i += 1 var x_samples = rf.rnorm(10, theta, 1); f_samples = f_samples.concat(rf.f_data(x_samples, rf.f, i)); theta = theta - 0.05 * rf.reinforce(x_samples, rf.f, rf.score_gen(theta)); pdfs = pdfs.concat([{"iter": i, "pdf": rf.pdf_data(grid, rf.gaussian_pdf(theta, 1))}]) svg.select("#pdf") .selectAll("path") .data(pdfs, (d) => d.iter).enter() .append("path") .attrs({ "d": (d) => pdf_line(d.pdf), "stroke": "black", "stroke-width": "2px", "fill": "none" }) svg.select("#pdf") .selectAll("path") .attrs({ "stroke-opacity": (d) => Math.exp(scales.iter(i - d.iter)) }) svg.select("#probe") .selectAll("circle") .data(f_samples, (d) => d.iter + "-" + d.x).enter() .append("circle") .attrs({ "cx": (d) => scales.x(d.x), "cy": (d) => scales.y(d.f), "r": 4 }) svg.select("#probe") .selectAll("circle") .attrs({ "fill-opacity": (d) => Math.exp(scales.iter(i - d.iter)), }); f_samples = f_samples.filter((d) => d.iter >= i - 10) pdfs = pdfs.filter((d) => d.iter >= i - 10) } update(); return svg.node(); }2f:Tb1d,f1 = { var height = 400; var x = initialize_x(200); function redraw_series(svg, scales, x) { svg.select("#series") .selectAll("circle") .data(x) .attrs({ "cy": (d) => scales.x(d.x), "fill": (d) => scales.fill(d.in_range), }); } function redraw_counts(svg, scales, counts) { svg.select("#counter") .selectAll("circle") .data(counts) .transition() .duration(300) .attrs({ "cy": (d) => scales.counter(d.count), "fill": (d) => scales.h(d.count) }) } function initialize_x(n) { var x = []; for (var i = 1; i < n; i++) { var cur_x = 2 * Math.sin(3.8 * Math.PI * i / n); x.push({"t": i / n, "x": cur_x, "in_range": in_range(cur_x)}); } return x; } function entry_counts(x) { var counts = [{"t": 0, "count": 1}], cur_count; for (var i = 1; i < x.length; i++) { if (in_range(x[i].x) & !(in_range(x[i - 1].x))) { cur_count = counts[counts.length - 1].count + 1; } else { cur_count = counts[counts.length - 1].count; } counts.push({"t": i / x.length, "count": cur_count}) } return counts; } function in_range(x) { return x <= 1 & x >= 0 ? 1 : 0 } function generate_scales(counts) { return { "t": d3.scaleLinear().domain([0, 1]).range([0, width]), "x": d3.scaleLinear().domain([-3, 3]).range([200, 0]), "h": d3.scaleLinear().domain([0, 5]).range(["#f3f3f6", "#93c572"]), "fill": d3.scaleOrdinal().domain([1, 0]).range(["#0f0f0f", "#858585"]), "counter": d3.scaleLinear().domain([0, Math.max(...counts.map((d) => d.count))]).range([400, 250]) }; } function click_fun(svg, d) { var div = d3.select(d); d3.select(window) .on("mousemove", mousemove); function mousemove() { update_series(svg, d3.mouse(div.node())) } } function update_series(svg, pos) { var n = 200 if (pos[1] > 200 || pos[1] < 0) return; var scales = generate_scales(entry_counts(x)), closest_ix = Math.floor(pos[0] / width * n) / n // it's hard coded... for (var i = 0; i < x.length; i++) { var dist = Math.abs(x[i].t - closest_ix); if (dist <= 0.05) { x[i].x = scales.x.invert((0.05 - dist) / 0.05 * pos[1] + dist / 0.05 * scales.x(x[i].x)); } x[i].in_range = in_range(x[i].x); } var counts = entry_counts(x), scales = generate_scales(counts); // update the counts scale redraw_series(svg, scales, x); redraw_counts(svg, scales, counts); } return { "x": x, "height": height, "generate_scales": generate_scales, "entry_counts": entry_counts, "click_fun": click_fun, "update_series": update_series, "in_range": in_range, "initialize_x": initialize_x } }30:T9bc,f2 = { var n = 200, x = initialize_x(n); function hidden_states(x) { var h = [], prev_h2 = 0; for (var i = 1; i < x.length; i++) { h.push({"t": i / n, "h": f1.in_range(x[i].x), "coord": 0}); var h2 = {"t": i / n, "h": prev_h2, "coord": 1}; if (f1.in_range(x[i].x) & !f1.in_range(x[i - 1].x)) { h2["h"] += 1; prev_h2 = h2["h"]; } h.push(h2); } return h; } function generate_scales(h) { return { "t": d3.scaleLinear().domain([0, 1]).range([0, width]), "x": d3.scaleLinear().domain([-3, 3]).range([200, 0]), "state": d3.scaleLinear().domain([0, 3]).range([200, 400]), "fill": d3.scaleOrdinal().domain([1, 0]).range(["#0f0f0f", "#858585"]), "h": d3.scaleLinear().domain([0, 5]).range(["#f3f3f6", "#93c572"]) }; } function click_fun(svg, d) { var div = d3.select(d); d3.select(window) .on("mousemove", mousemove); function mousemove() { update_series(svg, d3.mouse(div.node())) } } function update_series(svg, pos) { if (pos[1] > 200 || pos[1] < 0) return; var scales = generate_scales(hidden_states(x)), closest_ix = Math.floor(pos[0] / width * n) / n // it's hard coded... for (var i = 0; i < x.length; i++) { var dist = Math.abs(x[i].t - closest_ix); if (dist <= 0.05) { x[i].x = scales.x.invert((0.05 - dist) / 0.05 * pos[1] + dist / 0.05 * scales.x(x[i].x)); x[i].in_range = f1.in_range(x[i].x); } } var h = hidden_states(x), scales = generate_scales(h); // update the counts scale redraw_series(svg, scales, x); redraw_states(svg, scales, h); } function redraw_series(svg, scales, x) { svg.select("#series") .selectAll("circle") .data(x) .attrs({ "cy": (d) => scales.x(d.x), "fill": (d) => scales.fill(d.in_range), }); } function redraw_states(svg, scales, h) { svg.select("#states") .selectAll("rect") .data(h) .transition() .duration(500) .attr("fill", (d) => scales.h(d.h)); } function initialize_x(n) { var x = []; for (var i = 1; i < n; i++) { var cur_x = 2 * Math.sin(3.8 * Math.PI * i / n); x.push({"t": i / n, "x": cur_x, "in_range": f1.in_range(cur_x)}); } return x; } return { "x": x, "hidden_states": hidden_states, "generate_scales": generate_scales, "click_fun": click_fun } }31:T1577,f3 = { var height = 350, scales = generate_scales(); function generate_data(w, u, x) { var f = rnn_gen(u, w, math.zeros(2)), seq = d3.range(-1, 1, .23), h = {"pre": [], "post": []}; for (var i = 0; i < seq.length; i++) { for (var j = 0; j < seq.length; j++) { var cur_h0 = math.matrix([seq[i], seq[j]]); h["pre"].push(cur_h0); h["post"].push(f(x, cur_h0)) } } var links = [], points = []; for (var i = 0; i < h["pre"].length; i++) { links.push({"pre": h["pre"][i], "post": h["post"][i]}); points.push({"h": h["pre"][i], "type": "pre"}); points.push({"h": h["post"][i], "type": "post"}); } return {"links": links, "points": points} } function generate_scales() { return { "h0": d3.scaleLinear().domain([-1, 1]).range([0, .95 * 3 * width / 4]), "h1": d3.scaleLinear().domain([-1, 1]).range([height, 0]), "x0": d3.scaleLinear().domain([-1.2, 1.2]).range([3 * width / 4, width]), "x1": d3.scaleLinear().domain([-1.2, 1.2]).range([height / 2, 0]), "type": d3.scaleOrdinal().domain(["pre", "post"]).range(["#ff9966 ", "#9955bb"]), "wselect_x": d3.scaleLinear().domain([-3, 3]).range([0, width / 8]), "wselect_y": d3.scaleLinear().domain([-3, 3]).range([.95 * 3 * height / 4, 1.05 * height / 2]), "weight_opacity": d3.scaleLinear().domain([0, 3]).range([0.4, 1]), "row_offset": d3.scaleOrdinal().domain([0, 1]).range([3 * height / 4, 7 * height / 8]), "col_offset": d3.scaleOrdinal().domain([0, 1]).range([0, width / 16]), "weight_fill": d3.scaleOrdinal().domain(["w", "u", "x"]).range(["#298eab", "#29ab87", "#ab294d"]), "wtype": d3.scaleOrdinal().domain(["w", "u"]).range([3 * width / 4, 7 * width / 8]) }; } function rnn_gen(w, u, b) { function rnn(x, h) { var mixture = math.add(math.multiply(w, x), math.multiply(u, h), b); return (mixture).map(Math.tanh) } return rnn } function rows(w, u) { var weight_rows = []; for (var i = 0; i < w._size[0]; i++) { weight_rows.push({"value": math.row(w, i), "weight": "w", "row": i}) weight_rows.push({"value": math.row(u, i), "weight": "u", "row": i}) } return weight_rows; } function entries(w, u) { var weight_entries = []; for (var i = 0; i < w._size[0]; i++) { for (var j = 0; j < w._size[1]; j++) { weight_entries.push({"value": w._data[i][j], "row": i, "col": j, "weight": "w"}); weight_entries.push({"value": u._data[i][j], "row": i, "col": j, "weight": "u"}); } } return weight_entries } function update_figure(svg, points, links, x, w, u) { svg.select("#links") .selectAll("path") .data(links) .attrs({ "d": (d) => "M" + scales.h0(d["pre"]._data[0]) + "," + scales.h1(d["pre"]._data[1]) + "L" + scales.h0(d["post"]._data[0]) + "," + scales.h1(d["post"]._data[1]), }); svg.select("#points") .selectAll("circle") .data(points) .attrs({ "cx": (d) => scales.h0(d.h._data[0]), "cy": (d) => scales.h1(d.h._data[1]), }); svg.select("#x_input") .select("circle") .data([x]) .attrs({ "cx": (d) => scales.x0(d._data[0]), "cy": (d) => scales.x1(d._data[1]), }) svg.select("#weights_select") .selectAll("circle") .data(rows(w, u)) .attrs({ "cx": (d) => scales.wselect_x(d.value._data[0][0]) + scales.wtype(d.weight), "cy": (d) => scales.wselect_y(d.value._data[0][1]), }); svg.select("#weights") .selectAll("rect") .data(entries(w, u)) .attrs({ "fill-opacity": (d) => .5 * scales.weight_opacity(Math.abs(d.value)), }); svg.select("#weights_text") .selectAll("text") .data(entries(w, u)) .attrs({ "fill-opacity": (d) => scales.weight_opacity(Math.abs(d.value)) }) .text((d) => Number(d.value).toFixed(3)) } function draw_axes(svg, scales) { svg.append("g") .attr("transform", "translate(" + scales.h0(0) + ",0)") .call(d3.axisLeft().scale(scales.h1)) svg.append("g") .attr("transform", "translate(0," + scales.h1(0) + ")") .call(d3.axisBottom().scale(scales.h0)) svg.append("g") .attr("transform", "translate(" + scales.x0(0) + ",0)") .call(d3.axisLeft().scale(scales.x1).ticks(4)) svg.append("g") .attr("transform", "translate(0," + scales.x1(0) + ")") .call(d3.axisBottom().scale(scales.x0).ticks(4)) svg.append("g") .attr("transform", "translate(" + (scales.wtype("u") + scales.wselect_x(0)) + ",0)") .call(d3.axisLeft().scale(scales.wselect_y).ticks(4)) svg.append("g") .attr("transform", "translate(" + scales.wtype("u") + "," + scales.wselect_y(0) + ")") .call(d3.axisBottom().scale(scales.wselect_x).ticks(4)) svg.append("g") .attr("transform", "translate(" + (scales.wtype("w") + scales.wselect_x(0)) + ",0)") .call(d3.axisLeft().scale(scales.wselect_y).ticks(4)) svg.append("g") .attr("transform", "translate(" + scales.wtype("w") + "," + scales.wselect_y(0) + ")") .call(d3.axisBottom().scale(scales.wselect_x).ticks(4)) } return { "height": height, "generate_scales": generate_scales, "draw_axes": draw_axes, "generate_data": generate_data, "rows": rows, "entries": entries, "update_figure": update_figure } }32:Td31,f4 = { var height = width / 4, n = 100, w = initialize_w(n), x = initialize_x(n), scales = generate_scales(); function initialize_w(n) { var x = []; for (var i = 1; i < n; i++) { x.push({"t0": i / n, "t1": i / n, "dw": 0}); if (i > 3) { x[i - 1]["dw"] = x[i - 1] - x[i - 4] } } return x; } function initialize_x(n) { var x = []; for (var i = 1; i < n; i++) { x.push({"t0": i / n, "x": Math.sin(2 * Math.PI * i / n), "dw": 0}); } return x; } function generate_scales() { return { "t0": d3.scaleLinear().domain([0, 1]).range([0, .9 * width / 2]), "t1": d3.scaleLinear().domain([0, 1]).range([height, 0]), "x": d3.scaleLinear().domain([-1.2, 1.2]).range([height, 0]), "dw": d3.scaleLinear().domain([0, 3 / n, 5 / n]).range(["#915c83", "#d3d3d3", "#779ecb"]) } } function update_warping(pos) { if (pos[0] < 0 || pos[0] > width / 2 || pos[1] > height || pos[1] < 0) return; var closest_ix = Math.round(scales.t0.invert(pos[0] / 2) * n) / n var min_value = 0; for (var i = 0; i < w.length; i++) { var dist = Math.abs(w[i].t0 / 2 - closest_ix); var b = 1 / (2 * n); if (dist <= b) { w[i].t1 = scales.t1.invert((b - dist) / b * pos[1] + dist / b * scales.t1(w[i].t1)); for (var j = 0; j < w.length; j++) { if (j < i && w[j].t1 > w[i].t1) { w[j].t1 = w[i].t1 } else if (j > i && w[j].t1 < w[i].t1) { w[j].t1 = w[i].t1 } } } if (i > 3) { w[i].dw = (w[i].t1 - w[i - 3].t1) x[i].dw = w[i].dw } } } function update_series(pos) { if (pos[0] <= width / w || pos[0] >= width || pos[1] > height || pos[1] < 0) return; var closest_ix = Math.round(scales.t0.invert(n * (pos[0] - 1.1 * width / 2))) / n for (var i = 0; i < x.length; i++) { var dist = Math.abs(x[i].t0 - closest_ix); if (dist <= 0.05) { x[i].x = scales.x.invert((0.05 - dist) / 0.05 * pos[1] + dist / 0.05 * scales.x(x[i].x)); } } } function redraw_warping(svg, scales, x, w) { svg.select("#warping") .selectAll("circle") .data(w) .attrs({ "cy": (d) => scales.t1(d.t1), "fill": (d) => scales.dw(d.dw), }); for (var i = 0; i < w.length; i++) { x[i].t1 = w[i].t1 } svg.select("#series") .selectAll("circle") .data(x) .attrs({ "cx": (d) => 1.1 * width / 2 + scales.t0(d.t1), "cy": (d) => scales.x(d.x), "fill": (d) => scales.dw(d.dw), }); } function redraw_series(svg, scales, x) { svg.select("#unwarped") .selectAll("circle") .data(x) .attrs({ "cy": (d) => scales.x(d.x), }); } function click_fun(svg, d) { var div = d3.select(d); d3.select(window) .on("mousemove", mousemove); function mousemove() { var pos = d3.mouse(div.node()); update_series(pos); update_warping(pos); redraw_series(svg, scales, x); redraw_warping(svg, scales, x, w); } } return { "initialize_x": initialize_x, "initialize_w": initialize_w, "generate_scales": generate_scales, "w": w, "x": x, "height": height, "click_fun": click_fun } }33:T157c,f5 = { var height = 300, scales = generate_scales(); function generate_data(w, u, x, z, r) { var f = rnn_gen(u, w, math.zeros(2), z, r), seq = d3.range(-1, 1, .23), h = {"pre": [], "post": []}; for (var i = 0; i < seq.length; i++) { for (var j = 0; j < seq.length; j++) { var cur_h0 = math.matrix([seq[i], seq[j]]); h["pre"].push(cur_h0); h["post"].push(f(x, cur_h0)) } } var links = [], points = []; for (var i = 0; i < h["pre"].length; i++) { links.push({"pre": h["pre"][i], "post": h["post"][i]}); points.push({"h": h["pre"][i], "type": "pre"}); points.push({"h": h["post"][i], "type": "post"}); } return {"links": links, "points": points} } function generate_scales() { return { "h0": d3.scaleLinear().domain([-1, 1]).range([0, .95 * 3 * width / 4]), "h1": d3.scaleLinear().domain([-1, 1]).range([height, 0]), "x0": d3.scaleLinear().domain([-1.2, 1.2]).range([3 * width / 4, width]), "x1": d3.scaleLinear().domain([-1.2, 1.2]).range([height / 2, 0]), "zx": d3.scaleLinear().domain([0, 1]).range([3 * width / 4, width]), "zy": d3.scaleLinear().domain([0, 1]).range([.95 * height, 3 * height / 4]), "type": d3.scaleOrdinal().domain(["pre", "post"]).range(["#ff9966 ", "#9955bb"]), "wselect_x": d3.scaleLinear().domain([-3, 3]).range([0, width / 8]), "wselect_y": d3.scaleLinear().domain([-3, 3]).range([.95 * 3 * height / 4, 1.05 * height / 2]), "weight_opacity": d3.scaleLinear().domain([0, 3]).range([0.4, 1]), "row_offset": d3.scaleOrdinal().domain([0, 1]).range([3 * height / 4, 7 * height / 8]), "col_offset": d3.scaleOrdinal().domain([0, 1]).range([0, width / 16]), "weight_fill": d3.scaleOrdinal().domain(["w", "u", "x", "z", "r"]).range(["#298eab", "#29ab87", "#ab294d", "#ffba00", "#ffa6c9"]), "wtype": d3.scaleOrdinal().domain(["w", "u", "z", "r"]).range([3 * width / 4, 7 * width / 8, 3 * width / 4, 3 * width / 4 + 60]), "ztype": d3.scaleOrdinal().domain(["z", "r"]).range(["#ffba00", "#ffa6c9"]) }; } function rnn_gen(w, u, b, z, r) { function rnn(x, h) { var h_tilde = (math.add(math.multiply(w, x), math.multiply(u, math.dotMultiply(r, h)))).map(Math.tanh) return math.add(math.dotMultiply(math.subtract(1, z), h), math.dotMultiply(z, h_tilde)) } return rnn } function gates(z, r) { var result = []; result.push({"value": z, "weight": "z"}) result.push({"value": r, "weight": "r"}) return result } function update_figure(svg, points, links, x, w, u, z, r) { svg.select("#links") .selectAll("path") .data(links) .attrs({ "d": (d) => "M" + scales.h0(d["pre"]._data[0]) + "," + scales.h1(d["pre"]._data[1]) + "L" + scales.h0(d["post"]._data[0]) + "," + scales.h1(d["post"]._data[1]), }); svg.select("#points") .selectAll("circle") .data(points) .attrs({ "cx": (d) => scales.h0(d.h._data[0]), "cy": (d) => scales.h1(d.h._data[1]), }); svg.select("#x_input") .select("circle") .data([x]) .attrs({ "cx": (d) => scales.x0(d._data[0]), "cy": (d) => scales.x1(d._data[1]), }) svg.select("#weights_select") .selectAll("circle") .data(f3.rows(w, u)) .attrs({ "cx": (d) => scales.wselect_x(d.value._data[0][0]) + scales.wtype(d.weight), "cy": (d) => scales.wselect_y(d.value._data[0][1]) }); svg.select("#gates_select") .selectAll("circle") .data(gates(z, r)) .attrs({ "cx": (d) => scales.zx(d.value._data[0]), "cy": (d) => scales.zy(d.value._data[1]), }); } function draw_axes(svg, scales) { svg.append("g") .attr("transform", "translate(" + scales.h0(0) + ",0)") .call(d3.axisLeft().scale(scales.h1)) svg.append("g") .attr("transform", "translate(0," + scales.h1(0) + ")") .call(d3.axisBottom().scale(scales.h0)) svg.append("g") .attr("transform", "translate(" + scales.x0(0) + ",0)") .call(d3.axisLeft().scale(scales.x1).ticks(4)) svg.append("g") .attr("transform", "translate(0," + scales.x1(0) + ")") .call(d3.axisBottom().scale(scales.x0).ticks(4)) svg.append("g") .attr("transform", "translate(" + (scales.wtype("u") + scales.wselect_x(0)) + ",0)") .call(d3.axisLeft().scale(scales.wselect_y).ticks(4)) svg.append("g") .attr("transform", "translate(" + scales.wtype("u") + "," + scales.wselect_y(0) + ")") .call(d3.axisBottom().scale(scales.wselect_x).ticks(4)) svg.append("g") .attr("transform", "translate(" + (scales.wtype("w") + scales.wselect_x(0)) + ",0)") .call(d3.axisLeft().scale(scales.wselect_y).ticks(4)) svg.append("g") .attr("transform", "translate(" + scales.wtype("w") + "," + scales.wselect_y(0) + ")") .call(d3.axisBottom().scale(scales.wselect_x).ticks(4)) svg.append("g") .attr("transform", "translate(" + scales.zx(0) + ",0)") .call(d3.axisLeft().scale(scales.zy).ticks(4)) svg.append("g") .attr("transform", "translate(0," + scales.zy(0) + ")") .call(d3.axisBottom().scale(scales.zx).ticks(4)) } return { "generate_scales": generate_scales, "draw_axes": draw_axes, "generate_data": generate_data, "gates": gates, "update_figure": update_figure, "height": height } }34:T795,rf = { var height = 400; function f(x) { return .5 * (x - 3) ** 2 + 6 } function pdf_data(x, pdf_fun) { var result = []; for (var i = 0; i < x.length; i++) { result.push({ "x": x[i], "pdf": pdf_fun(x[i]) }); } return result; } function gaussian_pdf(theta, sigma) { return function(x) { return 1 / Math.sqrt(2 * Math.PI * sigma) * Math.exp(- 1 / (2 * sigma ** 2) * (x - theta) ** 2) } } function reinforce(x, f, score_fun) { var result = 0; for (var i = 0; i < x.length; i++) { result += f(x[i]) * score_fun(x[i]); } return result / x.length; } function generate_scales() { return { "x": d3.scaleLinear().domain([-6, 6]).range([0, width]), "y": d3.scaleLinear().domain([.01, 30]).range([height, 0]), "iter": d3.scaleLinear().domain([1, 10]).range([0, -4]) } } function score_gen(theta) { function score(x) { return x - theta; } return score; } function rnorm(n, mu, sigma) { var result = [], norm = jstat.normal(mu, sigma); for (var i = 0; i < n; i++) { result.push(norm.sample()); } return result; } function f_data(x, f, iter) { var result = []; for (var i = 0; i < x.length; i++) { result.push({ "x": x[i], "f": f(x[i]), "iter": iter }) } return result; } function score_samples(x, score_fun, iter) { var result = []; for (var i = 0; i < x.length; i++) { result.push({ "x": x[i], "score": score_fun(x[i]), "iter": iter }); } return result; } return { "height": height, "f": f, "pdf_data": pdf_data, "gaussian_pdf": gaussian_pdf, "reinforce": reinforce, "generate_scales": generate_scales, "score_gen": score_gen, "rnorm": rnorm, "f_data": f_data, "score_samples": score_samples } }35:T4b0,slide = { function slide() { const container = document.createElement("div"); container.className = "slide"; container.appendChild(md.apply(this, arguments)); return container; } function code(strings) { const container = document.createElement("div"); const pre = container.appendChild(document.createElement("pre")); const code = pre.appendChild(document.createElement("code")); let string = strings[0] + "", i = 0, n = arguments.length; while (++i < n) string += arguments[i] + "" + strings[i]; code.textContent = string.trim(); container.className = "slide slide--code"; return container; } slide.code = code; slide.js = function() { const container = code.apply(this, arguments); const content = container.firstChild.firstChild; content.className = "js hljs javascript"; hl.highlightBlock(content); return container; }; slide.img = function(strings) { const img = new Image; let string = strings[0] + "", i = 0, n = arguments.length; while (++i < n) string += arguments[i] + "" + strings[i]; img.src = string.trim(); img.className = "slide slide--img"; return img; }; return slide; }36:[] 27:{"id":"b23663a5650b59b3","slug":"untitled/4","title":"","thumbnail":"5166ffaaedb1756b1d6d7469bda6bf6f2987a955f16e782123ce6ea696a88662","access_level":"public","create_time":"$D2019-08-13T13:26:49.161Z","update_time":"$D2025-04-16T04:46:34.855Z","view_time":null,"first_public_time":"$D2019-08-17T17:29:03.125Z","has_unpublished_edits":null,"fork":false,"stars":2,"starred":false,"owner":{"type":"team","id":"58c635fe0c6a930c","login":"krisrs1128","name":"Kris Sankaran","avatar_url":"https://avatars-next.observableusercontent.com/384a4d4ac6e6ad0ce3c5b1bc69c03d2c693a0b2859424bd8395dfef0e462967e","tier":"starter_2024","bio":""},"version":1720,"publish_version":1720,"role":null,"authored":false,"authors":[{"type":"individual","id":"544fdf34bf62b87c","login":"krisrs1128","name":"Kris Sankaran","avatar_url":"https://avatars-next.observableusercontent.com/384a4d4ac6e6ad0ce3c5b1bc69c03d2c693a0b2859424bd8395dfef0e462967e","tier":null,"bio":""}],"members":[{"type":"individual","id":"544fdf34bf62b87c","login":"krisrs1128","name":"Kris Sankaran","avatar_url":"https://avatars-next.observableusercontent.com/384a4d4ac6e6ad0ce3c5b1bc69c03d2c693a0b2859424bd8395dfef0e462967e","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"titling":"auto","title":"","cells":[{"id":6,"value":"$29","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":214,"value":"slide`\n### Preliminaries\n* Say hi to your neighbor! You'll work with them on exercises\n* Speak up: share comments and ask questions\n* Slides are online: https://observablehq.com/@krisrs1128/untitled/4`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1655,"value":"slide`#### Comprehension at a glance\n\n* What would it be like to walk through a garden and memorize it at a glance?\n* What if we could open a raw dataset and understand it immediately?\n* There would be no need for machine learning.\n * But our brains don't work this way! We need summaries.\n\n${html`

`}\n\nThe tradeoff is that now we'll have to provide signal in the form of training examples.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":402,"value":"slide`#### How should units be defined?\n\nLinear transform, then nonlinearity.\n\n${mtex_block`\n\\begin{aligned}\n\\color{#9955bb}{h_{t}} &= \\tanh\\left(\\color{#298eab}{W} \\color{#ab294d}{x_{t}} \\color{black}+ \\color{#29ab87}{U} \\color{#ff9966}{h_{t - 1}} \\color{black}+ b\\right)\n\\end{aligned}`}\n\n(drag the blue, green, and red circles)\n${fig3()}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":556,"value":"slide`### Forms of Supervision\n\nIn each case, the loss can be backpropagated to optimize parameters ${mtex`W^l, U^l, b^l`} for processors at each layer ${mtex`l`}.\n\n${html`

`}\nFigure from [2].\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1520,"value":"slide`#### Vanishing Gradients\n\nAn alternative way to see this is that the units saturate after a few inputs. Moving early inputs ${mtex`x_t`} has little effect on later summaries ${mtex`h_t`}. (click on the grey box at the top right).\n${vanishing_grads}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":521,"value":"slide`#### Gating\n* Solution is to use new \"gated\" atoms\n* Gates make it possible to keep potentially useful information from being overwritten\n\n${html`

`}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1426,"value":"slide`#### Interpretation: Update Regularization\n\n* ${mtex`{\\color{ffba00}z_{t}}`} controls how different ${mtex`{\\color{#ff9966}{h_{t - 1}}}`} can be from ${mtex`{\\color{9955bb}h_{t}}`}\n* ${mtex`{\\color{ffa6c9}r_t}`} controls whether ${mtex`{\\color{#ff9966}{h_{t - 1}}}`} matters when computing ${mtex`{\\color{9955bb}h_{t}}`}\n* This alleviates the vanishing gradient problem, making downstream summaries sensitive to upstream inputs\n * Called \"Long Term Memory\"\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":462,"value":"slide`\n### Training the counter\n\n* We'll compose these units to learn complex transformations.\n* A labeled training set will give the required signal\n * Scalar output: number of entrances\n${error_analysis}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1646,"value":"slide`#### Learned Features\n\nEach layer here is a layer ${mtex`h^{l}`} of activations. At the end, they behave similarly to the hand-crafted counting features. The tradeoff is that the activations can't be understood so easily. (click the grey rectangle to update).\n${chart7}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1609,"value":"slide`#### Exercise\nHow would you set the ${mtex`{\\color{ffba00}z_{t}}`} and ${mtex`{\\color{ffa6c9}r_t}`} gates if you wanted to ensure that the first input ${mtex`x_1`} still had influence on the later summary ${mtex`h_{10}`}? Check that this works, using the interactive visualization.\n\n(solution at the end, but it's more fun on your own!)\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":639,"value":"slide`# Soft Attention`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":896,"value":"slide`\n#### An untapped opportunity\n\nUp to this point, we've required all our thinking about the future to depend entirely on the summary at the present moment.\n\nBut depending on the task, other summaries might have helped,\n\n>> **Good models let us navigate information**.\n\n${html`

`}\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":918,"value":"slide`\n#### Sequence-to-Sequence, abstractly\n\nSince words can be represented as vectors in some embedding space, we can think of translation as a problem of transforming sequences into similarly shaped ones.\n\n${html`

`}\nFigure from [4]`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":681,"value":"slide`#### The Plan\nWe'll follow the approach of [5]. We need to define modules for,\n* Where to crop\n* How to summarize what has cropped\n* How to classify, given the crops\n\n${html`

`}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":811,"value":"slide`#### REINFORCE\n\nFormally, this uses the identity,\n\n${mtex_block`\n\\begin{aligned}\n\\frac{\\partial}{\\partial \\mu}\\Esubarg{\\mu}{f\\left(l_t\\right)} &= \\Esubarg{\\mu}{f\\left(l_t\\right)\\nabla_\\mu \\log \\mathcal{N}\\left(l_t \\vert \\mu, \\sigma^{2}\\right)} \\\\\n&\\approx \\sum_{i} f\\left(l_{ti}\\right)\\left(l_{ti} - \\mu\\right)\n\\end{aligned}`}\n\n${reinforce_chart}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":807,"value":"slide`#### Putting it together\n\nYou now have the main ingredients,\n\n* Modules that learn to crop and summarize\n* A training regime that handles randomness\n\nThe neat thing is that you can avoid processing parts of large datasets that aren't crucial.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":843,"value":"slide`#### End of Part 1\n\n* Compression and navigation are critical for processing information\n* In deep learning, modules can be specially designed for this\n * Gating: Prevents vanishing gradient\n * Soft Attention: Reduces burden on any one summary\n * Hard Attention: Avoids processing unecessary elements\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1268,"value":"$2a","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1615,"value":"slide`\n#### Exercise Solution\nYou should make ${mtex`z_1 \\approx 1`}, and the rest of the ${mtex`z_t`} small. You should make all the ${mtex`r_t`} large. This esures that ${mtex`x_1`} has a lot of input into the first ${mtex`h_1`}, and that the information is preserved for the rest of the timepoints. You can tell that this works because dragging the first input will affect all the downstream outputs -- the figure below gives an example configuration.\n\n${html`

`}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1538,"value":"slide`\n#### Exercise Solution\n${slide.js`\n// Encode\nall_h = []\nfor t in range(T):\n h = rnn_encoder(x[t], h[t - 1])\n all_h.append(h)\n\n// Decode with attention\nh_tilde, y_hat = all_h[-1], []\nfor t in range(T_prime):\n y_hat.append(pred(h_tilde))\n alpha = attn_weights(y_hat[-1], h_tilde, all_h)\n h_tilde = rnn_decoder(y_hat[-1], h_tilde, alpha * all_h)\n`}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1690,"value":"slide`#### Bonus Interpretation of Gating: Time Warping\n\n* To remember something from the past, play it in slow motion.\n* ${mtex`{\\color{ffba00}z_{t}}`} can be interpreted as the slope of a \"warping function\" [7]\n* The summary ${mtex`{\\color{9955bb}h_{t}}`} of a gated RNN corresponds to the same summary of an ungated, but warped version of it. The warping means early inputs can influence later summaries.\n\n(mouseover either rectangle)\n${chart10}\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":10,"value":"$2b","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":310,"value":"$2c","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":418,"value":"$2d","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":875,"value":"$2e","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":12,"value":"$2f","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":313,"value":"$30","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":420,"value":"$31","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":467,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":575,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1418,"value":"import {vanishing_grads} from \"@krisrs1128/vanishing-gradients\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1438,"value":"import {error_analysis} from \"@krisrs1128/error-analysis\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1508,"value":"import {chart10} from \"@krisrs1128/remembrances-of-states-past\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1648,"value":"import {chart7} from \"@krisrs1128/remembrances-of-states-past\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":812,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":14,"value":"d3 = require(\"d3-selection\", \"d3-selection-multi\", \"d3-scale\", \"d3-array\", \"d3-axis\", \"d3-drag\", \"d3-brush\", \"d3-shape\", \"d3-fetch\", \"d3-scale-chromatic\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":169,"value":"$35","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":172,"value":"slide_style = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":372,"value":"img_style = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1604,"value":"html`\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":175,"value":"hl = require(\"@observablehq/highlight.js@1.0.0/highlight.min.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":178,"value":"md``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":181,"value":"md``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":215,"value":"function mtex() {\n return tex`\n\\def\\reals{\\mathbb{R}}\n\\def\\E{\\mathbb{E}} % Expectation symbol\n\\def\\P{\\mathbb{P}} % Expectation symbol\n\\def\\Earg#1{\\E\\left[{#1}\\right]}\n\\def\\Parg#1{\\P\\left[{#1}\\right]}\n\\def\\Esubarg#1#2{\\E_{#1}\\left[{#2}\\right]}\n${String.raw.apply(String, arguments)}\n`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":231,"value":"function mtex_block() {\n return tex.block`\n\\def\\reals{\\mathbb{R}}\n\\def\\E{\\mathbb{E}} % Expectation symbol\n\\def\\P{\\mathbb{P}} % Expectation symbol\n\\def\\Earg#1{\\E\\left[{#1}\\right]}\n\\def\\Esubarg#1#2{\\E_{#1}\\left[{#2}\\right]}\n\\def\\Parg#1{\\P\\left[{#1}\\right]}\n${String.raw.apply(String, arguments)}\n`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":422,"value":"math = require(\"https://cdnjs.cloudflare.com/ajax/libs/mathjs/6.0.3/math.min.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":824,"value":"jstat = require(\"jstat\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":472,"value":"import {coin_flipping} from \"@krisrs1128/sufficiency-illustrated\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":485,"value":"weights = d3.json(\"https://gist.githubusercontent.com/krisrs1128/21c8ad097a15e23e6b639afc7b1cb412/raw/e9517d7130db18775e31446a918dbb3e1665154b/gru.json\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1565,"value":"html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1571,"value":"html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1556,"value":"html`\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W36","stdlib":"1"}} 28:"$27:body"