2d:null 2e:["$","title",null,{"children":"Remembrances of States Past | Kris Sankaran | Observable"}] 33:T4cb,chart = { const svg = d3.select(DOM.svg(width, f1.height)); svg.selectAll("g") .data(["background", "counter", "range", "series"]).enter() .append("g") .attr("id", (d) => d); var counts = f1.entry_counts(f1.x), scales = f1.generate_scales(counts); svg.select("#background") .append("rect") .classed("active", true) .attrs({ "width": width, "height": 200, "fill": "#F7F7F7" }) .on("mousemove", function() { f1.click_fun(svg, this) }); svg.select("#range") .append("rect") .attrs({ "x": 0, "y": scales.x(1), "width": width, "height": scales.x(0) - scales.x(1), "fill": "#e3e3e3" }) svg.select("#series") .selectAll("circle") .data(f1.x).enter() .append("circle") .attrs({ "cx": (d) => scales.t(d.t), "cy": (d) => scales.x(d.x), "fill": (d) => scales.fill(d.in_range), "r": 2 }); svg.select("#counter") .selectAll("circle") .data(counts).enter() .append("circle") .attrs({ "cx": (d) => scales.t(d.t), "cy": (d) => scales.counter(d.count), "fill": (d) => scales.h(d.count), "r": 3, "opacity": 0.8 }); return svg.node() }34:T4fd,chart2 = { const svg = d3.select(DOM.svg(width, f1.height)); svg.selectAll("g") .data(["background", "counter", "range", "series", "states"]).enter() .append("g") .attr("id", (d) => d); var h = f2.hidden_states(f2.x), scales = f2.generate_scales(h); svg.select("#background") .append("rect") .classed("active", true) .attrs({ "width": width, "height": 200, "fill": "#F7F7F7" }) .on("mousemove", function() { f2.click_fun(svg, this) }); svg.select("#range") .append("rect") .attrs({ "x": 0, "y": scales.x(1), "width": width, "height": scales.x(0) - scales.x(1), "fill": "#e3e3e3" }) svg.select("#series") .selectAll("circle") .data(f2.x).enter() .append("circle") .attrs({ "cx": (d) => scales.t(d.t), "cy": (d) => scales.x(d.x), "fill": (d) => scales.fill(d.in_range), "r": 2 }); svg.select("#states") .selectAll("rect") .data(h).enter() .append("rect") .attrs({ "x": (d) => scales.t(d.t), "y": (d) => scales.state(d.coord), "fill": (d) => scales.h(d.h), "height": scales.state(0) - scales.state(1), "width": scales.t(1 / 200) - scales.t(0) }); return svg.node() }35:T4c9,md`The left-to-right arrows are still used to build a type of temporal memory, but now we have bottom-up arrows, representing layer-wise composition, which help in learning complex features from otherwise simple units. A common "atom" from which these sheets are defined is the function, ${mtex_block` \begin{aligned} \color{#9955bb}{h_{t}} &= \tanh\left(\color{#298eab}{W} \color{#ab294d}{x_{t}} \color{black}+ \color{#29ab87}{U} \color{#ff9966}{h_{t - 1}} \color{black}+ b\right) \end{aligned} `} which has the form ${mtex`f_{\theta}\left({\color{ab294d}x_{t}}, \color{#ff9966}{h_{t - 1}}\right)`} where ${mtex`\theta = \{{\color{#298eab}{W}}, {\color{#29ab87}{U}}, b\}`}. It's admittedly a relatively restricted class of functions. Through ${mtex`\color{#298eab}{W}`} and ${mtex`\color{#29ab87}{U}`}, it can tell whether certain linear mixtures of components of ${mtex`\color{ab294d}{x_{t}}`} or ${mtex`\color{#ff9966}{h_{t - 1}}`} are large or small, but in the end it can only return monotone functions of this mixtures, and restricted to ${mtex`\left[-1, 1\right]`} at that. To make this less abstract, we can visualize the entire family of functions in the case that the input and summary are both two-dimensional.`36:T1146,chart3 = { const svg = d3.select(DOM.svg(width, f3.height)); var scales = f3.generate_scales(), x = math.zeros(2); f3.draw_axes(svg, scales); svg.selectAll("g") .data(["links", "points", "labels", "weights_select", "weights", "weights_text", "x_input"], (d) => d).enter() .append("g") .attr("id", (d) => d); var w = math.zeros(2, 2), u = math.matrix([[1, 1], [-1, 1]]), all_data = f3.generate_data(u, w, x); // draw the links svg.select("#links") .selectAll("path") .data(all_data["links"]).enter() .append("path") .attrs({ "d": (d) => "M" + scales.h0(d["pre"]._data[0]) + "," + scales.h1(d["pre"]._data[1]) + "L" + scales.h0(d["post"]._data[0]) + "," + scales.h1(d["post"]._data[1]), "stroke": "#a4a4a4", "stroke-width": 0.3 }); svg.select("#points") .selectAll("circle") .data(all_data["points"]).enter() .append("circle") .attrs({ "cx": (d) => scales.h0(d.h._data[0]), "cy": (d) => scales.h1(d.h._data[1]), "r": 3, "fill-opacity": .6, "fill": (d) => scales.type(d.type) }); svg.select("#x_input") .selectAll("circle") .data([x]).enter() .append("circle") .attrs({ "cx": (d) => scales.x0(d._data[0]), "cy": (d) => scales.x1(d._data[1]), "r": 10, "fill": "#ab294d" }) .call(d3.drag().on("drag", update_x)) svg.select("#weights_select") .selectAll("circle") .data(f3.rows(w, u)).enter() .append("circle") .attrs({ "cx": (d) => scales.wselect_x(d.value._data[0][0]) + scales.wtype(d.weight), "cy": (d) => scales.wselect_y(d.value._data[0][1]), "r": 7, "fill": (d) => scales.weight_fill(d.weight) }) .call(d3.drag().on("drag", (d) => update_weights(d))); svg.select("#weights") .selectAll("rect") .data(f3.entries(w, u)).enter() .append("rect") .attrs({ "x": (d) => scales.wtype(d.weight) + scales.col_offset(d.col), "y": (d) => scales.row_offset(d.row), "width": scales.col_offset(1) - scales.col_offset(0), "height": scales.row_offset(1) - scales.row_offset(0), "fill": (d) => scales.weight_fill(d.weight), "fill-opacity": (d) => .5 * scales.weight_opacity(Math.abs(d.value)), }) svg.select("#weights_text") .selectAll("text") .data(f3.entries(w, u)).enter() .append("text") .attrs({ "x": (d) => scales.wtype(d.weight) + scales.col_offset(d.col), "y": (d) => scales.row_offset(d.row), "fill": (d) => scales.weight_fill(d.weight), "fill-opacity": (d) => scales.weight_fill(Math.abs(d.value)), "font-size": 15, "alignment-baseline": "hanging" }) .text((d) => Number(d.value).toFixed(3)) svg.select("#labels") .selectAll("text") .data([{"text": "W", "weight": "w"}, {"text": "U", "weight": "u"}, {"text": "x[t]", "weight": "x"}]).enter() .append("text") .attrs({ "x": (d) => scales.wtype(d.weight), "y": (d) => d.weight == "x" ? 10 : scales.wselect_y.range()[1] + 10, "fill": (d) => scales.weight_fill(d.weight) }) .text((d) => d.text); function update_x() { if (d3.event.x > width || d3.event.x < scales.x0.range()[0] + 10 || d3.event.y > scales.x1.range()[0] - 10 || d3.event.y < 0) { return; } x = math.matrix( [scales.x0.invert(d3.event.x), scales.x1.invert(d3.event.y)]); all_data = f3.generate_data(u, w, x); f3.update_figure(svg, all_data["points"], all_data["links"], x, w, u) } function update_weights(d) { var cur = [scales.wselect_x.invert(d3.event.x - scales.wtype(d.weight)), scales.wselect_y.invert(d3.event.y)]; if (cur[0] < scales.wselect_x.domain()[0] || cur[0] > scales.wselect_x.domain()[1] || cur[1] < scales.wselect_y.domain()[0] || cur[1] > scales.wselect_y.domain()[1]) { return; } if (d.weight == "w") { w.subset(math.index(d.row, [0, 1]), cur); } else { u.subset(math.index(d.row, [0, 1]), cur); } all_data = f3.generate_data(u, w, x); f3.update_figure(svg, all_data["points"], all_data["links"], x, w, u) } return svg.node() }37:T4b7,md`By stacking these units, we can achieve further complexity. In fact, we can almost recover our ${mtex`\left[0, 1\right]`} entrance counter just by stacking a few of these units. Define the first layer by, ${mtex_block` \begin{aligned} h^{1}_{t} &= \tanh\left(\begin{pmatrix} {\color{#cd5c5c}{+}} \\ {\color{cd955c}{-}} \end{pmatrix}x_{t} - \begin{pmatrix} {\color{#cd5c5c}0} \\ {\color{#cd955c}{-}}\end{pmatrix}\right) \end{aligned} `} so that the first coordinate tells us if we're larger than 0, and the second tells us if we're less than 1. Here, ${mtex`+`} and ${mtex`-`} are arbitrary positive and negative values, respectively. In the second layer, we combine these two pieces of information, to see if we're in the ${mtex`\left[0, 1\right]`} range, ${mtex_block` \begin{aligned} h_{t}^{2} &= \tanh(1^{T} h_{t}^{1}). \end{aligned} `} These two layer are visualized below. The coordinates of ${mtex`h_{t}^{1}`} are the red and orange functions at the bottom, while ${mtex`h_{t}^{2}`} is shown above. Use the sliders to adjust the values of the weights -- if you set them to their extremes, you can find something that is a reasonably good indicator of the interval ${mtex`\left[0, 1\right]`}`38:T4af,chart4 = { const svg = d3.select(DOM.svg(width, f4.height)); var x = d3.range(-1, 2, 0.02); var h1 = f4.layer_1(x, [weights1_0, weights1_1], [0, -weights1_1]); var h2 = f4.layer_2(x, [weights1_0, weights1_1], [0, -weights1_1]); var scales = f4.generate_scales(); svg.append("g").attr("id", "layer_1") svg.append("g").attr("id", "layer_2") // Draw the curves for layer 1 svg.select("#layer_1") .selectAll("circle") .data(h1).enter() .append("circle") .attrs({ "cx": (d) => scales.x(d.x), "cy": (d) => scales.h1(d.h), "r": 2, "opacity": 0.9, "fill": (d) => scales.coord(d.coord) }); svg.select("#layer_2") .selectAll("circle") .data(h2).enter() .append("circle") .attrs({ "cx": (d) => scales.x(d.x), "cy": (d) => scales.h2(d.h), "r": 3, "fill": "black" }); var axis1 = d3.axisBottom(scales.x); svg.append("g") .attr("transform", "translate(0," + (f4.height / 4) + ")") .call(axis1); var axis2 = d3.axisBottom(scales.x); svg.append("g") .attr("transform", "translate(0," + (3 * f4.height / 4) + ")") .call(axis1); return svg.node() }39:T59c,md`There is an elegant solution to this problem: gating [7, 8]. If the value of a feature at a particular point is potentially important for something later on, we block off any updates to it that would otherwise be made by all the incoming data. It's like we decide the current value is important, and we place it under a glass case, where it is untouched by anything else in the world. At any point, its value can be read, but not changed. If at some future point, we decide we need to change the feature's value, we take it out of the case. We ungate what had been a gated value. How can this be implemented in practice? In principle, it would be possible to define a binary value for each feature coordinate, specifying whether the value should be gated at the current timestep. A good pattern of 0s and 1s could be learned from training examples -- perhaps certain types of input are generally useful in the long run, and should always be gated. In practice, learning binary patterns is difficult, since such a mask wouldn't be differentiable. Instead, it is common to use sigmoid units, which are smooth surrogates that achieve the same effect. If you pursue this line of thinking for long enough, you would probably arrive at something similar (if not identical) to Long-Term Short Term Memory (LSTM) or Gated Recurrent Unit (GRU) cells. We'll investigate GRUs in detail, though similar views could be constructd for LSTMs.`3a:T4ab,md`A GRU is defined by the update, ${mtex_block` \begin{aligned} {\color{9955bb}h_{t}} &= \left(1 - {\color{ffba00}z_t}\right) \circ {\color{#ff9966}{h_{t - 1}}} + {\color{ffba00}z_{t}} \circ \tilde{h}_{t} \end{aligned} `} which interpolates between the previous state and some new candidate, according to a factor ${mtex`z_{t}`}. To make ${mtex`z_{t}`} learnable, it can be made a function of the current input and state ${mtex_block` \begin{aligned} {\color{ffba00}{z_t}} &= \sigma\left(W_z {\color{#ab294d}x_t} + U_z {\color{#ff9966}{h_{t - 1}}}\right) \end{aligned} `} The candidate is tricker, because it has a notion of a reset ${mtex`r_{t}`}, ${mtex_block` \begin{aligned} \tilde{h}_{t} &= \tanh\left({\color{#298eab}W}{\color{#ab294d}x_t} + {\color{#29ab87}{U}}\left({\color{ffa6c9}r_t} \circ {\color{#ff9966}h_{t - 1}}\right)\right) \end{aligned} `} Think of ${mtex`\color{ffba00}z_t`} as a kind of hesitant forgetting: the overwriting you have to do in order to write anything new to ${mtex`\color{9955bb}h_t`}, while ${mtex`\color{ffa6c9}r_t`} is a brutal "deliberate" forgetting, which wasn't strictly necessary. For clarity, we have removed all biases from these activations.`3b:T11f9,chart5 = { const svg = d3.select(DOM.svg(width, f3.height)); var scales = f5.generate_scales(), x = math.zeros(2); f5.draw_axes(svg, scales); svg.selectAll("g") .data(["links", "points", "labels", "weights_select", "gates_select", "x_input"], (d) => d).enter() .append("g") .attr("id", (d) => d); var w = math.zeros(2, 2), u = math.matrix([[1, 1], [-1, 1]]), z = math.matrix([.5, .5]), r = math.matrix([.5, .5]), all_data = f5.generate_data(u, w, x, z, r); // draw the links svg.select("#links") .selectAll("path") .data(all_data["links"]).enter() .append("path") .attrs({ "d": (d) => "M" + scales.h0(d["pre"]._data[0]) + "," + scales.h1(d["pre"]._data[1]) + "L" + scales.h0(d["post"]._data[0]) + "," + scales.h1(d["post"]._data[1]), "stroke": "#a4a4a4", "stroke-width": 0.3 }); svg.select("#points") .selectAll("circle") .data(all_data["points"]).enter() .append("circle") .attrs({ "cx": (d) => scales.h0(d.h._data[0]), "cy": (d) => scales.h1(d.h._data[1]), "r": 3, "fill-opacity": .6, "fill": (d) => scales.type(d.type) }); svg.select("#x_input") .selectAll("circle") .data([x]).enter() .append("circle") .attrs({ "cx": (d) => scales.x0(d._data[0]), "cy": (d) => scales.x1(d._data[1]), "r": 10, "fill": "#ab294d" }) .call(d3.drag().on("drag", update_x)) svg.select("#weights_select") .selectAll("circle") .data(f3.rows(w, u)).enter() .append("circle") .attrs({ "cx": (d) => scales.wselect_x(d.value._data[0][0]) + scales.wtype(d.weight), "cy": (d) => scales.wselect_y(d.value._data[0][1]), "r": 7, "fill": (d) => scales.weight_fill(d.weight) }) .call(d3.drag().on("drag", (d) => update_weights(d))); svg.select("#gates_select") .selectAll("circle") .data(f5.gates(z, r)).enter() .append("circle") .attrs({ "cx": (d) => scales.zx(d.value._data[0]), "cy": (d) => scales.zy(d.value._data[1]), "r": 7, "fill": (d) => scales.ztype(d.weight) }) .call(d3.drag().on("drag", (d) => update_gates(d))); svg.select("#labels") .selectAll("text") .data([{"text": "W", "weight": "w"}, {"text": "U", "weight": "u"}, {"text": "x[t]", "weight": "x"}, {"text": "z[t],", "weight": "z"}, {"text": "r[t]", "weight": "r"}, ]).enter() .append("text") .attrs({ "x": (d) => scales.wtype(d.weight) + 10, "y": (d) => d.weight == "x" ? 10 : (d.weight == "w" || d.weight == "u" ? scales.wselect_y.range()[1] + 10 : scales.zy.range()[1] + 10), "fill": (d) => scales.weight_fill(d.weight) }) .text((d) => d.text); function update_x() { if (d3.event.x > width || d3.event.x < scales.x0.range()[0] || d3.event.y > scales.x1.range()[0] || d3.event.y < 0) { return; } x = math.matrix( [scales.x0.invert(d3.event.x), scales.x1.invert(d3.event.y)]); all_data = f5.generate_data(u, w, x, z, r); f5.update_figure(svg, all_data["points"], all_data["links"], x, w, u, z, r) } function update_weights(d) { var cur = [scales.wselect_x.invert(d3.event.x - scales.wtype(d.weight)), scales.wselect_y.invert(d3.event.y)]; if (cur[0] < scales.wselect_x.domain()[0] || cur[0] > scales.wselect_x.domain()[1] || cur[1] < scales.wselect_y.domain()[0] || cur[1] > scales.wselect_y.domain()[1]) { return; } if (d.weight == "w") { w.subset(math.index(d.row, [0, 1]), cur); } else { u.subset(math.index(d.row, [0, 1]), cur); } all_data = f5.generate_data(u, w, x, z, r); f5.update_figure(svg, all_data["points"], all_data["links"], x, w, u, z, r) } function update_gates(d) { var cur = [scales.zx.invert(d3.event.x), scales.zy.invert(d3.event.y)]; if (cur[0] < scales.zx.domain()[0] || cur[0] > scales.zx.domain()[1] || cur[1] < scales.zy.domain()[0] || cur[1] > scales.zy.domain()[1]) { return; } if (d.weight == "z") { z._data = cur } else { r._data = cur } all_data = f5.generate_data(u, w, x, z, r); f5.update_figure(svg, all_data["points"], all_data["links"], x, w, u, z, r) } return svg.node() }3c:T5f2,chart9 = { const svg = d3.select(DOM.svg(width, f6.height)); var height = 200; var u = Math.random(1); var ix = Math.round(u * mds_data.length); var scales = { "x": d3.scaleLinear().domain([0, 198]).range([0, 2 * width / 3]), "y": d3.scaleLinear().domain([-1, 9]).range([height, 0]) } svg.append("rect") .attrs({ "x": 0, "y": scales.y(1), "width": 2 * width / 3, "height": scales.y(0) - scales.y(1), "fill": "#d3d3d3" }); var line = d3.line() .x((d) => scales.x(d.time)) .y((d) => scales.y(d.value)) .curve(d3.curveBasis) svg.selectAll("path") .data([mds_data[ix].x]).enter() .append("path") .attrs({ "d": line, "fill": "none", "stroke": "black", "stroke-width": 3 }) svg.selectAll("text") .data([mds_data[ix].counts]).enter() .append("text") .attrs({ "x": .1 * width, "y": .2 * height, "font-size": 20, }) .text((d) => "Entrances: " + d) { let i = 0; while (true) { var u = Math.random(1); var ix = Math.round(u * mds_data.length); svg.selectAll("path") .data([mds_data[ix].x]) .transition() .duration(500) .attrs({ "d": line, "fill": "none", "stroke": "black", "stroke-width": 3 }) svg.selectAll("text") .data([mds_data[ix].counts]) .text((d) =>"Entrances: " + d) yield Promises.delay(1000, svg.node()); } } }3d:Td5e,chart11 = { const svg = d3.select(DOM.svg(width, f11.height)); var scales = f11.generate_scales(); errors[0] = f11.add_jitter(errors[0]) for (var i = 1; i < Object.values(errors).length; i++) { for (var j = 0; j < errors[i].length; j++) { errors[i][j].jitter = errors[0][j].jitter } } f11.draw_axes(svg, scales); svg.selectAll("g") .data(["errors", "series", "loss"], (d) => d).enter() .append("g") .attr("id", (d) => d); svg.select("#loss").append("g").attr("id", "curves"); svg.select("#loss").append("g").attr("id", "epoch_sel"); var epoch = 0, cur_series = []; svg.select("#errors") .append("rect") .attrs({ "x": scales.y.range()[0], "y": scales.y_hat.range()[1], "width": scales.y.range()[1], "height": scales.y_hat.range()[0] - scales.y_hat.range()[1], "fill-opacity": 0 }) .on("mousemove", function() { new_curves(svg, scales, this) }); // Draw the predictions vs. truth svg.select("#errors") .selectAll("circle") .data(errors[epoch]).enter() .append("circle") .attrs({ "cx": (d) => scales.y(d.y) + scales.jitter(d.jitter) + scales.phase_offset(d.phase), "cy": (d) => scales.y_hat(d.y_hat), "fill": (d) => scales.phase(d.phase), "fill-opacity": 0.9, "r": 2 }) // Draw all the loss curves svg.select("#curves") .append("rect") .attrs({ "x": 1.1 * width / 2, "y": scales.loss.range()[1], "width": scales.epoch.range()[1], "height": scales.loss.range()[0] - scales.loss.range()[1], "fill": "white" }) .on("mousemove", function() { new_epoch(svg, scales, this) }); svg.select("#curves") .selectAll("path") .data(Object.values(epochs)).enter() .append("path") .attrs({ "d": d3.line().x((d) => scales.epoch(d.epoch)).y((d) => scales.loss(d.loss)), "fill": "none", "stroke": (_, i) => i == 0? "#a0a8e4": "#a8e4a0", "stroke-width": 2 }); svg.select("#epoch_sel") .append("line") .attrs({ "x1": 1.1 * width / 2, "x2": 1.1 * width / 2, "y1": scales.loss.range()[0], "y2": scales.loss.range()[1], "stroke": "#959595", "stroke-width": 1, "stroke-opacity": 0.9, "fill": "none" }); svg.select("#series") .append("rect") .attrs({ "x": 1.1 * width / 2, "y": scales.v(1), "width": width / 2, "height": scales.v(0) - scales.v(1), "fill": "#e3e3e3" }) function new_epoch(svg, scales, d) { var div = d3.select(d); d3.select(window) .on("mousemove", mousemove); function mousemove() { var pos = d3.mouse(div.node()); if (pos[0] < 0 || pos[0] > scales.epoch.range()[1] || pos[1] > f11.height || pos[1] < scales.loss.range()[1]) { return; } epoch = Math.round(scales.epoch.invert(pos[0])); f11.redraw_points(svg, errors[epoch], scales, epoch) } } function new_curves(svg, scales, d) { var div = d3.select(d); d3.select(window) .on("mousemove", mousemove); function mousemove() { var pos = d3.mouse(div.node()); if (pos[0] < 0 || pos[0] > width / 2 || pos[1] > f11.height || pos[1] < 0) { return; } cur_series = f11.filter_series(errors[epoch], pos, scales); f11.redraw_series(svg, cur_series, scales); } } return svg.node() }3e:T79d,chart6 = { var clicked_points = []; var scales = f6.generate_scales([[-.25, -2.65], [.35, 1.27]]); var line = d3.line() .x((d) => scales.x_path(d.time)) .y((d) => scales.y_path(d.value)); const svg = d3.select(DOM.svg(width, f6.height)); svg.selectAll("g") .data(["series", "background", "embedding", "brush"]).enter() .append("g") .attr("id", (d) => d); svg.select("#background") .append("rect") .attrs({ "x": 0, "y": 0, "width": width / 2, "height": f6.height, "fill": "#F7F7F7" }) svg.select("#embedding") .selectAll("circle") .data(mds_data, f6.scatter_id).enter() .append("circle") .attrs({ "cx": (d) => scales.x(d.mds[0]), "cy": (d) => scales.y(d.mds[1]), "r": 2, "fill-opacity": 0.7, "fill": (d) => scales.fill(d.counts), }); svg.select("#series") .selectAll("path") .data(mds_data.map((d) => d.x)).enter() .append("path") .attrs({ "d": line, "transform": (d, i) => "translate(" + scales.x_focus(mds_data[i].mds[0]) + "," + scales.y_focus(mds_data[i].mds[1]) + ")", "fill": "none", "stroke": (d, i) => scales.fill(mds_data[i].counts), "stroke-opacity": 0.6, "stroke-width": 1 }) var brush = d3.brush().extent([[0, 0], [width / 2, f6.height]]).on("brush", brushed); svg.select("#brush") .attr("class", "brush") .call(brush); function brushed() { var sel = d3.event.selection var new_extent = [[scales.x.invert(sel[0][0]), scales.y.invert(sel[0][1])], [scales.x.invert(sel[1][0]), scales.y.invert(sel[1][1])]]; scales = f6.generate_scales(new_extent); svg.select("#series") .selectAll("path") .attrs({ "transform": (d, i) => "translate(" + scales.x_focus(mds_data[i].mds[0]) + "," + scales.y_focus(mds_data[i].mds[1]) + ")" }) } return svg.node(); }3f:T5c0,chart7 = { const svg = d3.select(DOM.svg(width, f7.height)); var scales = f7.generate_scales() svg.selectAll("g") .data(["background", "range", "series", "background_b", "states"]).enter() .append("g") .attr("id", (d) => d); svg.select("#background") .append("rect") .attrs({ "width": width, "height": 200, "fill": "#F7F7F7" }) .on("mousemove", function() { f7.click_fun(svg, this) }) .on("click", function() { f7.double_click(svg, this) }); svg.select("#range") .append("rect") .attrs({ "x": 0, "y": scales.x(1), "width": width, "height": scales.x(0) - scales.x(1), "fill": "#e3e3e3" }) .on("click", function() { f7.double_click(svg, this) }); svg.select("#series") .selectAll("circle") .data(f7.x).enter() .append("circle") .attrs({ "cx": (d) => scales.t(d.t), "cy": (d) => scales.x(d.x), "fill": (d) => scales.fill(d.in_range), "r": 2 }) .on("click", function() { f7.double_click(svg, this) }); svg.select("#states") .selectAll("rect") .data(f7.plot_data["h"].filter((d) => d.coord <= 10)).enter() .append("rect") .attrs({ "x": (d) => scales.hx(d.time), "y": (d) => scales.hy(d.coord) + scales.hlayer(d.layer), "fill": (d) => scales.hfill(d.value), "width": 1.05 * (scales.hx(1) - scales.hx(0)), "height": 1.05 * (scales.hy(1) - scales.hy(0)) }); return svg.node() }40:T6f3,chart10 = { const svg = d3.select(DOM.svg(width, f9.height)); svg.selectAll("g") .data(["background_l", "background_r", "identity", "unwarped", "warping", "series"]).enter() .append("g") .attr("id", (d) => d); svg.select("#background_l") .append("rect") .attrs({ "width": .9 * width / 2, "height": f9.height, "fill": "#F7F7F7" }) .on("mousemove", function() { f9.click_fun(svg, this) }); svg.select("#background_r") .append("rect") .attrs({ "x": 1.1 * width / 2, "width": .9 * width / 2, "height": f9.height, "fill": "#F7F7F7" }) .on("mousemove", function() { f9.click_fun(svg, this) }); var scales = f9.generate_scales(f1.x); svg.select("#unwarped") .selectAll("circle") .data(f9.initialize_x(f9.x.length)).enter() .append("circle") .attrs({ "cx": (d) => 1.1 * width / 2 + scales.t0(d.t0), "cy": (d) => scales.x(d.x), "fill": "#d3d3d3", "r": 3 }) svg.select("#identity") .selectAll("circle") .data(f9.initialize_w(f9.w.length)).enter() .append("circle") .attrs({ "cx": (d) => scales.t0(d.t0), "cy": (d) => scales.t1(d.t1), "fill": "#d3d3d3", "r": 3 }); svg.select("#warping") .selectAll("circle") .data(f9.w).enter() .append("circle") .attrs({ "cx": (d) => scales.t0(d.t0), "cy": (d) => scales.t1(d.t1), "fill": (d) => "#d3d3d3", "r": 3 }); svg.select("#series") .selectAll("circle") .data(f9.x).enter() .append("circle") .attrs({ "cx": (d) => 1.1 * width / 2 + scales.t0(d.t0), "cy": (d) => scales.x(d.x), "fill": "#d3d3d3", "r": 3 }); return svg.node() }41:T93b,chart8 = { const svg = d3.select(DOM.svg(width, f7.height)); svg.selectAll("g") .data(["warped_links", "warped_series", "background", "range", "series", "gates"]).enter() .append("g") .attr("id", (d) => d); svg.select("#background") .append("rect") .classed("active", true) .attrs({ "width": width, "height": 200, "fill": "#fafafa" }) .on("mousemove", function() { f8.click_fun(svg, this) }) .on("click", function() { f8.double_click(svg, this) }); var scales = f8.generate_scales() svg.select("#range") .append("rect") .attrs({ "x": 0, "y": scales.x(1), "width": width, "height": scales.x(0) - scales.x(1), "fill": "#e3e3e3" }) svg.select("#series") .selectAll("circle") .data(f8.x).enter() .append("circle") .attrs({ "cx": (d) => scales.t(d.t), "cy": (d) => scales.x(d.x), "fill": (d) => scales.fill(d.in_range), "r": 2 }) .on("click", function() { f8.double_click(svg, this) }); var z_gates = f8.gates.filter((d) => d.coord == 0 & d.layer == 0 & d.type == "z") svg.select("#warped_series") .selectAll("circle") .data(f8.x).enter() .append("circle") .attrs({ "cx": (d) => scales.t(d.t), "cy": (d) => scales.xw(d.x), "fill": (d, i) => scales.hfill(z_gates[i].value), "r": 1.5, "fill-opacity": 0.9 }) var r0 = f8.plot_data.filter((d) => d.layer == 0 && d.coord == 0 && d.type == "r"); svg.select("#warped_links") .selectAll("path") .data(f8.r_links(f8.x, r0)).enter() .append("path") .attrs({ "d": (d) => "M" + scales.t(d["pre"].t) + "," + scales.xw(d["pre"].x) + "L" + scales.t(d["post"].t) + "," + scales.xw(d["post"].x), "stroke": (d) => scales.hfill(d.r), "stroke-width": 4 }); svg.select("#gates") .selectAll("rect") .data(f8.plot_data.filter((d) => d.coord <= 10)).enter() .append("rect") .attrs({ "x": (d) => scales.hx(d.time) + scales.ztype(d.type), "y": (d) => scales.hy(d.coord) + scales.hlayer(d.layer), "fill": (d) => scales.hfill(d.value), "width": 1.05 * (scales.hx(1) - scales.hx(0)), "height": 1.05 * (scales.hy(1) - scales.hy(0)) }) .on("mouseover", (d) => f8.update_warping(svg, f8.x, f8.gates, d.layer, d.coord)) return svg.node() }42:T825,md`[1] Stigler, S. M. (1973). Studies in the History of Probability and Statistics. XXXII: Laplace, Fisher, and the discovery of the concept of sufficiency. Biometrika, 60(3), 439-445. [2] Diaconis, P. (1988, August). Sufficiency as statistical symmetry. In Proceedings of the AMS Centennial Symposium (pp. 15-26). [3] Graves, A. (2012). Supervised sequence labelling. In Supervised sequence labelling with recurrent neural networks (pp. 5-13). Springer, Berlin, Heidelberg. [4] Bengio, Y., Simard, P., & Frasconi, P. (1994). Learning long-term dependencies with gradient descent is difficult. IEEE transactions on neural networks, 5(2), 157-166. [5] Hochreiter, S. (1998). The vanishing gradient problem during learning recurrent neural nets and problem solutions. International Journal of Uncertainty, Fuzziness and Knowledge-Based Systems, 6(02), 107-116. [6] Lipton, Z. C., Berkowitz, J., & Elkan, C. (2015). A critical review of recurrent neural networks for sequence learning. arXiv preprint arXiv:1506.00019. [7] Chung, J., Gulcehre, C., Cho, K., & Bengio, Y. (2014). Empirical evaluation of gated recurrent neural networks on sequence modeling. arXiv preprint arXiv:1412.3555. [8] Hochreiter, S., & Schmidhuber, J. (1997). Long short-term memory. Neural computation, 9(8), 1735-1780. [9] Nguyen, L. H., & Holmes, S. (2019). Ten quick tips for effective dimensionality reduction. PLOS Computational Biology, 15(6), e1006907. [10] Tallec, C., & Ollivier, Y. (2018). Can recurrent neural networks warp time?. arXiv preprint arXiv:1804.11188. [11] Graves, A., Wayne, G., & Danihelka, I. (2014). Neural turing machines. arXiv preprint arXiv:1410.5401. [12] Weston, J., Chopra, S., & Bordes, A. (2014). Memory networks. arXiv preprint arXiv:1410.3916. [13] Ba, J., Mnih, V., & Kavukcuoglu, K. (2014). Multiple object recognition with visual attention. arXiv preprint arXiv:1412.7755. [14] Olah, C., & Carter, S. (2016). Attention and augmented recurrent neural networks. Distill, 1(9), e1. [15] Madsen, A. (2019). Visualizing memorization in RNNs. Distill, 4(3), e16. `43:Tb1d,f1 = { var height = 400; var x = initialize_x(200); function redraw_series(svg, scales, x) { svg.select("#series") .selectAll("circle") .data(x) .attrs({ "cy": (d) => scales.x(d.x), "fill": (d) => scales.fill(d.in_range), }); } function redraw_counts(svg, scales, counts) { svg.select("#counter") .selectAll("circle") .data(counts) .transition() .duration(300) .attrs({ "cy": (d) => scales.counter(d.count), "fill": (d) => scales.h(d.count) }) } function initialize_x(n) { var x = []; for (var i = 1; i < n; i++) { var cur_x = 2 * Math.sin(3.8 * Math.PI * i / n); x.push({"t": i / n, "x": cur_x, "in_range": in_range(cur_x)}); } return x; } function entry_counts(x) { var counts = [{"t": 0, "count": 1}], cur_count; for (var i = 1; i < x.length; i++) { if (in_range(x[i].x) & !(in_range(x[i - 1].x))) { cur_count = counts[counts.length - 1].count + 1; } else { cur_count = counts[counts.length - 1].count; } counts.push({"t": i / x.length, "count": cur_count}) } return counts; } function in_range(x) { return x <= 1 & x >= 0 ? 1 : 0 } function generate_scales(counts) { return { "t": d3.scaleLinear().domain([0, 1]).range([0, width]), "x": d3.scaleLinear().domain([-3, 3]).range([200, 0]), "h": d3.scaleLinear().domain([0, 5]).range(["#f3f3f6", "#93c572"]), "fill": d3.scaleOrdinal().domain([1, 0]).range(["#0f0f0f", "#858585"]), "counter": d3.scaleLinear().domain([0, Math.max(...counts.map((d) => d.count))]).range([400, 250]) }; } function click_fun(svg, d) { var div = d3.select(d); d3.select(window) .on("mousemove", mousemove); function mousemove() { update_series(svg, d3.mouse(div.node())) } } function update_series(svg, pos) { var n = 200 if (pos[1] > 200 || pos[1] < 0) return; var scales = generate_scales(entry_counts(x)), closest_ix = Math.floor(pos[0] / width * n) / n // it's hard coded... for (var i = 0; i < x.length; i++) { var dist = Math.abs(x[i].t - closest_ix); if (dist <= 0.05) { x[i].x = scales.x.invert((0.05 - dist) / 0.05 * pos[1] + dist / 0.05 * scales.x(x[i].x)); } x[i].in_range = in_range(x[i].x); } var counts = entry_counts(x), scales = generate_scales(counts); // update the counts scale redraw_series(svg, scales, x); redraw_counts(svg, scales, counts); } return { "x": x, "height": height, "generate_scales": generate_scales, "entry_counts": entry_counts, "click_fun": click_fun, "update_series": update_series, "in_range": in_range, "initialize_x": initialize_x } }44:T9bc,f2 = { var n = 200, x = initialize_x(n); function hidden_states(x) { var h = [], prev_h2 = 0; for (var i = 1; i < x.length; i++) { h.push({"t": i / n, "h": f1.in_range(x[i].x), "coord": 0}); var h2 = {"t": i / n, "h": prev_h2, "coord": 1}; if (f1.in_range(x[i].x) & !f1.in_range(x[i - 1].x)) { h2["h"] += 1; prev_h2 = h2["h"]; } h.push(h2); } return h; } function generate_scales(h) { return { "t": d3.scaleLinear().domain([0, 1]).range([0, width]), "x": d3.scaleLinear().domain([-3, 3]).range([200, 0]), "state": d3.scaleLinear().domain([0, 3]).range([300, 100]), "fill": d3.scaleOrdinal().domain([1, 0]).range(["#0f0f0f", "#858585"]), "h": d3.scaleLinear().domain([0, 5]).range(["#f3f3f6", "#93c572"]) }; } function click_fun(svg, d) { var div = d3.select(d); d3.select(window) .on("mousemove", mousemove); function mousemove() { update_series(svg, d3.mouse(div.node())) } } function update_series(svg, pos) { if (pos[1] > 200 || pos[1] < 0) return; var scales = generate_scales(hidden_states(x)), closest_ix = Math.floor(pos[0] / width * n) / n // it's hard coded... for (var i = 0; i < x.length; i++) { var dist = Math.abs(x[i].t - closest_ix); if (dist <= 0.05) { x[i].x = scales.x.invert((0.05 - dist) / 0.05 * pos[1] + dist / 0.05 * scales.x(x[i].x)); x[i].in_range = f1.in_range(x[i].x); } } var h = hidden_states(x), scales = generate_scales(h); // update the counts scale redraw_series(svg, scales, x); redraw_states(svg, scales, h); } function redraw_series(svg, scales, x) { svg.select("#series") .selectAll("circle") .data(x) .attrs({ "cy": (d) => scales.x(d.x), "fill": (d) => scales.fill(d.in_range), }); } function redraw_states(svg, scales, h) { svg.select("#states") .selectAll("rect") .data(h) .transition() .duration(500) .attr("fill", (d) => scales.h(d.h)); } function initialize_x(n) { var x = []; for (var i = 1; i < n; i++) { var cur_x = 2 * Math.sin(3.8 * Math.PI * i / n); x.push({"t": i / n, "x": cur_x, "in_range": f1.in_range(cur_x)}); } return x; } return { "x": x, "hidden_states": hidden_states, "generate_scales": generate_scales, "click_fun": click_fun } }45:T1577,f3 = { var height = 400, scales = generate_scales(); function generate_data(w, u, x) { var f = rnn_gen(u, w, math.zeros(2)), seq = d3.range(-1, 1, .23), h = {"pre": [], "post": []}; for (var i = 0; i < seq.length; i++) { for (var j = 0; j < seq.length; j++) { var cur_h0 = math.matrix([seq[i], seq[j]]); h["pre"].push(cur_h0); h["post"].push(f(x, cur_h0)) } } var links = [], points = []; for (var i = 0; i < h["pre"].length; i++) { links.push({"pre": h["pre"][i], "post": h["post"][i]}); points.push({"h": h["pre"][i], "type": "pre"}); points.push({"h": h["post"][i], "type": "post"}); } return {"links": links, "points": points} } function generate_scales() { return { "h0": d3.scaleLinear().domain([-1, 1]).range([0, .95 * 3 * width / 4]), "h1": d3.scaleLinear().domain([-1, 1]).range([height, 0]), "x0": d3.scaleLinear().domain([-1.2, 1.2]).range([3 * width / 4, width]), "x1": d3.scaleLinear().domain([-1.2, 1.2]).range([height / 2, 0]), "type": d3.scaleOrdinal().domain(["pre", "post"]).range(["#ff9966 ", "#9955bb"]), "wselect_x": d3.scaleLinear().domain([-3, 3]).range([0, width / 8]), "wselect_y": d3.scaleLinear().domain([-3, 3]).range([.95 * 3 * height / 4, 1.05 * height / 2]), "weight_opacity": d3.scaleLinear().domain([0, 3]).range([0.4, 1]), "row_offset": d3.scaleOrdinal().domain([0, 1]).range([3 * height / 4, 7 * height / 8]), "col_offset": d3.scaleOrdinal().domain([0, 1]).range([0, width / 16]), "weight_fill": d3.scaleOrdinal().domain(["w", "u", "x"]).range(["#298eab", "#29ab87", "#ab294d"]), "wtype": d3.scaleOrdinal().domain(["w", "u"]).range([3 * width / 4, 7 * width / 8]) }; } function rnn_gen(w, u, b) { function rnn(x, h) { var mixture = math.add(math.multiply(w, x), math.multiply(u, h), b); return (mixture).map(Math.tanh) } return rnn } function rows(w, u) { var weight_rows = []; for (var i = 0; i < w._size[0]; i++) { weight_rows.push({"value": math.row(w, i), "weight": "w", "row": i}) weight_rows.push({"value": math.row(u, i), "weight": "u", "row": i}) } return weight_rows; } function entries(w, u) { var weight_entries = []; for (var i = 0; i < w._size[0]; i++) { for (var j = 0; j < w._size[1]; j++) { weight_entries.push({"value": w._data[i][j], "row": i, "col": j, "weight": "w"}); weight_entries.push({"value": u._data[i][j], "row": i, "col": j, "weight": "u"}); } } return weight_entries } function update_figure(svg, points, links, x, w, u) { svg.select("#links") .selectAll("path") .data(links) .attrs({ "d": (d) => "M" + scales.h0(d["pre"]._data[0]) + "," + scales.h1(d["pre"]._data[1]) + "L" + scales.h0(d["post"]._data[0]) + "," + scales.h1(d["post"]._data[1]), }); svg.select("#points") .selectAll("circle") .data(points) .attrs({ "cx": (d) => scales.h0(d.h._data[0]), "cy": (d) => scales.h1(d.h._data[1]), }); svg.select("#x_input") .select("circle") .data([x]) .attrs({ "cx": (d) => scales.x0(d._data[0]), "cy": (d) => scales.x1(d._data[1]), }) svg.select("#weights_select") .selectAll("circle") .data(rows(w, u)) .attrs({ "cx": (d) => scales.wselect_x(d.value._data[0][0]) + scales.wtype(d.weight), "cy": (d) => scales.wselect_y(d.value._data[0][1]), }); svg.select("#weights") .selectAll("rect") .data(entries(w, u)) .attrs({ "fill-opacity": (d) => .5 * scales.weight_opacity(Math.abs(d.value)), }); svg.select("#weights_text") .selectAll("text") .data(entries(w, u)) .attrs({ "fill-opacity": (d) => scales.weight_opacity(Math.abs(d.value)) }) .text((d) => Number(d.value).toFixed(3)) } function draw_axes(svg, scales) { svg.append("g") .attr("transform", "translate(" + scales.h0(0) + ",0)") .call(d3.axisLeft().scale(scales.h1)) svg.append("g") .attr("transform", "translate(0," + scales.h1(0) + ")") .call(d3.axisBottom().scale(scales.h0)) svg.append("g") .attr("transform", "translate(" + scales.x0(0) + ",0)") .call(d3.axisLeft().scale(scales.x1).ticks(4)) svg.append("g") .attr("transform", "translate(0," + scales.x1(0) + ")") .call(d3.axisBottom().scale(scales.x0).ticks(4)) svg.append("g") .attr("transform", "translate(" + (scales.wtype("u") + scales.wselect_x(0)) + ",0)") .call(d3.axisLeft().scale(scales.wselect_y).ticks(4)) svg.append("g") .attr("transform", "translate(" + scales.wtype("u") + "," + scales.wselect_y(0) + ")") .call(d3.axisBottom().scale(scales.wselect_x).ticks(4)) svg.append("g") .attr("transform", "translate(" + (scales.wtype("w") + scales.wselect_x(0)) + ",0)") .call(d3.axisLeft().scale(scales.wselect_y).ticks(4)) svg.append("g") .attr("transform", "translate(" + scales.wtype("w") + "," + scales.wselect_y(0) + ")") .call(d3.axisBottom().scale(scales.wselect_x).ticks(4)) } return { "height": height, "generate_scales": generate_scales, "draw_axes": draw_axes, "generate_data": generate_data, "rows": rows, "entries": entries, "update_figure": update_figure } }46:T508,f4 = { var height = 200; function generate_scales() { return { "x": d3.scaleLinear() .domain([-1, 2]) .range([0, 2 * width / 3]), "h1": d3.scaleLinear() .domain([-1.2, 1.2]) .range([height, height / 2]), "h2": d3.scaleLinear() .domain([-1.2, 1.2]) .range([height / 2, 0]), "coord": d3.scaleOrdinal() .domain([0, 1]) .range(["#cd5c5c", "#cd955c"]) } } function layer_1(x, weights, biases) { var h_data = []; for (var i = 0; i < x.length; i++) { h_data.push({ "x": x[i], "h": Math.tanh(weights[0] * x[i] + biases[0]), "coord": 0 }) h_data.push({ "x": x[i], "h": Math.tanh(weights[1] * x[i] + biases[1]), "coord": 1 }) } return h_data; } function layer_2(x, weights, biases) { var h_data = []; for (var i = 0; i < x.length; i++) { var h1 = Math.tanh(weights[0] * x[i] + biases[0]), h2 = Math.tanh(weights[1] * x[i] + biases[1]); h_data.push({ "x": x[i], "h": Math.tanh(h1 + h2) }) } return h_data; } return { "layer_1": layer_1, "layer_2": layer_2, "generate_scales": generate_scales, "height": height } }47:T157c,f5 = { var height = 400, scales = generate_scales(); function generate_data(w, u, x, z, r) { var f = rnn_gen(u, w, math.zeros(2), z, r), seq = d3.range(-1, 1, .23), h = {"pre": [], "post": []}; for (var i = 0; i < seq.length; i++) { for (var j = 0; j < seq.length; j++) { var cur_h0 = math.matrix([seq[i], seq[j]]); h["pre"].push(cur_h0); h["post"].push(f(x, cur_h0)) } } var links = [], points = []; for (var i = 0; i < h["pre"].length; i++) { links.push({"pre": h["pre"][i], "post": h["post"][i]}); points.push({"h": h["pre"][i], "type": "pre"}); points.push({"h": h["post"][i], "type": "post"}); } return {"links": links, "points": points} } function generate_scales() { return { "h0": d3.scaleLinear().domain([-1, 1]).range([0, .95 * 3 * width / 4]), "h1": d3.scaleLinear().domain([-1, 1]).range([height, 0]), "x0": d3.scaleLinear().domain([-1.2, 1.2]).range([3 * width / 4, width]), "x1": d3.scaleLinear().domain([-1.2, 1.2]).range([height / 2, 0]), "zx": d3.scaleLinear().domain([0, 1]).range([3 * width / 4, width]), "zy": d3.scaleLinear().domain([0, 1]).range([.95 * height, 3 * height / 4]), "type": d3.scaleOrdinal().domain(["pre", "post"]).range(["#ff9966 ", "#9955bb"]), "wselect_x": d3.scaleLinear().domain([-3, 3]).range([0, width / 8]), "wselect_y": d3.scaleLinear().domain([-3, 3]).range([.95 * 3 * height / 4, 1.05 * height / 2]), "weight_opacity": d3.scaleLinear().domain([0, 3]).range([0.4, 1]), "row_offset": d3.scaleOrdinal().domain([0, 1]).range([3 * height / 4, 7 * height / 8]), "col_offset": d3.scaleOrdinal().domain([0, 1]).range([0, width / 16]), "weight_fill": d3.scaleOrdinal().domain(["w", "u", "x", "z", "r"]).range(["#298eab", "#29ab87", "#ab294d", "#ffba00", "#ffa6c9"]), "wtype": d3.scaleOrdinal().domain(["w", "u", "z", "r"]).range([3 * width / 4, 7 * width / 8, 3 * width / 4, 3 * width / 4 + 35]), "ztype": d3.scaleOrdinal().domain(["z", "r"]).range(["#ffba00", "#ffa6c9"]) }; } function rnn_gen(w, u, b, z, r) { function rnn(x, h) { var h_tilde = (math.add(math.multiply(w, x), math.multiply(u, math.dotMultiply(r, h)))).map(Math.tanh) return math.add(math.dotMultiply(math.subtract(1, z), h), math.dotMultiply(z, h_tilde)) } return rnn } function gates(z, r) { var result = []; result.push({"value": z, "weight": "z"}) result.push({"value": r, "weight": "r"}) return result } function update_figure(svg, points, links, x, w, u, z, r) { svg.select("#links") .selectAll("path") .data(links) .attrs({ "d": (d) => "M" + scales.h0(d["pre"]._data[0]) + "," + scales.h1(d["pre"]._data[1]) + "L" + scales.h0(d["post"]._data[0]) + "," + scales.h1(d["post"]._data[1]), }); svg.select("#points") .selectAll("circle") .data(points) .attrs({ "cx": (d) => scales.h0(d.h._data[0]), "cy": (d) => scales.h1(d.h._data[1]), }); svg.select("#x_input") .select("circle") .data([x]) .attrs({ "cx": (d) => scales.x0(d._data[0]), "cy": (d) => scales.x1(d._data[1]), }) svg.select("#weights_select") .selectAll("circle") .data(f3.rows(w, u)) .attrs({ "cx": (d) => scales.wselect_x(d.value._data[0][0]) + scales.wtype(d.weight), "cy": (d) => scales.wselect_y(d.value._data[0][1]) }); svg.select("#gates_select") .selectAll("circle") .data(gates(z, r)) .attrs({ "cx": (d) => scales.zx(d.value._data[0]), "cy": (d) => scales.zy(d.value._data[1]), }); } function draw_axes(svg, scales) { svg.append("g") .attr("transform", "translate(" + scales.h0(0) + ",0)") .call(d3.axisLeft().scale(scales.h1)) svg.append("g") .attr("transform", "translate(0," + scales.h1(0) + ")") .call(d3.axisBottom().scale(scales.h0)) svg.append("g") .attr("transform", "translate(" + scales.x0(0) + ",0)") .call(d3.axisLeft().scale(scales.x1).ticks(4)) svg.append("g") .attr("transform", "translate(0," + scales.x1(0) + ")") .call(d3.axisBottom().scale(scales.x0).ticks(4)) svg.append("g") .attr("transform", "translate(" + (scales.wtype("u") + scales.wselect_x(0)) + ",0)") .call(d3.axisLeft().scale(scales.wselect_y).ticks(4)) svg.append("g") .attr("transform", "translate(" + scales.wtype("u") + "," + scales.wselect_y(0) + ")") .call(d3.axisBottom().scale(scales.wselect_x).ticks(4)) svg.append("g") .attr("transform", "translate(" + (scales.wtype("w") + scales.wselect_x(0)) + ",0)") .call(d3.axisLeft().scale(scales.wselect_y).ticks(4)) svg.append("g") .attr("transform", "translate(" + scales.wtype("w") + "," + scales.wselect_y(0) + ")") .call(d3.axisBottom().scale(scales.wselect_x).ticks(4)) svg.append("g") .attr("transform", "translate(" + scales.zx(0) + ",0)") .call(d3.axisLeft().scale(scales.zy).ticks(4)) svg.append("g") .attr("transform", "translate(0," + scales.zy(0) + ")") .call(d3.axisBottom().scale(scales.zx).ticks(4)) } return { "generate_scales": generate_scales, "draw_axes": draw_axes, "generate_data": generate_data, "gates": gates, "update_figure": update_figure, "height": height } }48:T15ea,f7 = { var n = 200, x = f1.initialize_x(n), f_stack = cell_stack(weights), h = hidden_states(x), plot_data = flatten(h), scales = generate_scales(), height = 400; function hidden_states(x) { var x_seq = math.transpose(math.matrix([x.map((d) => d.x)])), h0 = [].concat(...Array(5).fill([math.zeros(20)])); return cell_seq(f_stack, x_seq, h0); } function click_fun(svg, d) { var div = d3.select(d); d3.select(window) .on("mousemove", mousemove); function mousemove() { update_series(svg, d3.mouse(div.node())) } } function update_series(svg, pos) { if (pos[1] > 200 || pos[1] < 0) return; var closest_ix = Math.floor(pos[0] / width * n) / n // it's hard coded... for (var i = 0; i < x.length; i++) { var dist = Math.abs(x[i].t - closest_ix); if (dist <= 0.05) { x[i].x = scales.x.invert((0.05 - dist) / 0.05 * pos[1] + dist / 0.05 * scales.x(x[i].x)); x[i].in_range = f1.in_range(x[i].x); } } redraw_series(svg, scales, x); } function double_click(svg) { var h = hidden_states(x); redraw_series(svg, scales, x); redraw_states(svg, scales, h); } function redraw_series(svg, scales, x) { svg.select("#series") .selectAll("circle") .data(x) .attrs({ "cy": (d) => scales.x(d.x), "fill": (d) => scales.fill(d.in_range), }); } function redraw_states(svg, scales, h) { svg.select("#states") .selectAll("rect") .data(flatten(h)["h"].filter((d) => d.coord <= 10)) .attr("fill", (d) => scales.hfill(d.value)); } function flatten(model_data) { var result = {"h": [], "z": [], "r": []}; for (var time in model_data) { for (var layer in model_data[time]) { var cur_data = model_data[time][layer]["h"]._data; for (var i = 0; i < cur_data.length; i++) { result["h"].push({ "time": parseInt(time), "coord": parseInt(i), "layer": parseInt(layer), "value": cur_data[i] }) } } } return result; } function cell_seq(f_stack, x_seq, h_prev) { var time_len = math.size(x_seq)._data[0], outputs = []; for (var i = 0; i < time_len; i++) { outputs.push(f_stack(math.squeeze(math.row(x_seq, i)), h_prev)); h_prev = []; for (var k = 0; k < outputs[i].length; k++) { h_prev.push(outputs[i][k]["h"]); } } return outputs } function cell_stack(params_list) { var cell_funs = []; for (var key in params_list) { var funs = gru_funs(params_list[key]); cell_funs.push(gru_cell(funs["r"], funs["z"], funs["n"])); } function f(x, h_prev) { var h_in = x, outputs = [], i = 0; for (var key in cell_funs) { outputs.push(cell_funs[key](h_in, h_prev[i])); h_in = outputs[i]["h"]; i += 1; } return outputs; } return f; } function gru_cell(r_fun, z_fun, n_fun) { function f(x, h) { var r = r_fun(x, h), z = z_fun(x, h), n = n_fun(x, h, r); h = math.dotMultiply(math.subtract(1, z), n); h = math.add(h, math.dotMultiply(z, h)) return {"h": h, "n": n, "z": z, "r": r}; } return f; } function gru_funs(params) { var r_fun = sigm_factory(math.matrix(params["r"]["Wi"]), math.matrix(params["r"]["Wh"]), math.matrix(params["r"]["bi"]), math.matrix(params["r"]["bh"])), z_fun = sigm_factory(math.matrix(params["z"]["Wi"]), math.matrix(params["z"]["Wh"]), math.matrix(params["z"]["bi"]), math.matrix(params["z"]["bh"])), n_fun = n_factory(math.matrix(params["n"]["Wi"]), math.matrix(params["n"]["Wh"]), math.matrix(params["n"]["bi"]), math.matrix(params["n"]["bh"])); return {"r": r_fun, "z": z_fun, "n": n_fun}; } function sigm_factory(W, U, bx, bh) { function sigm(x, h) { var Wx = math.multiply(W, x), Uh = math.multiply(U, h); Wx = math.squeeze(Wx) bx = math.squeeze(bx) bh = math.squeeze(bh) return sigmoid(math.add(Wx, Uh, bx, bh)); } return sigm; } function n_factory(W, U, bx, bh) { function n(x, h, r) { var Wx = math.multiply(W, x), Uh = math.multiply(U, h); Wx = math.squeeze(Wx); bx = math.squeeze(bx); bh = math.squeeze(bh); var h_tilde = math.dotMultiply(r, math.add(Uh, bh)); return (math.add(Wx, bx, h_tilde).map(Math.tanh)); } return n; } function sigmoid(z) { var denom = math.add(1, math.exp(math.multiply(-1, z))); return math.dotDivide(1, denom); } function generate_scales(indices) { return { "t": d3.scaleLinear().domain([0, 1]).range([0, width]), "x": d3.scaleLinear().domain([-3, 3]).range([200, 0]), "fill": d3.scaleOrdinal().domain([1, 0]).range(["#0f0f0f", "#858585"]), "hx": d3.scaleLinear().domain([0, 200]).range([0, width]), "hy": d3.scaleLinear().domain([0, 10]).range([0, .95 * height / 10]), "hfill": d3.scaleLinear().domain([-1, 0, 1]).range(["#a472c5", "white", "#93c572"]), "hlayer": d3.scaleLinear().domain([4, 0]).range([200, height - height / 10]) } } return { "x": x, "plot_data": plot_data, "update_series": update_series, "click_fun": click_fun, "double_click": double_click, "scales": scales, "height": height, "hidden_states": hidden_states, "generate_scales": generate_scales } }49:T11fa,f8 = { var h = f7.hidden_states(f7.x), plot_data = flatten(h), n = 200, x = f1.initialize_x(n), scales = generate_scales(); function generate_scales() { return { "t": d3.scaleLinear().domain([0, 1]).range([0, width]), "x": d3.scaleLinear().domain([-3, 3]).range([200, 0]), "xw": d3.scaleLinear().domain([-3, 3]).range([f7.height / 2, 200]), "fill": d3.scaleOrdinal().domain([1, 0]).range(["#0f0f0f", "#858585"]), "hx": d3.scaleLinear().domain([0, 200]).range([0, width / 2]), "hy": d3.scaleLinear().domain([0, 10]).range([0, .9 * f7.height / 12]), "hfill": d3.scaleLinear().domain([0, .5, 1]).range(["#915c83", "white", "#779ecb"]), "hlayer": d3.scaleLinear().domain([4, 0]).range([f7.height / 2, f7.height - f7.height / 12]), "ztype": d3.scaleOrdinal().domain(["z", "r"]).range([0, width / 2]) } } function r_links(x, r) { var result = []; for (var i = 0; i < r.length - 1; i++) { result.push({ "pre": {"t": x[i].t, "x": x[i].x}, "post": {"t": x[i + 1].t, "x": x[i + 1].x}, "r": r[i].value }); } return result } function flatten(model_data) { var result = []; for (var time in model_data) { for (var layer in model_data[time]) { for (var type of ["z", "r"]) { var cur_data = model_data[time][layer][type]._data; for (var i = 0; i < cur_data.length; i++) { result.push({ "time": parseInt(time), "coord": parseInt(i), "layer": parseInt(layer), "value": cur_data[i], "type": type }) } } } } return result; } function click_fun(svg, d) { var div = d3.select(d); d3.select(window) .on("mousemove", mousemove); function mousemove() { update_series(svg, d3.mouse(div.node())) } } function update_series(svg, pos) { if (pos[1] > 200 || pos[1] < 0) return; var closest_ix = Math.floor(pos[0] / width * n) / n // it's hard coded... for (var i = 0; i < x.length; i++) { var dist = Math.abs(x[i].t - closest_ix); if (dist <= 0.05) { x[i].x = scales.x.invert((0.05 - dist) / 0.05 * pos[1] + dist / 0.05 * scales.x(x[i].x)); x[i].in_range = f1.in_range(x[i].x); } } redraw_series(svg, scales, x); } function double_click(svg) { var h = f7.hidden_states(x); var gates = flatten(h); redraw_series(svg, scales, x); redraw_states(svg, scales, gates); } function redraw_series(svg, scales, x) { svg.select("#series") .selectAll("circle") .data(x) .attrs({ "cy": (d) => scales.x(d.x), "fill": (d) => scales.fill(d.in_range), }); } function redraw_states(svg, scales, gates) { svg.select("#gates") .selectAll("rect") .data(gates.filter((d) => d.coord <= 10)) .attrs({"fill": (d) => scales.hfill(d.value)}); update_warping(svg, x, gates, 0, 0) } function update_warping(svg, x, gates, layer, coord) { var z_gates = gates.filter((d) => d.layer == layer && d.coord == coord && d.type == "z"), tw = warped_times(x.map((d) => d.t), z_gates), links = r_links(x, gates.filter((d) => d.layer == layer && d.coord == coord && d.type == "r")); svg.select("#warped_series") .selectAll("circle") .data(x) .transition() .duration(500) .attrs({ "cx": (d) => scales.t(tw[d.t]), "cy": (d) => scales.xw(d.x), "fill": (d, i) => scales.hfill(z_gates[i].value) }) svg.select("#warped_links") .selectAll("path") .data(links) .transition() .duration(500) .attrs({ "d": (d) => "M" + scales.t(tw[d["pre"].t]) + "," + scales.xw(d["pre"].x) + "L" + scales.t(tw[d["post"].t]) + "," + scales.xw(d["post"].x), "stroke": (d) => scales.hfill(d.r), }); } function warped_times(times, z) { var zv = z.map((d) => d.value); var z_sum = []; zv.reduce(function(a, b, i) { return z_sum[i] = (a + b); }, 0); var tw_values = z_sum.map((d) => d / d3.sum(zv)) var tw = {}; for (var i = 0; i < times.length; i++) { tw[times[i]] = tw_values[i] } return tw } return { "generate_scales": generate_scales, "r_links": r_links, "gates": flatten(h), "plot_data": plot_data, "click_fun": click_fun, "double_click": double_click, "update_warping": update_warping, "x": x } }4a:Td31,f9 = { var height = width / 4, n = 100, w = initialize_w(n), x = initialize_x(n), scales = generate_scales(); function initialize_w(n) { var x = []; for (var i = 1; i < n; i++) { x.push({"t0": i / n, "t1": i / n, "dw": 0}); if (i > 3) { x[i - 1]["dw"] = x[i - 1] - x[i - 4] } } return x; } function initialize_x(n) { var x = []; for (var i = 1; i < n; i++) { x.push({"t0": i / n, "x": Math.sin(2 * Math.PI * i / n), "dw": 0}); } return x; } function generate_scales() { return { "t0": d3.scaleLinear().domain([0, 1]).range([0, .9 * width / 2]), "t1": d3.scaleLinear().domain([0, 1]).range([height, 0]), "x": d3.scaleLinear().domain([-1.2, 1.2]).range([height, 0]), "dw": d3.scaleLinear().domain([0, 3 / n, 5 / n]).range(["#915c83", "#d3d3d3", "#779ecb"]) } } function update_warping(pos) { if (pos[0] < 0 || pos[0] > width / 2 || pos[1] > height || pos[1] < 0) return; var closest_ix = Math.round(scales.t0.invert(pos[0] / 2) * n) / n var min_value = 0; for (var i = 0; i < w.length; i++) { var dist = Math.abs(w[i].t0 / 2 - closest_ix); var b = 1 / (2 * n); if (dist <= b) { w[i].t1 = scales.t1.invert((b - dist) / b * pos[1] + dist / b * scales.t1(w[i].t1)); for (var j = 0; j < w.length; j++) { if (j < i && w[j].t1 > w[i].t1) { w[j].t1 = w[i].t1 } else if (j > i && w[j].t1 < w[i].t1) { w[j].t1 = w[i].t1 } } } if (i > 3) { w[i].dw = (w[i].t1 - w[i - 3].t1) x[i].dw = w[i].dw } } } function update_series(pos) { if (pos[0] <= width / w || pos[0] >= width || pos[1] > height || pos[1] < 0) return; var closest_ix = Math.round(scales.t0.invert(n * (pos[0] - 1.1 * width / 2))) / n for (var i = 0; i < x.length; i++) { var dist = Math.abs(x[i].t0 - closest_ix); if (dist <= 0.05) { x[i].x = scales.x.invert((0.05 - dist) / 0.05 * pos[1] + dist / 0.05 * scales.x(x[i].x)); } } } function redraw_warping(svg, scales, x, w) { svg.select("#warping") .selectAll("circle") .data(w) .attrs({ "cy": (d) => scales.t1(d.t1), "fill": (d) => scales.dw(d.dw), }); for (var i = 0; i < w.length; i++) { x[i].t1 = w[i].t1 } svg.select("#series") .selectAll("circle") .data(x) .attrs({ "cx": (d) => 1.1 * width / 2 + scales.t0(d.t1), "cy": (d) => scales.x(d.x), "fill": (d) => scales.dw(d.dw), }); } function redraw_series(svg, scales, x) { svg.select("#unwarped") .selectAll("circle") .data(x) .attrs({ "cy": (d) => scales.x(d.x), }); } function click_fun(svg, d) { var div = d3.select(d); d3.select(window) .on("mousemove", mousemove); function mousemove() { var pos = d3.mouse(div.node()); update_series(pos); update_warping(pos); redraw_series(svg, scales, x); redraw_warping(svg, scales, x, w); } } return { "initialize_x": initialize_x, "initialize_w": initialize_w, "generate_scales": generate_scales, "w": w, "x": x, "height": height, "click_fun": click_fun } }4b:T104b,f11 = { var height = 350; function redraw_series(svg, cur_series, scales) { var ix = [], focus = []; for (var i = 0; i < curves.length; i++) { for (var j = 0; j < cur_series.length; j++) { if (curves[i].ix == cur_series[j].ix && curves[i].phase == cur_series[j].phase) { focus.push(curves[i].data) ix.push(curves[i].phase + "_" + curves[i].ix) continue } } } var focus_curves = d3.line() .x((d) => scales.i(d.i)) .y((d) => scales.v(d.v)) .curve(d3.curveBasis); var paths = svg.select("#series") .selectAll("path") .data(focus, (d, i) => ix[i]); paths.exit().remove() paths.enter() .append("path") .attrs({ "d": focus_curves, "stroke-width": 2.5, "fill": "none", "stroke": (d, i) => ix[i].startsWith("train") ? "#a0a8e4" : "#a8e4a0", "stroke-opacity": 1 }); if (ix.length > 0) { svg.select("#errors") .selectAll("circle") .transition() .duration(10) .attrs({ "fill-opacity": (d) => ix.indexOf(d.phase + "_" + d.ix) == -1 ? .1 : .9 }); } else { svg.select("#errors") .selectAll("circle") .transition() .duration(500) .attrs({ "fill-opacity": .9 }); } } function redraw_points(svg, err, scales, epoch) { svg.select("#errors") .selectAll("circle") .data(err) .transition() .duration(700) .attrs({ "cx": (d) => scales.y(d.y) + scales.jitter(d.jitter) + scales.phase_offset(d.phase), "cy": (d) => scales.y_hat(d.y_hat), }) svg.select("#epoch_sel") .select("line") .attrs({ "x1": (d) => scales.epoch(epoch), "x2": (d) => scales.epoch(epoch) }); } function filter_series(errors, pos, scales) { var result = []; for (var i = 0; i < errors.length; i++) { var pos_i = scales.y(errors[i].y) + scales.jitter(errors[i].jitter) + scales.phase_offset(errors[i].phase); var dist = Math.sqrt((pos_i - pos[0]) ** 2 + (scales.y_hat(errors[i].y_hat) - pos[1]) ** 2); if (dist < 6) { result.push({"ix": errors[i].ix, "phase": errors[i].phase}) } } return result; } function draw_axes(svg, scales) { svg.append("g") .attr("transform", "translate(" + scales.epoch(0) + ",0)") .call(d3.axisLeft().scale(scales.loss)) svg.append("g") .attr("transform", "translate(0," + scales.loss(0) + ")") .call(d3.axisBottom().scale(scales.epoch)) svg.append("g") .attr("transform", "translate(" + scales.y(0) + ",0)") .call(d3.axisLeft().scale(scales.y_hat).ticks(4)) svg.append("g") .attr("transform", "translate(0," + scales.y_hat(0) + ")") .call(d3.axisBottom().scale(scales.y).ticks(4)) } function add_jitter(x) { for (var i = 0; i < x.length; i++) { x[i]["jitter"] = Math.random(); } return x; } function generate_scales() { var scales = { "y": d3.scaleLinear().domain([0, 6]).range([.05 * width, .8 * width / 2]), "y_hat": d3.scaleLinear().domain([-.5, 7]).range([.95 * height, .1 * height]), "phase": d3.scaleOrdinal().domain(["validation", "train"]).range(["#a8e4a0", "#a0a8e4"]), "epoch": d3.scaleLinear().domain([0, 29]).range([1.1 * width / 2, width]), "loss": d3.scaleLinear().domain([0, 4]).range([.95 * height, 0.75 * height]), "v": d3.scaleLinear().domain([-6, 7]).range([.6 * height, 0]), "i": d3.scaleLinear().domain([0, 200]).range([1.1 * width / 2, width]), } var bw = scales["y"](1) - scales["y"](0) scales["phase_offset"] = d3.scaleOrdinal().domain(["train", "validation"]).range([0, .2 * bw]) scales["jitter"] = d3.scaleLinear().domain([0, 1]).range([0, 0.2 * bw]) return scales; } return { "redraw_series": redraw_series, "redraw_points": redraw_points, "add_jitter": add_jitter, "generate_scales": generate_scales, "height": height, "draw_axes": draw_axes, "filter_series": filter_series } }4c:[] 31:{"id":"3012524f2482428a","slug":"remembrances-of-states-past","title":"Remembrances of States Past","thumbnail":"e365d2947d8cc20f0d3de1217d14c9a476201894669d02132ee61c231b0a9e4a","access_level":"public","create_time":"$D2019-07-20T22:46:14.272Z","update_time":"$D2023-11-11T19:23:51.169Z","view_time":null,"first_public_time":"$D2019-07-27T01:26:10.882Z","has_unpublished_edits":null,"fork":false,"stars":4,"starred":false,"owner":{"type":"team","id":"58c635fe0c6a930c","login":"krisrs1128","name":"Kris Sankaran","avatar_url":"https://avatars-next.observableusercontent.com/384a4d4ac6e6ad0ce3c5b1bc69c03d2c693a0b2859424bd8395dfef0e462967e","tier":"starter_2024"},"version":1341,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"544fdf34bf62b87c","login":"krisrs1128","name":"Kris Sankaran","avatar_url":"https://avatars-next.observableusercontent.com/384a4d4ac6e6ad0ce3c5b1bc69c03d2c693a0b2859424bd8395dfef0e462967e","tier":null}],"members":[{"type":"individual","id":"544fdf34bf62b87c","login":"krisrs1128","name":"Kris Sankaran","avatar_url":"https://avatars-next.observableusercontent.com/384a4d4ac6e6ad0ce3c5b1bc69c03d2c693a0b2859424bd8395dfef0e462967e","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"title":"Remembrances of States Past","cells":[{"id":0,"value":"md`# Remembrances of States Past`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2,"value":"md`Good models provide a form of data compression. If a sequence of coin flips\nare drawn independently, then a binomial model can reduce arbitrarily long\nstreams of outcomes into a single number: the probability that a flip comes up\nheads. That such a compression may be possible even in more complex settings --\nand that certain forms may be optimal -- lies at the heart of statistics, where\nit is called the sufficiency principle [1, 2]. This principle allows us to\nintelligently process infinite streams of data, making it possible to gracefully\nreason about otherwise overwhelming amounts of information.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":612,"value":"md`This notion of sufficiency is rarely addressed directly in modern machine\nlearning, but it has a kind of doppelganger: memory. In this research community,\na system is said to have an effective memory mechanism when it can store\nrelevant incoming information, discarding everything else [3]. Indeed, some of the\nfoundational aims of AI were to build systems capable of capturing and\nnavigating memories over long, wandering sequences [4, 5].`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":615,"value":"md`While there are clear parallels between sufficiency and memory, there is one\nimportant point difference. AI systems are designed to learn effective\ncompressions automatically, typically through some proxy task. For example, a\nstatistician would argue that a sequence of independent coin flips constitutes a\ndraw from the binomial distribution, and hence that a reduction to the fraction\nof observed heads is optimal. In contrast, an AI system would be expected to\ndiscover this representation automatically, just by being asked to predict the\nnext flip's outcome.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":619,"value":"md`The idea that memory mechanisms can be created to learn how to summarize\nincoming information is a powerful one, facilitating applications beyond the\nreach of classical models. However, by shifting the burden of designing\nsufficient statistics from the modeller to the algorithm, these systems have\nintroduced a layer of inscrutability. The purpose of this notebook is to shed\nsome light into how AI systems sequentially process new information, by diving\ndeeply into simple examples and introducing some new views of the most common\nbuilding blocks of memory mechanisms.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"md`## Sequential Processing`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":4,"value":"md`We begin with the humblest of sequential data processors, the\nrecurrent neural network (RNN) cell [6],\n\n${mtex_block`\n\\begin{aligned}\nh_t &= f_{\\theta}\\left(x_t, h_{t - 1}\\right).\n\\end{aligned}\n`}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":967,"value":"md`The ${mtex`x_t`} are raw input data at the ${mtex`t^{th}`} position in the sequence (\"time ${mtex`t`}\"). \nThink of ${mtex`h_{t}`} as a running summary of everything you've seen so far.\nConcretely, the input and summary data are just vectors: ${mtex`x_t \\in\n\\mathbb{R}^{n}, h_{t} \\in \\mathbb{R}^{k}`}. The vectors are related by the\nfunction ${mtex`f_{\\theta}`}, which updates the current\nsummary based on new data ${mtex`x_t`}.\n\nFor example, let's say that in our summary, we only care about the number of\ntimes our sequence enters the interval ${mtex`\\left[0, 1\\right]`}. If we imagine\nthat the **grey bar** demarcates this interval, then the resulting ${mtex`h_t`}\nshould look like the **green** counting function below (mousover the top panel to redraw the function):`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":35,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":122,"value":"md`This can be easily implemented according by choosing a function ${tex`f_{\\theta}`}\nappropriately\n\n${tex.block`\n\\begin{aligned}\nh_{t} = f_{\\theta}\\left(x_{t}, h_{t - 1}\\right) \n:= \\begin{pmatrix}\n\\mathbb{1}\\{x_{t} \\in \\left[0, 1\\right]\\} \\\\\nh_{t - 1,2} + \\mathbb{1}\\{h_{t - 1, 1} = 0 \\text{ and } x_{t} \\in \\left[0, 1\\right]\\}\n\\end{pmatrix}\n\\end{aligned}.\n`}\n\nThe two coordinates of ${tex`h_t`} encode two features. The first checks whether\n${tex`x_t`} is in ${tex`\\left[0, 1\\right]`}. The second coordinate checks whether we're in\nthis range, and if at the previous step we _weren't_. If so, then it increments\nits value, because an entrance has just occurred.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":134,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":165,"value":"md`This is perhaps one of the simplest examples of a system with memory. It takes a\ncomplicated stream of input, and carefully records the number of ${tex`{\\left[0,\n1\\right]}`} entrances.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":177,"value":"md`## Composition and Feature Learning`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":180,"value":"md`In our formulation of the counter ${tex`f_{\\theta}`}, we have committed a grave sin \nfrom the view of of deep learning: we have hard-coded our features. We looked at the \nproblem, declared that the number of ${tex`\\left[0, 1\\right]`} entrances was what really\nmattered, and hand-engineered the necessary processing unit.\n\nIn the real-world, this usually won't be possible. For example, many\nphrases might be relevant for telling whether a movie review is positive\nor negative, and attempting to tabluate them would make for a very complicated\nsentiment analyzer.\n\nTo resolve this, we appeal to one of the central tenets of deep learning,\n\n> By composing simple differentiable units, we can learn useful features\n> automatically.\n\nGraphically, we shift our perspective from chains of RNN cells,\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":187,"value":"md`\n

\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":185,"value":"md`where ${mtex`h_{t} = f_{\\theta}\\left(x_{t}, h_{t - 1}\\right)`} might be a complex, hand-engineered function, to chains _and\nlayers_ of simpler processing units, ${mtex`h_{t}^{l} = f_{\\theta}^{l}\\left(h^{l - 1}_{t}, h^{l}_{t - 1}\\right)`}, with the convention that ${mtex`h^{0}_{t} := x_{t}`}.\n\n

\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":227,"value":"$35","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1094,"value":"md`Modifying the location of the points in the scatterplot on the right changes\nthe values of ${mtex`\\color{ab294d}x_t`}, ${mtex`\\color{#298eab}W`}, and\n${mtex`\\color{#29ab87}U`}. The values of the resulting weight matrices are given\nin the bottom right.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":260,"value":"$36","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":247,"value":"$37","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":318,"value":"viewof weights1_0 = slider({min: 0, max: 20, value: 1, title: \"+\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":319,"value":"viewof weights1_1 = slider({min: -20, max: 0, value: -1, title: \"-\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":312,"value":"$38","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":345,"value":"md`To construct a counter, and not just an interval detector, we need to\nintroduce a sense of history. To that end, define,\n\n${mtex_block`\n\\begin{aligned}\nh_{t}^{3} &= \\tanh\\left(w_{1} h_{t - 1}^{3} + w_{2} h_{t}^{2}\\right)\n\\end{aligned}\n`}\n\nThe idea here is to use the first term, ${mtex`w_{1}h_{t - 1}^{3}`} to \nmaintain a memory of the current counter state, while the second,\n${mtex`w_{2}h_{t}^{2}`} adds to the counter whenever it detects that we have made\na ${mtex`\\left[0, 1\\right]`} entrance.\n\nHowever, there is a tradeoff. Making ${mtex`w_{1} \\gg w_{2}`} means that increments to the count are hard\nto notice, but they are retained in memory for a long time. On the other hand,\nwhen ${mtex`w_1 \\ll w_2`}, it is easy to detect new entrances, but there\nis a risk that the counter \"forgets\" its previous values.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":354,"value":"md`## Gating and Memory`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":353,"value":"md`What we're noticing is an instance of the \"vanishing gradient\" problem [4, 5]. \nBy nature of the RNN update mechanism, new inputs gradually overwrite new\nones, and it becomes hard for the network to recall anything from many time \nsteps ago.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":360,"value":"$39","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":367,"value":"md`### The mechanics of GRUs`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":374,"value":"$3a","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":641,"value":"md`We can revisit our RNN update figure, including gating this time. We've\nreplaced the matrices with the raw weight values with a new controller, this\ntime for the values of ${mtex`\\color{ffba00}z_t`} and\n${mtex`\\color{ffa6c9}r_t`}. Since each of these gating values is two\ndimensional, corresponding to the two dimensions of\n${mtex`\\color{9955bb}h_{t}`}, we can plot them as two points on a scatterplot.\nSending ${mtex`\\color{ffba00}z_t`} to 1 collapses the next step onto the\ncandidate, while setting it to zero prevents any changes. Changing\n${mtex`\\color{ffa6c9}r_t`} modulates the influence of the previous position\n${mtex`\\color{ff9966}h_{t - 1}`} when computing the updated\n${mtex`\\color{9955bb}h_{t}`}.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":653,"value":"md`Qualitatively, ${mtex`\\color{ffba00}z_t`} controls the norm of each update, while ${mtex`\\color{ffa6c9}r_t`}\nexpands or contracts the region within which candidates can live.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":643,"value":"$3b","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":659,"value":"md`## Feature Learning for the Counter`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":661,"value":"md`Let's see how a gating and memory strategy can be learned from data, for the\ncounting problem. Instead of supplying features to count the number of\n${mtex`\\left[0, 1\\right]`} entrances, we will provide a training set labeled\nwith counts for the numbers of ${mtex`\\left[0, 1\\right]`} entrances, with the\nhope that appropriate features will emerge.\n\nWe'll need an appropriate strategy for generating training curves. More regular\nexamples will be easier to learn from, but more diverse ones will ensure the\nsystem works across a broader range of functions. Our compromise is to generate\nregular sinusoids and distort them, by stretching and compressing time. For\ndetails on the generating procedure, see the appendix. Random examples are\nplotted below.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":670,"value":"$3c","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":662,"value":"md`We can now train a gated RNN from these examples -- predicting the number of\n${mtex`\\left[0, 1\\right]`} entrances from the original sequence values. The\nlearning procedure finds task-adapted configurations of the weights ${mtex`W`}\nand ${mtex`U`} that we had been tinkering with before, along with weights to\ncontrol gating. That said, our interest lies less in the raw weights themselves,\nand more in the actual features and gatings they induce.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1207,"value":"md`The training process is summarized below. On the bottom right are **training** and **validation** error curves -- apparently, they track each other quite closely in this problem. On the left is a scatterplot of the true vs. predicted response. Each point is a curve, which has between 0 and 6 entrances into the interval ${mtex`\\left[0, 1\\right]`}. At the start, the predictions are all at about 2.5, which is close to the mean. A small amount of jitter has been added to the ${mtex`x`} axis, since otherwise the entrances are integer valued. We've also shaded the curves according to whether they belonged to the **training** or **validation** set. We have randomly sampled to only 300 of the original 10,000 data points, for clarity in the figure.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1218,"value":"md`Hovering over the loss curves on the bottom right updates the scatterplot on the left, showing how individual points become better predicted throughout training. Hovering instead over the points in the scatterplot reveals the corresponding curve, displayed in the top right, with the ${mtex`\\left[0, 1\\right]`} range **grayed out**.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1180,"value":"$3d","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":770,"value":"md`Before studying how the network responds to user provided data, we can look\ninto how the model learns features. In the plot below, we have applied a\ndimensionality reduction technique, multidimensional scaling [9], to the last\ntime point of the top layer of the learned features. The coordinates of each\nseries with respect to the resulting embedding are given in the left plot. The\nidea is that sequence that appear most similar to the model when it makes its\nfinal classification will be plotted closer to one another in these figures.\n\nBrushing on the context on the left, you can focus on individual series in the\nright plot. Color here corresponds to the true number of entrances for that\nseries -- brighter yellow series have more entrances. There is no obvious\nclustering by count, though this is likely a failure of the dimensionality\nreduction strategy, since we know that the validation performance is good.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":776,"value":"$3e","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":780,"value":"md`We can interact with the learned features in the figure below. As before, the\nupper panel gives the series that we aim to process. The bottom panel gives\ncorresponding features. Each layer is a 20-dimensional ${mtex`h_t`} -- the\nmodel's four layers correspond to the four heatmaps -- but we only plot the\nfirst 10 coordinates, for clarity. Activations are colored from\n${mtex`\\color{#a472c5}-1`} to ${mtex`\\color{#93c572}1`}. Since passing a series\nthrough the network is more intensive than passing it through a hard-coded\nfeature extractor, the activations only update once you click.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":677,"value":"$3f","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1243,"value":"md`The bottom layers seem to have learned detectors for when the series is above\nor below the ${mtex`\\left[0, 1\\right]`} range. Their main difference seems to be\nat what point in the range to change activation. The top layers are more subtle,\npartly because they don't activate all the way to the bounds of the\n${mtex`tanh`} function. They no longer seem to be range detectors, but aren't so\nobviously counters, like those we constructed above. This is not the most\nsatisfying conclusion, but a closer look at the regression coefficients that\nconvert these features into predictions might illuminate the situation.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":696,"value":"md`The evolution of features over time already gives some sense of how the network\ntracks past states and responds to new inputs. We can more directly characterize\nthis by looking at the learned gatings. Before visualizing this, however, we\nreview the relationship between gating and time warping, first brought to light\nin [10].`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":703,"value":"md`### Gating and Time Warping`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":706,"value":"md`Let ${mtex`w`} be a monotone mapping from ${mtex`\\left[0, 1\\right] \\to\n\\left[0, 1\\right]`}. We interpret it as a warping from one set of times to\nanother. The identity ${mtex`w\\left(x\\right) = x`} maintains the original time\nsequence, parts that are steeper than 1 accelerate time, and parts that shallow\nkeep time unchanged. This is illustrated in the figure below.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1055,"value":"md`The left panel defines the warping ${mtex`w`}. The panel on the right\nincludes both the original and warped functions. Parts of the warping function\nthat are steeper than 1 stretch out time, and are shaded **blue**. Parts that contract time are shaded **red**. Mouseover the left panel to redraw the warping\nfunction, and mouseover the right panel to change the function that is being\nwarped.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":925,"value":"$40","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":731,"value":"md`It will be convenient to consider a continuous state flow\n${mtex`h\\left(t\\right)`}, rather than the more typical discretized\n${mtex`h_{t}`} -- its dynamics are governed by the continuous time analogue of\nthe RNN update: ${mtex`h\\left(t + \\delta\\right) \\approx f\\left(x\\left(t + \\delta\\right),\nh\\left(t\\right)\\right)`}. If we suppose that ${mtex`\\delta = 1`} is small window\nof time with respect to the time scale at which changes in ${mtex`h`} and\n${mtex`x`} appear, then two expressions for the evolution from ${mtex`h\\left(t +\n1\\right)`} are`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":709,"value":"md`${mtex_block`\n\\begin{aligned}\nh\\left(t + 1\\right) &\\approx f\\left(x\\left(t + 1\\right), h\\left(t\\right)\\right)\n\\end{aligned}\n`}\nand, by a Taylor series expansion,\n${mtex_block`\n\\begin{aligned}\nh\\left(t + 1\\right) &\\approx h\\left(t\\right) + \\frac{\\partial h\\left(t\\right)}{dt}.\n\\end{aligned}\n`}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":713,"value":"md`Define a warped version of the original RNN by ${mtex`\\tilde{h}\\left(t\\right) =\nh\\left(w\\left(t\\right)\\right)`}. Then, applying the chain rule and making\nsubstitutions according to the previous two equations, we have\n${mtex_block`\n\\begin{aligned}\n\\tilde{h}\\left(t + 1\\right) &\\approx \\tilde{h}\\left(t\\right) + \\frac{\\partial h\\left(w\\left(t\\right)\\right)}{dw\\left(t\\right)}\\frac{\\partial w\\left(t\\right)}{\\partial t} \\\\\n&\\approx \\tilde{h}\\left(t\\right) + \\left[f\\left(\\tilde{x}\\left(t + 1\\right), \\tilde{h}\\left(t\\right)\\right) - \\tilde{h}\\left(t\\right)\\right] \\frac{\\partial w\\left(t\\right)}{\\partial t} \\\\\n&= \\left(1 - \\frac{\\partial w\\left(t\\right)}{\\partial t}\\right) \\tilde{h}\\left(t\\right) + \\frac{\\partial w\\left(t\\right)}{\\partial t}f\\left(\\tilde{x}\\left(t + 1\\right), \\tilde{h}\\left(t\\right)\\right)\n\\end{aligned}\n`}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":740,"value":"md`The punchline is that ${mtex`\\frac{\\partial w\\left(t\\right)}{\\partial t}`}\nserves a role similar to ${mtex`z_t`} in the discussion of GRUs above. This is\nconsistent with our interpretation of these gates from before -- small values of\n${mtex`z_t`} correspond to accelerations, and large values cause time to be\nplayed in slow motion.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":746,"value":"md`More than being an interesting theoretical observation, this connection provides\na practical means for visualizing the gatings ${mtex`z_t`}. For any observed\nseries, we can construct a time warping that is consistent with the series'\npattern of ${mtex`z_t`}. We can then plot the warped and original input side by\nside. To construct a consistent time warping, we set\n${mtex`w\\left(t\\right) = \\sum_{j = 1}^{t} {z_j}`} and then normalize so that\n${mtex`w\\left(t\\right)`} has an appropriate time range.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":749,"value":"md`With this in mind, we can finally plot our learned gatings. The heatmaps on the\nright and left provide raw values for ${mtex`z_t`} and ${mtex`r_t`},\nrespectively. Again, layers are arranged bottom up. In the center, we show a\nversion of the input ${mtex`x\\left(t\\right)`}, warped according to ${mtex`z_t`}. The path between points\nis shaded according to ${mtex`r_t`}: **redder** links \nindicate times where the previous hidden state ${mtex`h_t`} passes little information \nto the next timepoint's candidate, while the points themselves are shaded by ${mtex`z_t`}.\nHovering over different dimensions of the gating units reveals the different warpings and \nreset patterns associated with different coordinates of ${mtex`z_t`} and ${mtex`r_t`}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":758,"value":"$41","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1255,"value":"md`Again, we find the values of the gates reflect the input series' position\nwith respect to the ${mtex`\\left[0, 1\\right]`} range. In general, the gatings\nfor ${mtex`z_t`} seem closer to either 0 or 1, while those for ${mtex`r_t`}\nusually hover around 0.5 (with the exception of certain features, which have\nhard reset expanses). Interestingly, it seems that some gates learn specifically\nto contract the hills, while others learn to expand them. As before, the most\nextreme activations occur in the lower and intermediate layers, with the top\nlayers giving a more tempered set of gatings.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":753,"value":"md`## Discussion`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":623,"value":"md`Memory is more of a problem solving strategy than anything explicitly codified,\nand there are variants of what we have presented here that would be interesting\nto illuminate. For example, we have assumed that memory is a sequence of states\n${mtex`h_t`}-- there are more modern approaches that consider external memories that\ncan be explicitly read from or written to [11, 12]. Other approaches learn to directly\navoid processing certain parts of the input streams entirely, via hard\nattention [13]. There are increasingly intricate means of designing memory into AI\nsystems -- and while studies have illuminated the structure of of some of these\nmechanisms [14, 15], much remains unknown.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":541,"value":"md`For example, what might a study of richer data types yield? What happens when we\nleave the world of coin flips and one-dimensional series, and what is the\nbehavior of these techniques when applied to data that are not traditionally\nconsidered sequential -- say, paths on graphs, or glimpses from an image?`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":690,"value":"md`By carefully designing visualizations in these scenarios, we may develop a\nclearer sense of how modern models learn to compress data that have complex\ninterrelationships.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":939,"value":"md`## References`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":941,"value":"$42","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1040,"value":"md`## Appendix`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1042,"value":"md`### Curve generation strategy`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":429,"value":"mds_data = d3.json(\"https://gist.githubusercontent.com/krisrs1128/1b6b673ece3bbdf349833e15f24e8be7/raw/485dd5e5aadc9825b2c466ab8a3f618f58f76835/mds.json\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":452,"value":"weights = d3.json(\"https://gist.githubusercontent.com/krisrs1128/21c8ad097a15e23e6b639afc7b1cb412/raw/e9517d7130db18775e31446a918dbb3e1665154b/gru.json\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1190,"value":"curves = d3.json(\"https://gist.githubusercontent.com/krisrs1128/3c816018f76606090b42d1f15036791e/raw/15f82b98cd63ca90b269610c5b67b3179df11f40/curves.json\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1191,"value":"epochs = d3.json(\"https://gist.githubusercontent.com/krisrs1128/67c45bf9109849e1d233803906767653/raw/36170b3ed38e02d3dfcaa14525ace53d42b4dab7/epochs.json\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1193,"value":"errors = d3.json(\"https://gist.githubusercontent.com/krisrs1128/d94616ad1fa977f94a2df4f3c010ae65/raw/3f92f72dda24c71f0b1f287a875f8db810c8408e/errors.json\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":56,"value":"d3 = require(\"d3-selection@1\", \"d3-selection-multi\", \"d3-scale\", \"d3-array\", \"d3-axis\", \"d3-drag@1.2.5\", \"d3-brush\", \"d3-shape\", \"d3-fetch\", \"d3-scale-chromatic\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":261,"value":"math = require(\"https://cdnjs.cloudflare.com/ajax/libs/mathjs/6.0.3/math.min.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":324,"value":"import {slider} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":87,"value":"$43","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":133,"value":"$44","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":257,"value":"$45","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":313,"value":"$46","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":389,"value":"$47","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":414,"value":"f6 = {\n \n var ts_id = (d) => d[0].value + \"-\" + d[1].value,\n scatter_id = (d) => d.mds[0] + \"-\" + d.mds[1],\n height = 300;\n \n function generate_scales(brush_extent) {\n return {\n \"x\": d3.scaleLinear().domain([-1.17817, 3.017237]).range([0, width / 2]),\n \"y\": d3.scaleLinear().domain([-1.66336, 2.02766951138]).range([height, 0]),\n \"x_focus\": d3.scaleLinear().domain([brush_extent[0][0], brush_extent[1][0]]).range([width / 2, width]),\n \"y_focus\": d3.scaleLinear().domain([brush_extent[1][1], brush_extent[0][1]]).range([height, 0]),\n \"x_path\": d3.scaleLinear().domain([0, 100]).range([0, 20]),\n \"y_path\": d3.scaleLinear().domain([0, 20]).range([20, 0]),\n \"fill\": (d) => d3.interpolatePlasma(d / 5)\n }\n }\n \n return {\n \"generate_scales\": generate_scales,\n \"ts_id\": ts_id,\n \"scatter_id\": scatter_id,\n \"height\": height\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":446,"value":"$48","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":504,"value":"$49","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":926,"value":"$4a","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1182,"value":"$4b","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":15,"value":"function mtex() {\n return tex`\n\\def\\reals{\\mathbb{R}}\n\\def\\E{\\mathbb{E}} % Expectation symbol\n\\def\\P{\\mathbb{P}} % Expectation symbol\n\\def\\Earg#1{\\E\\left[{#1}\\right]}\n\\def\\Parg#1{\\P\\left[{#1}\\right]}\n\\def\\Esubarg#1#2{\\E_{#1}\\left[{#2}\\right]}\n${String.raw.apply(String, arguments)}\n`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":16,"value":"function mtex_block() {\n return tex.block`\n\\def\\reals{\\mathbb{R}}\n\\def\\E{\\mathbb{E}} % Expectation symbol\n\\def\\P{\\mathbb{P}} % Expectation symbol\n\\def\\Earg#1{\\E\\left[{#1}\\right]}\n\\def\\Esubarg#1#2{\\E_{#1}\\left[{#2}\\right]}\n\\def\\Parg#1{\\P\\left[{#1}\\right]}\n${String.raw.apply(String, arguments)}\n`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W4c","stdlib":"1","titling":"auto"}} 32:"$31:body"