AI Research Scientist, Director of SAP AI Research
Published
Edited
Feb 11, 2022
Transformer Language Model Dynamics
chart = {
// var svg = d3
// .select("#chart")
let svg = d3.create("svg")
.attr("viewBox", [-5, -35, width + 10, height + 40])
// .style("margin", "0 100px")
.style("background", "black");
svg
.append("g")
.attr(
"transform",
"translate(" + margin.left + 20 + "," + margin.top + 20 + ")"
);
var index = -1;
function preprocess(data) {
index = index + 1;
if (index == data.length) {
index = 0;
}
return {
min_domain_x: data[index].data_range.min,
max_domain_x: data[index].data_range.max,
data: data[index].data_set1
};
}
function update(myData) {
var max_domain_x = myData.max_domain_x * 1.1;
var min_domain_x = myData.min_domain_x * 0.4;
// 1. Add the SVG to the page and employ #2
// X axis: scale and draw:
var x = d3
.scaleLinear()
.domain([min_domain_x, max_domain_x])
.range([0, width]);
var categories = [
"1",
"2",
"3",
"4",
"5",
"6",
"7",
"8",
"9",
"10",
"11",
"12"
];
var col_array = d3.quantize(
d3.interpolateHcl("#69b36b", "#69b3a2"),
categories.length
);
// Create the Y axis
var yName = d3
.scaleBand()
.domain(categories)
.range([0, height])
.paddingInner(1);
svg.append("g");
// number of samples for KDE
var n_kde = 250;
// determine number of data samples
var n_data = myData.data.length;
var allDensity = [];
// this variable is needed to scale data
var maxData = [];
// compute the x scale of the data (x Axis), y axis computation is deferred until after KDE
var x = d3
.scaleLinear()
.domain([min_domain_x, max_domain_x])
.nice()
.range([0, width]);
for (var i = 0; i < categories.length; i++) {
var key = categories[i];
var mean = d3.randomUniform(30, 90)();
var variance = d3.randomUniform(5, 10)();
var data_set = myData.data[i];
var kde_thresholds = x.ticks(n_kde);
var bandwidth = 0.3;
var density = kde(epanechnikov(bandwidth), kde_thresholds, data_set);
allDensity.push({ key: key, density: density });
// push data for finding the max value
density.forEach((element) => {
maxData.push(element[1]);
});
}
// determine max value to scale nicely
maxData = d3.max(maxData);
var y = d3
.scaleLinear()
.domain([0, maxData * categories.length * (1 - 0.2)])
.range([height, 0]);
// Add areas
var u = svg
.selectAll("areas")
.data(allDensity)
.enter()
.append("path")
.attr("stroke", function (d) {
return col_array[d.key - 1];
})
.attr("transform", function (d) {
return "translate(0," + (yName(d.key) - height) + ")";
})
.datum(function (d) {
return d.density;
})
.attr("fill-opacity", "0.0")
.attr("stroke-opacity", "0.0")
.attr("stroke-width", 1.5)
.attr(
"d",
d3
.line()
.curve(d3.curveBasis)
.x(function (d) {
return x(d[0]);
})
.y(function (d) {
return y(d[1]);
})
)
.transition()
.ease(d3.easeLinear)
.transition()
.style("stroke-opacity", "1.0")
.duration(1000)
.transition()
.style("fill", "69b3a2")
.style("fill-opacity", "0.5")
.duration( 1500)
.transition()
.style("stroke-width", 1)
.style("opacity", 0.0)
.duration(4000);
}
// inspired by:
// https://bl.ocks.org/d3noob/464c92429ac05c6a19a1
d3.interval(() => {
update(preprocess(data));
}, 2000);
yield svg.node();
}
d3 = require("d3@5")
margin = ({top: 0, right: 0, bottom: 0, left: 0})
width = 600 - margin.left - margin.right
height = 300 - margin.top - margin.bottom
url = "https://raw.githubusercontent.com/TJKlein/tjklein.github.io/master/data/highres_transformer.json"
data = (await fetch(url)).json()
function kde(kernel, thresholds, data) {
return thresholds.map(t => [t, d3.mean(data, d => kernel(t - d))]);
}
function epanechnikov(bandwidth) {
return x => Math.abs(x /= bandwidth) <= 1 ? 0.75 * (1 - x * x) / bandwidth : 0;
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
