Transformer Language Model Dynamics

3e:null 3f:["$","title",null,{"children":"Transformer Language Model Dynamics | Tassilo Klein | Observable"}] 45:Tcc8,chart = { var svg = d3.select("#chart").append("svg") .attr("viewBox", [-10, -50, width+25, height+65]) .style("margin", "0 100px") .style("background", "black") svg .append("g") .attr("transform", "translate(" + margin.left +20 + "," + margin.top +20+ ")"); var index = -1 function preprocess(data) { index = index +1 if (index == data.length) { index = 0 } return ({min_domain_x: data[index].data_range.min, max_domain_x: data[index].data_range.max, data: data[index].data_set1}) } function update(myData) { var max_domain_x = myData.max_domain_x*1.1 var min_domain_x = myData.min_domain_x*.4 // 1. Add the SVG to the page and employ #2 // X axis: scale and draw: var x = d3.scaleLinear() .domain([min_domain_x,max_domain_x]) .range([0, width]); var categories = ["1","2","3","4","5","6","7","8","9","10","11","12"]; var col_array = d3.quantize(d3.interpolateHcl("#69b36b", "#69b3a2"), categories.length) // Create the Y axis var yName = d3.scaleBand() .domain(categories) .range([0, height]) .paddingInner(1) svg.append("g") // number of samples for KDE var n_kde = 250 // determine number of data samples var n_data = myData.data.length var allDensity = [] // this variable is needed to scale data var maxData = [] // compute the x scale of the data (x Axis), y axis computation is deferred until after KDE var x = d3.scaleLinear() .domain([min_domain_x,max_domain_x]).nice() .range([0, width]) for (var i = 0; i < categories.length; i++) { var key = categories[i] var mean = d3.randomUniform(30,90)() var variance = d3.randomUniform(5,10)() var data_set = myData.data[i] var kde_thresholds = x.ticks(n_kde) var bandwidth = 0.30 var density = kde(epanechnikov(bandwidth), kde_thresholds, data_set) allDensity.push({key: key, density: density}) // push data for finding the max value density.forEach((element) => { maxData.push(element[1])}); } // determine max value to scale nicely maxData = d3.max(maxData) var y = d3.scaleLinear() .domain([0, maxData*categories.length*(1-0.2)]) .range([ height, 0]); // Add areas var u = svg.selectAll("areas") .data(allDensity) .enter() .append("path") .attr("stroke", function(d){ return(col_array[d.key-1])}) .attr("transform", function(d){ return("translate(0," + (yName(d.key)-height) +")" )}) .datum(function(d){return(d.density)}) .attr("fill-opacity","0.0") .attr("stroke-opacity", "0.0") .attr("stroke-width", 1.5) .attr("d", d3.line() .curve(d3.curveBasis) .x(function(d) { return x(d[0]); }) .y(function(d) { return y(d[1]); }) ) .transition() .ease(d3.easeLinear) .transition() .style("stroke-opacity","1.0") .duration(1000) .transition() .style("fill", "69b3a2") .style("fill-opacity","0.5") .duration(2000) .transition() .style("stroke-width", 1) .style("opacity", 0.0) .duration(10000); } // inspired by: // https://bl.ocks.org/d3noob/464c92429ac05c6a19a1 d3.interval(() => { update(preprocess(data)); }, 2000); return svg.node(); }46:[{"id":"b1fef423a58d948ef02df549d34d1ed3ff9762b9819a89fad68fecd0ab68848619b0e9314f4370725b0165d45893c7efc055a76ee08af09e6b2002328a037048","href":"https://static.observableusercontent.com/files/b1fef423a58d948ef02df549d34d1ed3ff9762b9819a89fad68fecd0ab68848619b0e9314f4370725b0165d45893c7efc055a76ee08af09e6b2002328a037048","name":"highres_transformer.json","type":"application/json","lastModified":1644601806507.453,"size":2218913}] 43:{"id":"8dd668317b882934","slug":"transformer-language-model-dynamics-visualization","title":"Transformer Language Model Dynamics","thumbnail":"90616889b3f497ddd80ca6342258a4a7330097c5cdb0860d82ae575a681bb3da","access_level":"unlisted","create_time":"$D2022-02-11T17:27:10.216Z","update_time":"$D2022-02-11T18:32:46.590Z","view_time":null,"first_public_time":null,"has_unpublished_edits":null,"fork":{"id":"1743a7a451cd13b1","slug":"transformer-language-model-dynamics","title":"Transformer Language Model Dynamics","owner":{"id":"d5b0c272632e1d22","login":"tjklein","name":"Tassilo Klein"}},"stars":0,"starred":false,"owner":{"type":"team","id":"d5b0c272632e1d22","login":"tjklein","name":"Tassilo Klein","avatar_url":"https://avatars-next.observableusercontent.com/2bf705475e7b33a06a94a42c4a03afcc7ab7e029a47d2f819ecd72f07c1a1af5","tier":"starter_2024","bio":"AI Research Scientist, Director of SAP AI Research"},"version":175,"publish_version":175,"role":null,"authored":false,"authors":[{"type":"individual","id":"78bfd32b66ed7101","login":"tjklein","name":"Tassilo Klein","avatar_url":"https://avatars-next.observableusercontent.com/2bf705475e7b33a06a94a42c4a03afcc7ab7e029a47d2f819ecd72f07c1a1af5","tier":null,"bio":"AI Research Scientist, Director of SAP AI Research"}],"members":[{"type":"individual","id":"78bfd32b66ed7101","login":"tjklein","name":"Tassilo Klein","avatar_url":"https://avatars-next.observableusercontent.com/2bf705475e7b33a06a94a42c4a03afcc7ab7e029a47d2f819ecd72f07c1a1af5","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":1,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Transformer Language Model Dynamics","cells":[{"id":0,"value":"# Transformer Language Model Dynamics","pinned":true,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":8,"value":"html`\n\n \n Transformer Language Model Dynamics\n\n \n

\n\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":44,"value":"$45","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":14,"value":"d3 = require(\"d3@5\")","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":22,"value":"margin = ({top: 0, right: 0, bottom: 0, left: 0})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":34,"value":"width = 600 - margin.left - margin.right","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":42,"value":"height = 300 - margin.top - margin.bottom","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":70,"value":"//data = (await fetch(url)).json()\ndata = FileAttachment(\"highres_transformer.json\").json()","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":76,"value":"function kde(kernel, thresholds, data) {\n return thresholds.map(t => [t, d3.mean(data, d => kernel(t - d))]);\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":79,"value":"function epanechnikov(bandwidth) {\n return x => Math.abs(x /= bandwidth) <= 1 ? 0.75 * (1 - x * x) / bandwidth : 0;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W46","stdlib":"1"}} 44:"$43:body"