2b:null 2c:["$","title",null,{"children":"News? | Yin Shanyang | Observable"}] 31:T662,{ const data = countries .map((country) => ({ country, count: articles .filter(({ source }) => source.location.type === 'country' ? source.location.label.eng === country : source.location.country.label.eng === country ).length, score: articles .filter(({ source }) => source.location.type === 'country' ? source.location.label.eng === country : source.location.country.label.eng === country ) .reduce((memo, { wgt }) => memo + wgt, 0) })) const scales = { x: d3.scaleLinear() .domain([0, d3.extent(data.map(({ score }) => score))[1]]) .range([0, 320]) } return html`

${ data.map(({ country, count, score }) => html`

${country} [${count}] ${score}

`) }

` }32:T65e,{ const data = state.country && state.concept ? articles .filter(({ source }) => source.location.type === 'country' ? source.location.label.eng === state.country : source.location.country.label.eng === state.country ) .filter(({ concepts }) => concepts.map(({ label }) => label.eng).includes(state.concept)) : [] if (sort === 'social') { data.sort((a, b) => a.wgt > b.wgt ? -1 : a.wgt < b.wgt ? 1 : 0) } else if (sort === 'date') { data.sort((a, b) => a.date > b.date ? -1 : a.date < b.date ? 1 : 0) } else {} return html` ${ data.map((article) => html`

${article.title}

${article.url}
${article.date} ${ Object.keys(article.shares) .map((key) => ({ key, value: article.shares[key] })) .map(({ key, value }) => html` ${key}: ${value} `) }
${article.body.slice(0, 300)}…

`) } ` }33:T1032,// should return nothing, appends the svg to the selection renderTsne = (selection, data, onSelect) => { // generate embeddings const concepts = data .map(({ concepts }) => concepts) .reduce((memo, d) => [ ...memo, ...d ], []) .reduce((memo, { label }) => memo.includes(label.eng) ? memo : [ ...memo, label.eng ], []) .sort((a, b) => a > b ? 1 : a < b ? -1 : 0) const embeddings = concepts.map(() => concepts.slice().fill(0)) data.forEach((article) => { const { wgt } = article article.concepts.forEach((concept) => { const others = article.concepts.filter((d) => d !== concept) others.forEach((other) => { embeddings[concepts.indexOf(concept.label.eng)][concepts.indexOf(other.label.eng)] += 1 // embeddings[concepts.indexOf(concept.label.eng)][concepts.indexOf(other.label.eng)] += wgt }) }) }) // create model const model = new tSNE.tSNE({ dim: 2, perplexity: 15 }) model.initDataRaw(embeddings) // render chart const width = 360 const height = 360 const margin = 0 const nodes = concepts.map((label) => ({ label, score: 0, x: width / 2 + margin, y: height / 2 + margin })) nodes.iterations = 300 nodes.step = 0 data.forEach((article) => { const { wgt } = article article.concepts.forEach((concept) => { nodes[concepts.indexOf(concept.label.eng)].score += concept.score * wgt }) }) const scales = { r: d3.scaleSqrt() .domain([0, d3.extent(nodes.map(({ score }) => score))[1]]) .range([0, 16]) } const svg = selection.append('svg') .style('border', '1px solid #999') .attr('width', width + 2 * margin) .attr('height', height + 2 * margin) const layer = svg.append('g') const node = layer.selectAll('g').data(nodes) .enter().append('g') .attr('transform', ({ x, y }) => `translate(${x}, ${y})`) .each(function (node) { const _this = d3.select(this) _this.append('text').datum(node) .attr('font-size', ({ score }) => scales.r(score)) .attr('text-anchor', 'middle') .attr('dy', '.35em') .text(({ label }) => label) }) .on('click', ({ label }) => onSelect(label)) const counter = svg.append('text').datum(nodes) .attr('transform', `translate(${width - 8}, ${8 + 10})`) .attr('font-family', 'sans-serif') .attr('font-size', 10) .attr('text-anchor', 'end') .text(({ step, iterations }) => `${step}/${iterations}`) // interaction, zoom const zoom = d3.zoom() .scaleExtent([0.5, Infinity]) // .translateExtent([[-100, -100], [width + 100, height + 100]]) .clickDistance(10) .on('zoom', zoomed) svg.call(zoom) function zoomed () { layer.attr('transform', d3.event.transform) node.each(function (node) { d3.select(this).select('text') .attr('transform', `scale(${ 1 / d3.event.transform.k })`) }) } // iterate model and update svg function update () { model.step() const positions = model.getSolution() const scales = { // x: d3.scaleLinear() // .domain(d3.extent(positions.map(([ x ]) => x))) // .range([margin, width + margin]), // y: d3.scaleLinear() // .domain(d3.extent(positions.map(([ x, y ]) => y))) // .range([margin, height + margin]) x: d3.scaleLinear() .domain(d3.extent(positions.map(([ x ]) => x))) .range([-width * 0.5, width * 1.5]), y: d3.scaleLinear() .domain(d3.extent(positions.map(([ x, y ]) => y))) .range([-height * 0.5, height * 1.5]), } nodes.forEach((node, index) => { node.x = scales.x(positions[index][0]) node.y = scales.x(positions[index][1]) }) node .attr('transform', ({ x, y }) => `translate(${x}, ${y})`) counter .attr('visibility', ({ step, iterations }) => step === iterations ? 'hidden' : 'visible') .text(({ step, iterations }) => `${step}/${iterations}`) } function loop () { if (nodes.step === nodes.iterations) return update() nodes.step++ setTimeout(() => loop(), 0) } loop() return }34:Tb29,// should return nothing, appends the svg to the selection renderTrends = (selection, data, onSelect) => { // generate embeddings const concepts = data .map(({ concepts }) => concepts) .reduce((memo, d) => [ ...memo, ...d ], []) .reduce((memo, { label }) => memo.includes(label.eng) ? memo : [ ...memo, label.eng ], []) .sort((a, b) => a > b ? 1 : a < b ? -1 : 0) let scores = concepts.map(() => []) data.forEach((article) => { const { date, wgt } = article article.concepts.forEach(({ type, label, score }) => { const ref = scores[concepts.indexOf(label.eng)] ref.push({ date: new Date(date), social: wgt, score: score }) ref.label = label.eng ref.type = type }) }) scores.forEach((score) => { score.total = score.reduce((memo, { score, social }) => memo + score * social, 0) }) scores.sort((a, b) => a.total > b.total ? -1 : a.total < b.total ? 1 : 0) scores = scores.slice(0, 40) const width = 360 const height = 16 const scales = { x: d3.scaleTime() .domain( d3.extent( articles.map(({ date }) => date) ).map((d) => new Date(d)) ) .range([0, width]), y: d3.scaleLinear() .range([height, 0]) } const histogram = d3.histogram() .value(({ date }) => date) .domain(scales.x.domain()) .thresholds(scales.x.ticks(d3.timeDay)) const bins = scores .map((score) => histogram(score)) .map((bin, index) => { const series = bin.map((d) => { const date = d.x0 const value = d.reduce((memo, { social, score }) => memo + social * score, 0) return { date, value } }) const { label, type, total } = scores[index] series.label = label series.type = type series.total = total return series }) scales.y.domain([ 0, d3.extent( bins.map((bin) => bin.map(({ value }) => value)).reduce((a, b) => [ ...a, ...b ]) )[1] ]) const area = d3.area() .x(({ date }) => scales.x(date)) .y1(({ value }) => scales.y(value)) .y0(scales.y(0)) .curve(d3.curveCatmullRom.alpha(0.5)) const container = selection.append('div') .style('width', `${width}px`) const plots = container.selectAll('svg').data(bins) .enter().append('svg') .attr('width', width) .attr('height', height) .each(function (bin) { const _this = d3.select(this) const path = _this.append('path').datum(bin) .attr('d', area) .attr('fill', '#999') const label = _this.append('text').datum(bin) .attr('dy', '1em') .attr('text-anchor', 'end') .attr('x', width) .style('font-size', '12px') .text(({ label }) => label) }) .on('click', ({ label }) => onSelect(label)) return }35:T2d6b,tSNE = { // create main global object var tsnejs = tsnejs || { REVISION: 'ALPHA' }; (function(global) { "use strict"; // utility function var assert = function(condition, message) { if (!condition) { throw message || "Assertion failed"; } } // syntax sugar var getopt = function(opt, field, defaultval) { if(opt.hasOwnProperty(field)) { return opt[field]; } else { return defaultval; } } // return 0 mean unit standard deviation random number var return_v = false; var v_val = 0.0; var gaussRandom = function() { if(return_v) { return_v = false; return v_val; } var u = 2*Math.random()-1; var v = 2*Math.random()-1; var r = u*u + v*v; if(r == 0 || r > 1) return gaussRandom(); var c = Math.sqrt(-2*Math.log(r)/r); v_val = v*c; // cache this for next function call for efficiency return_v = true; return u*c; } // return random normal number var randn = function(mu, std){ return mu+gaussRandom()*std; } // utilitity that creates contiguous vector of zeros of size n var zeros = function(n) { if(typeof(n)==='undefined' || isNaN(n)) { return []; } if(typeof ArrayBuffer === 'undefined') { // lacking browser support var arr = new Array(n); for(var i=0;i 1e-7) Hhere -= pj * Math.log(pj); } // adjust beta based on result if(Hhere > Htarget) { // entropy was too high (distribution too diffuse) // so we need to increase the precision for more peaky distribution betamin = beta; // move up the bounds if(betamax === Infinity) { beta = beta * 2; } else { beta = (beta + betamax) / 2; } } else { // converse case. make distrubtion less peaky betamax = beta; if(betamin === -Infinity) { beta = beta / 2; } else { beta = (beta + betamin) / 2; } } // stopping conditions: too many tries or got a good precision num++; if(Math.abs(Hhere - Htarget) < tol) { done = true; } if(num >= maxtries) { done = true; } } // console.log('data point ' + i + ' gets precision ' + beta + ' after ' + num + ' binary search steps.'); // copy over the final prow to P at row i for(var j=0;j 0 ? 1 : x < 0 ? -1 : 0; } var tSNE = function(opt) { var opt = opt || {}; this.perplexity = getopt(opt, "perplexity", 30); // effective number of nearest neighbors this.dim = getopt(opt, "dim", 2); // by default 2-D tSNE this.epsilon = getopt(opt, "epsilon", 10); // learning rate this.iter = 0; } tSNE.prototype = { // this function takes a set of high-dimensional points // and creates matrix P from them using gaussian kernel initDataRaw: function(X) { var N = X.length; var D = X[0].length; assert(N > 0, " X is empty? You must have some data!"); assert(D > 0, " X[0] is empty? Where is the data?"); var dists = xtod(X); // convert X to distances using gaussian kernel this.P = d2p(dists, this.perplexity, 1e-4); // attach to object this.N = N; // back up the size of the dataset this.initSolution(); // refresh this }, // this function takes a given distance matrix and creates // matrix P from them. // D is assumed to be provided as a list of lists, and should be symmetric initDataDist: function(D) { var N = D.length; assert(N > 0, " X is empty? You must have some data!"); // convert D to a (fast) typed array version var dists = zeros(N * N); // allocate contiguous array for(var i=0;i date)\n )\n return md`${extent[0]} – ${extent[1]}`\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":83,"value":"md`\n## Count of Articles and Total Social Shares per Country\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":86,"value":"$31","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":141,"value":"md`\n## Concepts by Country, weighted by Social Score\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":312,"value":"{\n const container = d3.select(DOM.element('div'))\n \n const country = container.selectAll('div').data(countries)\n .enter().append('div')\n .style('display', 'inline-block')\n .style('vertical-align', 'top')\n .style('margin', '16px')\n .each(function (country) {\n const _this = d3.select(this)\n const header = _this.append('h3').datum(country)\n .text((d) => d)\n const data = articles\n .filter(({ source }) =>\n source.location.type === 'country'\n ? source.location.label.eng === country\n : source.location.country.label.eng === country\n )\n _this.call(renderTsne, data, handleSelect(country))\n _this.call(renderTrends, data, handleSelect(country))\n })\n \n return container.node()\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":559,"value":"{\n return state.country && state.concept\n ? md`## Articles from ${state.country} related to ${state.concept}`\n : md`## Articles`\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":663,"value":"{\n const options = ['most viral to least viral', 'most recent to least recent']\n const select = DOM.select(options)\n d3.select(select).on('change', () => {\n if (select.value === options[0]) { mutable sort = 'social' }\n else { mutable sort = 'date' }\n })\n return select\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":585,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":358,"value":"md`\n---\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":550,"value":"md`\n### state\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":552,"value":"mutable state = ({\n country: null,\n concept: null\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":556,"value":"mutable sort = 'shares'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":158,"value":"md`\n### utils\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":161,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":437,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":571,"value":"handleSelect = (country) => (concept) => {\n mutable state = { country, concept }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":10,"value":"md`\n### data\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":148,"value":"countries = ['Belgium', 'France', 'Germany', 'Italy', 'Luxembourg', 'Netherlands', 'Russia', 'Spain', 'Switzerland', 'United Kingdom']\n// countries = ['Germany']","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":15,"value":"articles = JSON.parse(await Files.text(file))\n .filter(({ sim }) => sim > 0)\n .filter(({ isDuplicate }) => !isDuplicate)\n .filter(({ source }) => source.location)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":57,"value":"locations = articles\n .map(({ source }) =>\n source.location.type === 'country'\n ? source.location.label.eng\n : source.location.country.label.eng\n )\n .reduce((memo, country) => {\n const ref = memo.find((d) => d.country === country)\n if (ref) {\n ref.count++\n return memo\n }\n else {\n return [ ...memo, { country, count: 1 } ]\n }\n }, [])\n .sort((a, b) => a.country > b.country ? 1 : a.country < b.country ? -1 : 0)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":65,"value":"concepts = articles\n .map(({ sim, concepts }) => concepts.map(({ type, label, score }) => ({ type, label: label.eng, score: score * sim })))\n .reduce((memo, d) => [ ...memo, ...d ], [])\n .reduce((memo, d) => {\n const ref = memo.find(({ concept }) => concept === d.label)\n if (ref) {\n ref.score += d.score\n return memo\n }\n else {\n return [ ...memo, { concept: d.label, score: d.score } ]\n }\n }, [])\n .sort((a, b) => a.score > b.score ? -1 : a.score < b.score ? 1 : 0)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":6,"value":"md`\n### imports\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":8,"value":"d3 = require('d3')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":163,"value":"$35","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"}],"files":"$W36","stdlib":"1","titling":"auto"}} 30:"$2f:body"