Published
Edited
Jul 9, 2020
3 forks
17 stars
Insert cell
md`# Final Version`
Insert cell
md`## Topic Modeling`
Insert cell
topicLavoro = FileAttachment("topic_lavoro.json").json()
Insert cell
topicLavoroGraph = radialTree(topicLavoro, "lavoro")
Insert cell
topicScuola = FileAttachment("topic_scuola.json").json()
Insert cell
topicScuolaGraph = radialTree(topicScuola, "scuola")
Insert cell
sentimentDataMap = FileAttachment("sentiment_hashtag.json").json()
Insert cell
myHashtags = ["#maturità2020","#Azzolina", "#università"]
Insert cell
hashtagSentimentChart = create_small_multiples(sentimentDataMap, myHashtags, "#fff5ee", stacked1)
Insert cell
smartworkingSentiment = sentimentTime(sentimentDataMap, "#smartworking", "#f9fcf5", stacked)
Insert cell
viewof hashtagSelected = {
const selected = select({options: ['#webinar','#elearning']})
return selected;
}
Insert cell
webinarElearningSentimentChart = sentimentTime(sentimentDataMap, hashtagSelected, "#f9fcf5", stacked)
Insert cell
lavoroEmotionPerTopic = FileAttachment("emotionPerTopicLavoro@2.json").json()
Insert cell
scuolaEmotionPerTopic = FileAttachment("emotionsPerTopicScuola.json").json()
Insert cell
radarChartLavoro = radar_chart(lavoroEmotionPerTopic, ["benefici", "flessibilità", "diritti e tutele", "privacy", "opportunità", "conversione digitale", "disuguaglianze di genere", "colleghi","videoconferenze", "riunioni"], ["diritti e tutele", "opportunità"], "#f9fcf5")
Insert cell
radarChartScuola = radar_chart(scuolaEmotionPerTopic, ["divario tecnologico", "diritto allo studio", "maturità", "difficoltà tecniche", "riapertura scuole", "organizzazione familiare", "strumenti digitali", "organizzazione didattica"],["divario tecnologico", "strumenti digitali"], "#fff5ee")
Insert cell
viewof stacked = select({options: ['no', 'si'], title: "Normalizzazione"})
Insert cell
viewof stacked1 = select({options: ['no', 'si'], title: "Normalizzazione"})
Insert cell
emotionPerWeekSmartWorking = FileAttachment("emotionsPerWeekSmart.json").json()
Insert cell
emotionPerWeekUniversità = FileAttachment("emotionsPerWeekUni.json").json()
Insert cell
emotionPerDaySmart = FileAttachment("emotionsPerDaySmart.json").json()
Insert cell
viewof emotionData = select({options: ['smartworking', 'didattica']})
Insert cell
emotionPerDayChart = emotionTimeline(emotionData == 'smartworking'? emotionPerDaySmart: emotionPerDayScuola)
Insert cell
emotionPerDayScuola = FileAttachment("emotionsPerDayScuola.json").json()
Insert cell
emotionPerDayScuolaChart = emotionTimeline(emotionPerDayScuola)
Insert cell
allEmotions = ["tristezza", "sorpresa", "gioia", "paura", "rabbia", "disgusto"]
Insert cell
getTimeLine = emotionMapPerDay => {
const listDate = Object.keys(emotionMapPerDay);
return [...new Set(listDate.flat())].sort(); //flat della lista precedente, set per prendere i valori unici quindi trasformo di nuovo in un array e ordino
}
Insert cell
getEmotionDictionary = emotionMapPerDay => {
let emotionDicPerWeek ={}
const timeLine = getTimeLine(emotionMapPerDay);
allEmotions.map(e => {
let emoList = [];
timeLine.map(t => {
emoList.push(emotionMapPerDay[t][e])
})
emotionDicPerWeek[e] = emoList;
})
return emotionDicPerWeek
}
Insert cell
emotionTimeline = emotionPerDate => {
const div = DOM.element('div');
const emotionDicPerWeek = getEmotionDictionary(emotionPerDate);
let dataT = []
allEmotions.map(emo => {
const trace = {
name:emo,
mode:'line',
line: {
width: 2,
shape: 'spline', // questo crea uno smooth delle linee
color: colorEmotion(emo),
opacity: 0.5,// qui potete passare una funzione per assegnare colori personalizzati
},
y: emotionDicPerWeek[emo],
x: Object.keys(emotionPerDate),
stackgroup: 'one',
groupnorm:'percent',
fillcolor: colorEmotion(emo),
fillopacity: 0.5,
mode: 'none',
text:emotionDicPerWeek[emo],
hoverinfo:"text",
}
dataT.push(trace);
})

const layout = {
title: ``,
showlegend: true,
line: {
width: 2,
shape: 'spline' // questo crea uno smooth delle linee
// color: 'red' // qui potete passare una funzione per assegnare colori personalizzati
},
width: 1000,
height: 500,
margin: {pad:5,bottom:100},
legend:{
orientation:'v'
},
paper_bgcolor:'rgba(0,0,0,0)',
plot_bgcolor:'rgba(0,0,0,0)',
// xaxis: {
// rangeselector: {
// buttons: [{
// step: 'all',
// }]},
// rangeslider: {}
// },
};
Plotly.newPlot(div, dataT, layout,{displayModeBar: false});
return div;
}
Insert cell
radar_chart = (emotions, fields, fieldsToVisualize, backgorund) => {
const data = fields.map(d => ({
type: 'scatterpolar',
r: Object.values(emotions[d]).concat(Object.values(emotions[d])[0]), // .concat(...) serve per copiare il primo valore della serie in fondo all'array. Serve per chiudere la line
theta: Object.keys(emotions[d]).concat(Object.keys(emotions[d])[0]),
fill: 'toself', // crea il riempimento
name: d,
opacity: 0.5,
line: {
width: 2,
shape: 'spline', // questo crea uno smooth delle linee
color: radialColor(d) // qui potete passare una funzione per assegnare colori personalizzati
},
marker: {
size: 6
},
visible: fieldsToVisualize.indexOf(d) === -1 ? 'legendonly': true,
// un template html per formattare il box visibile al passggio del mouse
hovertemplate: '<b>%{theta}</b>' + '<br>%{r:.2f}<br>' + "<extra></extra>"
}));

const layout = {
width: 700,
height: (width / 3) * 2,
polar: {
angularaxis: {
linewidth: 1,
color: 'gray',
showline: false
},
radialaxis: {
gridcolor: 'white',
gridwidth: 2,
visible: true,
range: [0.1, 0.2], // il range dell'asse [min,max]
color: 'gray',
showline: false
},
bgcolor: backgorund // colore di sfondo
},
plot_bgcolor:backgorund,
paper_bgcolor:backgorund,
};

const div = DOM.element('div');
Plotly.newPlot(div, data, layout);
return div;
}
Insert cell
create_small_multiples = (data, fields, background, stacked) => {
const wrapper = html`<div class="multiples"></div>`;

fields.forEach(f => {
const div = sentimentTime(data, f, background, stacked);

wrapper.append(div);
});

return wrapper;
}
Insert cell
sentimentTime = (sentimentDataObject, hashtag, background, stacked) => {
const fields = [
{
field: 'positive',
name: 'Positivi'
},
{
field: 'neutral',
name: 'Neutrali'
},
{
field: 'negative',
name: 'Negativi'
}
];

const data = fields.map(f => ({
x: Object.entries(sentimentDataObject[hashtag]).map(d => d[0]),
y: extractSentimentSeries(Object.entries(sentimentDataObject[hashtag]), f.field),
name: f.name,
stackgroup: 'single',
marker: {
color: colorSentiment(f.field)
},
line: {
'shape': 'spline'
},
groupnorm: stacked == 'no'? 'none': 'percent'
}));

const cOptions = {
margin: {
l: 100,
t: 30,
r: 10,
pad: 5
},
plot_bgcolor:background,
paper_bgcolor:background,
width: 600,
height: 400,
yaxis: {
tickfont: {
size: 10
}
}
};

const layout = {
title: `${hashtag}`,
...cOptions
};

const div = html`<div class='multiple'></div>`;
Plotly.newPlot(div, data, layout);
return div;
}
Insert cell
topicChart = data => {
const root = packTopic(data);
let focus = root;
let view;

const svg = d3.create("svg")
.attr("viewBox", `-${width / 2} -${height / 2} ${width} ${height}`)
.style("display", "block")
.style("margin", "0 -14px")
.style("background", 'white')
.style("cursor", "pointer")
.on("click", () => zoom(root));

const node = svg.append("g")
.selectAll("circle")
.data(root.descendants().slice(0))
.join("circle")
.attr("fill", d => colorTopic(d.depth + 2))
.attr("pointer-events", d => !d.children ? "none" : null)
.on("mouseover", function() { d3.select(this).attr("stroke", "#000"); })
.on("mouseout", function() { d3.select(this).attr("stroke", null); })
.on("click", d => focus !== d && (zoom(d), d3.event.stopPropagation()));

const label = svg.append("g")
.style("font", "10px sans-serif")
.attr("pointer-events", "none")
.attr("text-anchor", "middle")
.selectAll("text")
.data(root.descendants())
.join("text")
.style("fill-opacity", d => d.parent === root ? 1 : 0)
.style("display", d => d.parent === root ? "inline" : "none")
.text(d => d.data.name);

zoomTo([root.x, root.y, root.r * 2]);

function zoomTo(v) {
const k = width / v[2];

view = v;

label.attr("transform", d => `translate(${(d.x - v[0]) * k},${(d.y - v[1]) * k})`);
node.attr("transform", d => `translate(${(d.x - v[0]) * k},${(d.y - v[1]) * k})`);
node.attr("r", d => d.r * k);
}

function zoom(d) {
const focus0 = focus;

focus = d;

const transition = svg.transition()
.duration(d3.event.altKey ? 7500 : 750)
.tween("zoom", d => {
const i = d3.interpolateZoom(view, [focus.x, focus.y, focus.r * 2]);
return t => zoomTo(i(t));
});

label
.filter(function(d) { return d.parent === focus || this.style.display === "inline"; })
.transition(transition)
.style("fill-opacity", d => d.parent === focus ? 1 : 0)
.on("start", function(d) { if (d.parent === focus) this.style.display = "inline"; })
.on("end", function(d) { if (d.parent !== focus) this.style.display = "none"; });
}

return svg.node();
}
Insert cell
topicLavoro.name
Insert cell
topicLavoro.children.map(d=> d.children.reduce((a,b)=>a+parseFloat(b.value),0))
Insert cell
radialTree = (dataTree, topicName) => {

const svg = d3.select(DOM.svg(width, width))
.style("width", "100%")
.style("height", "auto")
.style("padding", "10px")
.style("box-sizing", "border-box")
.style("font", "18px sans-serif");
const g = svg.append("g");
const linkgroup = g.append("g")
.attr("fill", "none")
.attr("stroke", "#555")
.attr("stroke-opacity", 0.4)
.attr("stroke-width", 1.5);

const nodegroup = g.append("g")
.attr("stroke-linejoin", "round")
.attr("stroke-width", 3);

function newdata (animate = true) {
let root = tree(dataTree);
let links_data = root.links();
let links = linkgroup
.selectAll("path")
.data(links_data, d => d.source.data.name+"_"+d.target.data.name);
links.exit().remove();
let newlinks = links
.enter()
.append("path")
.attr("d", d3.linkRadial()
.angle(d => d.x)
.radius(0.1));

let t = d3.transition()
.duration(animate ? 400 : 0)
.ease(d3.easeLinear)
.on("end", function() {
const box = g.node().getBBox();
svg.transition().duration(1000).attr("viewBox", `${box.x} ${box.y} ${box.width} ${box.height}`);
});
let alllinks = linkgroup.selectAll("path")
alllinks
.transition(t)
.attr("stroke", d => d.target.data.children? radialColor(d.target.data.name) : radialColor(d.source.data.name))
.attr("d", d3.linkRadial()
.angle(d => d.x)
.radius(d => d.y));

let nodes_data = root.descendants().reverse();
let nodes = nodegroup
.selectAll("g")
.data(nodes_data, function (d) {
if (d.parent) {
return d.parent.data.name+d.data.name;
}
return d.data.name});
nodes.exit().remove();

let newnodes = nodes
.enter().append("g");
let allnodes = animate ? nodegroup.selectAll("g").transition(t) : nodegroup.selectAll("g");
allnodes
.attr("transform", d => `
rotate(${d.x * 180 / Math.PI - 90})
translate(${d.y},0)
`);
newnodes.append("circle")
.attr("r", d => d.data.children ? scaleCircle(topicName, d.data.children.reduce((accumulator, current) => accumulator + parseFloat(current.value),0)) : scaleCircle(topicName,d.data.value))
.on ("click", function (d) {
let altChildren = d.data.altChildren || [];
let children = d.data.children;
d.data.children = altChildren;
d.data.altChildren = children;
newdata ();
});
nodegroup.selectAll("g circle").attr("fill", d => d.data.children ? radialColor(d.data.name) : radialColor(d.parent.data.name))
.attr("fill-opacity",0.9)
.attr("stroke",d => d.data.children ? radialColor(d.data.name) : radialColor(d.parent.data.name))
.attr("stroke-width",0.5)

newnodes.append("text")
.attr("dy", "0.31em")
.text(d => d.data.name)
.clone(true).lower()
.attr("stroke", "white");
nodegroup.selectAll("g text")
.attr("x", d => d.x < Math.PI === !d.children ? scaleCircle(topicName, d.data.value)+3: -scaleCircle(topicName,d.data.value)-3)
.attr("text-anchor", d => d.x < Math.PI === !d.children ? "start" : "end")
.attr("transform", d => d.x >= Math.PI ? "rotate(180)" : null);

}
newdata (false);
document.body.appendChild(svg.node());

const box = g.node().getBBox();
//box.width = box.height = Math.max(box.width, box.height)*1.2;
svg.remove()
.attr("width", box.width)
.attr("height", box.height)
.attr("viewBox", `${box.x} ${box.y} ${box.width} ${box.height}`);

return svg.node();
}
Insert cell
lavoroWord2Vec = FileAttachment("lavoro_wb.json").json()
Insert cell
wordcloudresult = wordCloud(getWordCloudData(lavoroWord2Vec), 'label', 'value', 35, 'group', Object.keys(lavoroWord2Vec))
Insert cell
scuolaWord2Vec = FileAttachment("scuola_wb.json").json()
Insert cell
wordCloudscuole = wordCloud(getWordCloudData(scuolaWord2Vec), 'label', 'value', 30, 'group', Object.keys(scuolaWord2Vec))
Insert cell
wordCloudscuole.split()
Insert cell
wordcloudresult.split()
Insert cell
getWordCloudData = word2VecMap => {
let arrayNodes = []
Object.entries(word2VecMap).map((d,i) => {
Object.entries(d[1]).map(t => {
let mynode = {}
mynode["group"] = i
mynode["label"] = t[0]
mynode["value"] = t[1]*t[1]
arrayNodes.push(mynode)
});
})
return arrayNodes
}
Insert cell
wordCloud = (data, word, frequency, maxSize, type, topics) => {
if (type===undefined) {
data.map(d => d.type = 0)
}
//const root = partition(data_wc);
const heightf = 500;
const widthf = 1275;
const height = 450;
const width = 1350;
const margin = {
t: 0,
l: 120,
r: 120,
b: 0,
}
const padding = 2;
// create the SVG container
let ext = d3.extent(data, d => d[type])
//console.log(ext)
let xScale = d3.scaleLinear()
.domain(ext)
.range([margin.l, widthf - margin.l - margin.r]);
let svg = d3.select(html`<svg></svg>`)
.attr('width', widthf)
.attr('height', heightf);
const colors = d3.schemeSet1
.slice(0, new Set(data.map(d=> d[type])).size)
// color text black or white according to the luminance value of the node
const luminance = 30;
const textColor = d3.scaleQuantile().range(["#fff", "#000"]).domain([0, luminance, 100]);
// define scale for radius
const r = d3.scaleSqrt()
.domain([0, d3.max(data, d => d[frequency])])
.range([0, maxSize]);
// define the simualtion engine
var simulation = d3.forceSimulation(data)
.force("x", d3.forceX(widthf/2).strength(0.008))
.force("y", d3.forceY(heightf/2).strength(0.08))
.force("collide", d3.forceCollide().radius(d=> r(d[frequency]) + padding).iterations(5))
.force('center', d3.forceCenter(widthf/2, 1.0*heightf/2))
.alpha(0.7);
let x = d3
.scalePoint()
.domain(topics)
.range([margin.l, widthf - margin.l - margin.r]);
///////////////////

function split() {
simulation
.force("x", d3.forceX(d=> xScale(d[type])).strength(0.8))
.force("y", d3.forceY(heightf/2).strength(0.5))
.force("collide", d3.forceCollide().radius(d=> r(d[frequency]) + padding).iterations(5))
.alpha(0.5)
.restart();
}

/////////////
// create a layer or group
let gBubble = svg
.selectAll('gBubble')
.data(data);
gBubble.exit().remove();
let bubble = gBubble.enter()
.append('g')
.classed('gBubble', true)
.attr('id',d => d[word]);
bubble
.append('circle')
.attr('r', d => r(d[frequency]))
.attr('fill',d => radialColor(d[type]))
.attr('fill-opacity', 0.4)
.attr('stroke', d => radialColor[+d[type]])
.attr('stroke-width', 1)
.style("cursor", "pointer");

let text= bubble
.append('text');
const textLabels = text
.text( d => (d[word]))
.style('text-anchor','middle')
.attr("dominant-baseline", "central")
.attr('font-family', 'sans-serif')
.attr('font-size', '10px' )
// .attr('font-size', d => r(d[frequency])/2.5+'px' )
.attr('font-weight','normal')
.attr('fill', d => textColor(d3.hcl(colors[+d[type]]).l))
.style("cursor", "pointer");
const xaxis = svg
.append("g")
.attr("transform", `translate(0,${height - 50})`)
.call(d3.axisBottom(x));
gBubble = gBubble
.merge(bubble);
gBubble.call(drag(simulation));
simulation.nodes(data)
.on('tick', () => {
gBubble
.attr('transform', d => 'translate('+ (d.x) +','+ (d.y)+')');
})

return Object.assign(svg.node(), { split });
}
Insert cell
drag = simulation => {
function dragstarted(d) {
if (!d3.event.active) simulation.alphaTarget(0.3).restart();
d.fx = d.x;
d.fy = d.y;
}
function dragged(d) {
d.fx = d3.event.x;
d.fy = d3.event.y;
}
function dragended(d) {
if (!d3.event.active) simulation.alphaTarget(0);
d.fx = null;
d.fy = null;
}
return d3.drag()
.on("start", dragstarted)
.on("drag", dragged)
.on("end", dragended);
}
Insert cell
radius = 517.7777777777777
Insert cell
d3.scaleSequential().domain([1,10])(2)
Insert cell
radialColor("prova")
Insert cell
d3.schemeCategory10
Insert cell
radialColor = d3.scaleOrdinal(["#ffd65a","#CF8140","#9e2b25","#4f1631","#00364B","#6EB3B7","#7a918d","#dfe2e1","#4a9554"]);
Insert cell
tree = data => d3.tree()
.size([2 * Math.PI, radius])
.separation((a, b) => (a.parent == b.parent ? 1 : 3) / a.depth)
(d3.hierarchy(data))
Insert cell
scaleCircle = (name, value) => {
return d3.scaleSqrt().domain([0,maxValue(name)]).range([0,30])(value); // min max dei valori che puà assumere il cerchio.
}
Insert cell
scaleCircle("lavoro", 30)
Insert cell
maxValue("lavoro")
Insert cell
maxValue= name => {
const topics = name === 'scuola'? topicScuola : topicLavoro;
return d3.max(d3.values(topics)[1].map(d=> d.children.map(e=>+e.value)).flat())
}
Insert cell
extractSentimentSeries = (mydata, field) =>
mydata.map((d, i) => d[1][field])
Insert cell
width = 932
Insert cell
height = width
Insert cell
packTopic = data => d3.pack()
.size([width, height])
.padding(3)
(d3.hierarchy(data)
.sum(d => d.value)
.sort((a, b) => b.value - a.value))
Insert cell
colorWord = {
const scale = d3.scaleOrdinal(d3.schemeCategory10);
return d => scale(d.group);
}
Insert cell
colorTopic = d3.scaleSequential().domain([1,10])
.interpolator(d3.interpolatePuRd);
Insert cell
colorSentiment("rabbia")
Insert cell
colorSentiment = d3
.scaleOrdinal()
.domain(d3.keys(attrColors))
.range(d3.values(attrColors))
Insert cell
attrColors = ({
positive: '#4a9554',
negative: '#9e2b25',
neutral: '#ffd65a',
})
Insert cell
colorEmotion = d3.scaleOrdinal()
.domain(d3.keys(attrEmoColors))
.range(d3.values(attrEmoColors));
Insert cell

attrEmoColors = ({
gioia: '#4a9554',
rabbia: '#9e2b25',
sorpresa: '#ffd65a',
tristezza: '#7A918D',
disgusto: '#6EB3B7',
paura: '#00364b'
})
Insert cell
html`<style>
div.multiples{
text-align: center;
display:flex;
}

div.multiple {

flex: 1;
}
</style>`
Insert cell
md`## APPENDIX`
Insert cell
d3 = require("d3@5")
Insert cell
Plotly = require("https://cdn.plot.ly/plotly-latest.min.js")
Insert cell
import { select, slider } from "@jashkenas/inputs"
Insert cell
import {swatches} from "@d3/color-legend"
Insert cell

One platform to build and deploy the best data apps

Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.
Learn more