2e:null 2f:["$","title",null,{"children":"Part 3 - What Are We Talking About? | Misha Tsvelik | Observable"}] 34:T579,word_cloud_chart = html ` Timeline ${timeline_chart} Sentiment ${sentiment_byweekyear_chart} Post volume ${viewof rangesliderdemo_exploded}

Entities found in '${word_cloud_category_input}' category posts, dates: ${date_range_selected}

${viewof word_cloud_category_input}

${viewof word_cloud_threshold_input}

${senders_stacked_bar_chart}

`35:T6bd,md `### Appendix: Notes

Sentiment scores and categorisations are made on a per post basis. So the limitation is that context from posts before is not taken into account.

The green for positive /red for negative weekly sentiment over time chart under the timeline has a minimum threshold of n=30 posts per week to calculate the sentiment score average. This is to smooth out the outliers.

Here I am making use of the following Google Natural Language services. Further information can be found [here](https://cloud.google.com/natural-language/docs/basics)

Sentiment analysis inspects the given text and identifies the prevailing emotional opinion within the text, especially to determine a writer's attitude as positive, negative, or neutral. Overall Sentiment Score has two components Sentiment Score (positive or negative) and Sentiment Magnitude (always positive and increases with the length of the post). This way long posts have a higher weighted score than short.
Entity analysis inspects the given text for known entities (Proper nouns such as public figures, landmarks, and so on. Common nouns such as restaurant, stadium, and so on.) and returns information about those entities.
Behind the scenes, to get the sentiment scores, Entity sentiment analysis inspects the given text for known entities (proper nouns and common nouns), returns information about those entities, and identifies the prevailing emotional opinion of the entity within the text, especially to determine a writer's attitude toward the entity as positive, negative, or neutral.
Content classification analyzes text content and returns a content category for the content.

`36:T1397,timeline_chart_gen = Highcharts.chart('timeline', { chart: { zoomType: 'x', type: 'timeline', height: 130, marginLeft:15, marginRight:0 }, credits: { enabled: false }, xAxis: { type: 'datetime', visible: false }, yAxis: { gridLineWidth: 1, title: null, labels: { enabled: false } }, legend: { enabled: false }, title: { text: null }, subtitle: { text: null }, tooltip: { style: { width: 300 }, pointFormatter: function(d,i){ let rawDate = new Date(this.x); return `${rawDate.getDate() + '/' + (rawDate.getMonth()+1) + '/' + rawDate.getFullYear()}: ${this.description}` } }, series: [{ dataLabels: { allowOverlap: true, format: '● ' + '{point.x:%d %b %Y}
{point.label}', alternate: true, distance: -50 }, marker: { symbol: 'circle' }, data: [{ x: Date.UTC(main_file[0]['datetime_asdate'].getFullYear(), main_file[0]['datetime_asdate'].getMonth(), main_file[0]['datetime_asdate'].getDate()), name: 'Book club starts', label: 'Book club starts', description: "First post by Will." }, { x: Date.UTC(2016, 5, 10), name: 'Euro 2016', label: 'Euro 2016', description: "Euro 2016 Championships starts in France and lasts for a month." }, { x: Date.UTC(2016, 5, 23), name: 'EU Referndum', label: 'EU Referndum', description: "UK votes to leave the EU." }, { x: Date.UTC(2016, 10, 8), name: 'US Election, Trump wins', label: 'US Election, Trump wins', description: "US Election starts, Donald Trump voted in as US President." }, { x: Date.UTC(2017, 5, 8), name: 'UK Election, May wins', label: 'UK Election, May wins', description: "UK Election starts, Theresa May voted in." }, { x: Date.UTC(2018, 5, 14), name: '2018 World Cup', label: '2018 World Cup', description: "2018 World Cup starts in Russia and goes on for a month." }, { x: Date.UTC(2019, 11, 12), name: 'UK Election, Johnson wins', label: 'UK Election, Johnson wins', description: "UK Election starts, Boris Johnson voted in." }, /*{ x: Date.UTC(2020, 0, 31), name: 'First 2 cases of coronavirus in UK', label: 'First 2 cases of coronavirus in UK', description: "First 2 cases of coronavirus confirmed in UK." },*/ { x: Date.UTC(2020, 1, 22), name: 'Fury vs Wilder 2', label: 'Fury vs Wilder 2', description: "Fury vs Wilder 2 boxing." }, { x: Date.UTC(2020,2,23), name: '1st UK lockdown starts', label: '1st UK lockdown starts', description: "1st hard UK lockdown starts." }, /*{ x: Date.UTC(2020,4,10), name: '1st UK lockdown eases', label: '1st UK lockdown eases', description: "!st lockdown eased and regional traffic light system introduced." },*/ { x: Date.UTC(2020, 10, 3), name: 'US Election, Biden wins', label: 'US Election, Biden wins', description: "US Election starts, Joe Biden eventually voted in as US President." }, /*{ x: Date.UTC(2020, 10, 5), name: '2nd UK lockdown starts', label: '2nd UK lockdown starts', description: "2nd UK lockdown starts." },*/ /*{ x: Date.UTC(2020, 11, 2), name: '2nd UK lockdown eases', label: '2nd UK lockdown eases', description: "2nd UK lockdown eases." },*/ /*{ x: Date.UTC(2020, 11, 20), name: '3rd UK lockdown starts', label: '3rd UK lockdown starts', description: "3rd UK lockdown starts, with tier 4 implemented in London and South East, rest of England follows shortly." },*/ { x: Date.UTC(main_file[main_file.length-1]['datetime_asdate'].getFullYear(), main_file[main_file.length-1]['datetime_asdate'].getMonth(), main_file[main_file.length-1]['datetime_asdate'].getDate()), name: 'End', label: 'End', description: "Last point in dataset." }] }] }); 37:T45d,entities_per_topic_map = { const entities_to_array = function(input){ return input.replace("['","").replace("']","").split("', '").filter(el => el != "[]"); } const list = d3.groups(rangesliderdemo_exploded.data, d => d['Google classification category - top'] ).map(function(e){ let topic = e[0]; let by_topic_data = e[1];//['Google entity names']; return by_topic_data.map(function(el) { return { 'topic': topic, 'entities': entities_to_array(el['Google entity names']) } }) }) .reduce(function(arr, elem) { return arr.concat(elem); }, []); const all_list = rangesliderdemo_exploded.data.map(function(e){ return { 'topic': "All", 'entities': entities_to_array(e['Google entity names']) } }) .reduce(function(arr, elem) { return arr.concat(elem); }, []); let grouped_list = d3.rollup( all_list.concat(list), v => v.reduce(function(arr, elem) { return arr.concat(elem.entities); }, []), d => d.topic ); return grouped_list; }38:T417,indexes_per_topic_map = { const entities_to_array = function(input){ return input.replace("['","").replace("']","").split("', '").filter(el => el != "[]"); } const list = d3.groups(rangesliderdemo_exploded.data, d => d['Google classification category - top'] ).map(function(e){ let topic = e[0]; let by_topic_data = e[1];//['Google entity names']; return by_topic_data.map(function(el) { return { 'topic': topic, 'indexes': el['index'] } }) }) .reduce(function(arr, elem) { return arr.concat(elem); }, []); const all_list = rangesliderdemo_exploded.data.map(function(e){ return { 'topic': "All", 'indexes': e['index'] } }) .reduce(function(arr, elem) { return arr.concat(elem); }, []); let grouped_list = d3.rollup( all_list.concat(list), v => v.reduce(function(arr, elem) { return arr.concat(elem.indexes); }, []), d => d.topic ); return grouped_list; }39:T502,senders_stacked_bar_chart_gen = Highcharts.chart('senders_stacked_bar', { chart: { type: 'column', height: word_cloud_height }, title: { text: null }, credits: { enabled: false }, xAxis: { categories: ['sender'], labels: { enabled: true }, opposite:true }, yAxis: { min: 0, max: 1, gridLineWidth:0, lineWidth: 0, minorGridLineWidth: 0, title: { text: 'Sender breakdown' }, visible: false }, legend: { enabled: false }, tooltip: { enabled: false }, plotOptions: { column: { stacking: 'normal', pointPadding: 0, groupPadding: 0, dataLabels: { enabled: true, color: '#FFFFFF', formatter: function () { return this.series.name + ':' + Math.round(100 * this.y / this.total) + '%'; } } }, }, series:stacked_bar_array }); 3a:T40b,vocab_chart_gen = Highcharts.chart('unique_words', { chart: { type: 'column', height: 250 }, xAxis: { categories: vocab_array.sort(function(a,b){ return b[vocab_input] - a[vocab_input]; }).map(function(e){ return e['sender']}) }, title: { text: null }, credits: { enabled: false }, legend: { enabled: false }, yAxis: { title: { text: `${vocab_input}` }, gridLineWidth:0, lineWidth: 0, minorGridLineWidth: 0, labels: { enabled: false } }, plotOptions: { column: { colorByPoint: true }, series: { dataLabels: { enabled: true, pointFormat: vocab_input == 'total unique words'? "{point.y:.0f}": "{point.y:.1f}" } } }, series: [ { name: 'Sender', data: vocab_array.map(function(e){ return { y: Number(e[vocab_input]), color: colors[e['sender']] } }).sort(function(a,b){ return b["y"] - a["y"]; }) } ] });3b:T482,vocab_array = { let vocabulary_richness_index = d3.rollups( main_file, v => d3.max(v.map(d => d['avg_unique_words_per_post'])), d => d['sender'] ) .map(function(e){ return { 'sender': e[0], 'vocabulary richness index (total unique words / total posts)': e[1] } }); let word_count_per_post = d3.rollups( main_file, v => d3.mean(v.map(d => d['words'])), d => d['sender'] ) .map(function(e){ return { 'sender': e[0], 'average word count per post': e[1] } }); /*let unique_to_words_ratio = d3.rollups( main_file, v => d3.mean(v.map(d => d['unique words']/d['words'])), d => d['sender'] ) .map(function(e){ return { 'sender': e[0], 'ratio': e[1] } });*/ let output = word_count_per_post.map(function(e,i){ //e['unique:word count ratio'] = unique_to_words_ratio[i]['ratio']; e['average word count per post'] = word_count_per_post[i]['average word count per post']; e['vocabulary richness index (total unique words / total posts)'] = vocabulary_richness_index[i]['vocabulary richness index (total unique words / total posts)'] return e; }) return output; }3c:T13af,post_count_heatmap_chart_gen = { const data = topic_posts_objects_array; const id = 'post_count_heatmap_d3'; // set the dimensions and margins of the graph const margin = {top: 24, right: 5, bottom: 20, left: 150}, heatmap_width = width - margin.left - margin.right, //width declared outside of this block of code heatmap_height = 500 - margin.top - margin.bottom; // append the svg object to the body of the page d3.select("#" + id).html(''); let svg = d3.select("#" + id) .append("svg") .attr("width", heatmap_width + margin.left + margin.right) .attr("height", heatmap_height + margin.top + margin.bottom) .append("g") .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); // Labels of row and columns -> unique identifier of the column called 'group' and 'variable' //these vars are for specifying x / y axis values and cell color range only let myGroups = Array.from(new Set(data.map(d => d.group))) .sort((a,b) => members_by_post_volume_array.indexOf(a) > members_by_post_volume_array.indexOf(b)?1:-1); let myVars = Array.from(new Set(data.map(d => d.variable))) .sort((a,b) => post_categories_top15_array.indexOf(a) > post_categories_top15_array.indexOf(b)?1:-1); let myValues = Array.from(new Set(post_count_heatmap_input == 'as count'? data.map(d => d.value): data.map(d => d.value/post_classifiable_volume_map.get(d.group)))); // Build X scales and axis: let x = d3.scaleBand() .range([ 0, heatmap_width ]) .domain(myGroups) .padding(0.05); svg.append("g") .style("font-size", 15) .attr("transform", "translate(0," + heatmap_height + ")") .call(d3.axisBottom(x).tickSize(0)) .select(".domain").remove() // Build Y scales and axis: let y = d3.scaleBand() .range([ heatmap_height, 0 ]) .domain(myVars) .padding(0.05); svg.append("g") .style("font-size", 15) .call(d3.axisLeft(y).tickSize(0)) .select(".domain").remove() // Build color scale let myColor = d3.scaleSequential() //.interpolator(d3.interpolateRdYlGn) //red to green .interpolator(d3.interpolateBlues) .domain([d3.min(myValues),d3.max(myValues)]) // Three function that change the tooltip when user hover / move / leave a cell let mouseover = function(d) { //tooltip // .style("opacity", 1) d3.select(this) .style("stroke", "black") .style("opacity", 1) } let mouseleave = function(d) { //tooltip // .style("opacity", 0) d3.select(this) .style("stroke", "none") .style("opacity", 0.8) } // add the squares svg.selectAll() .data(data, function(d) {return d.group+':'+d.variable;}) .enter() .append("rect") .attr("x", function(d) { return x(d.group) }) .attr("y", function(d) { return y(d.variable) }) .attr("rx", 4) .attr("ry", 4) .attr("width", x.bandwidth() ) .attr("height", y.bandwidth() ) .style("fill", function(d) { return myColor(post_count_heatmap_input == 'as count'? d.value: d.value/post_classifiable_volume_map.get(d.group))} ) .style("stroke-width", 4) .style("stroke", "none") .style("opacity", 0.8) .on("mouseover", mouseover) .on("mouseleave", mouseleave) let valuesTextStroke = svg.selectAll(".valuesTextStroke") .data(data) .enter() .append("text") .attr("x", function(d) { return x(d.group) + x.bandwidth()/(post_count_heatmap_input == 'as count'?2.67:4) }) .attr("y", function(d) { return y(d.variable) + (y.bandwidth()/1.5) }) .attr("width", x.bandwidth() ) .attr("height", y.bandwidth() ) .text(d => post_count_heatmap_input == 'as count'? d.value:((100*d.value)/post_classifiable_volume_map.get(d.group)).toFixed(0).toString() + '%' ) .attr("fill", "none") .attr("stroke", "white") .attr("stroke-width", 2); let valuesText = svg.selectAll(".valuesText") .data(data) .enter() .append("text") .attr("x", function(d) { return x(d.group) + x.bandwidth()/(post_count_heatmap_input == 'as count'?2.67:4) }) .attr("y", function(d) { return y(d.variable) + (y.bandwidth()/1.5) }) .attr("width", x.bandwidth() ) .attr("height", y.bandwidth() ) .text(d => post_count_heatmap_input == 'as count'? d.value: ((100*d.value)/post_classifiable_volume_map.get(d.group)).toFixed(0).toString() + '%' ) /*// Add title to graph svg.append("text") .attr("x", 0) .attr("y", -50) .attr("text-anchor", "left") .style("font-size", "22px") .text("Heatmap: Sender vs topic post count");*/ // Add subtitle to graph svg.append("text") .attr("x", 0) .attr("y", -10) .attr("text-anchor", "left") .style("font-size", "14px") .style("fill", "grey") .style("max-width", 400) .text('' + (100 - (100*(Number(post_categories_map.get("Unclassified"))/Number(posts_count)).toFixed(2))) + '% of all posts categorised by GNL API, breakdown as follows'); return svg; } 3d:T1302,sender_topic_heatmap_chart_gen = { const data = topic_sentiments_objects_array; const id = 'topic_sentiments_heatmap_d3'; // set the dimensions and margins of the graph const margin = {top: 24, right: 5, bottom: 20, left: 150}, heatmap_width = width - margin.left - margin.right, //width declared outside of this block of code heatmap_height = 500 - margin.top - margin.bottom; // append the svg object to the body of the page d3.select("#" + id).html(''); let svg = d3.select("#" + id) .append("svg") .attr("width", heatmap_width + margin.left + margin.right) .attr("height", heatmap_height + margin.top + margin.bottom) .append("g") .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); // Labels of row and columns -> unique identifier of the column called 'group' and 'variable' let myGroups = Array.from(new Set(data.map(d => d.group))) .sort((a,b) => members_by_post_volume_array.indexOf(a) > members_by_post_volume_array.indexOf(b)?1:-1); let myVars = Array.from(new Set(data.map(d => d.variable))) .sort((a,b) => post_categories_top15_array.indexOf(a) > post_categories_top15_array.indexOf(b)?1:-1); let myValues = Array.from(new Set(data.map(d => d.value))); // Build X scales and axis: let x = d3.scaleBand() .range([ 0, heatmap_width ]) .domain(myGroups) .padding(0.05); svg.append("g") .style("font-size", 15) .attr("transform", "translate(0," + heatmap_height + ")") .call(d3.axisBottom(x).tickSize(0)) .select(".domain").remove() // Build Y scales and axis: let y = d3.scaleBand() .range([ heatmap_height, 0 ]) .domain(myVars) .padding(0.05); svg.append("g") .style("font-size", 15) .call(d3.axisLeft(y).tickSize(0)) .select(".domain").remove() // Build color scale let myColor = d3.scaleSequential() //.interpolator(d3.interpolateRdYlGn) //red to green .interpolator(d3.interpolateRdYlGn) .domain([d3.min(myValues),d3.max(myValues)]) // create a tooltip /*let tooltip = d3.select("#" + id) .append("div") .style("opacity", 0) .attr("class", "tooltip") .style("background-color", "white") .style("border", "solid") .style("border-width", "2px") .style("border-radius", "5px") .style("padding", "5px")*/ // Three function that change the tooltip when user hover / move / leave a cell let mouseover = function(d) { //tooltip // .style("opacity", 1) d3.select(this) .style("stroke", "black") .style("opacity", 1) } let mousemove = function(d) { /*tooltip .html("The exact value of
this cell is: " + d.value) .style("left", (d3.mouse(this)[0]+70) + "px") .style("top", (d3.mouse(this)[1]) + "px")*/ } let mouseleave = function(d) { //tooltip // .style("opacity", 0) d3.select(this) .style("stroke", "none") .style("opacity", 0.8) } // add the squares svg.selectAll() .data(data, function(d) {return d.group+':'+d.variable;}) .enter() .append("rect") .attr("x", function(d) { return x(d.group) }) .attr("y", function(d) { return y(d.variable) }) .attr("rx", 4) .attr("ry", 4) .attr("width", x.bandwidth() ) .attr("height", y.bandwidth() ) .style("fill", function(d) { return myColor(d.value)} ) .style("stroke-width", 4) .style("stroke", "none") .style("opacity", 0.8) .on("mouseover", mouseover) .on("mousemove", mousemove) .on("mouseleave", mouseleave) let valuesTextStroke = svg.selectAll(".valuesTextStroke") .data(data) .enter() .append("text") .attr("x", function(d) { return x(d.group) + x.bandwidth()/5 }) .attr("y", function(d) { return y(d.variable) + (y.bandwidth()/1.5) }) .attr("width", x.bandwidth() ) .attr("height", y.bandwidth() ) .text(d => d.value) .attr("fill", "none") .attr("stroke", "white") .attr("stroke-width", 2); let valuesText = svg.selectAll(".valuesText") .data(data) .enter() .append("text") .attr("x", function(d) { return x(d.group) + x.bandwidth()/5 }) .attr("y", function(d) { return y(d.variable) + (y.bandwidth()/1.5) }) .attr("width", x.bandwidth() ) .attr("height", y.bandwidth() ) .text(d => d.value) /*// Add title to graph svg.append("text") .attr("x", 0) .attr("y", -50) .attr("text-anchor", "left") .style("font-size", "22px") .text("Heatmap: Sender vs topic sentiment");*/ // Add subtitle to graph svg.append("text") .attr("x", 0) .attr("y", -10) .attr("text-anchor", "left") .style("font-size", "14px") .style("fill", "grey") .style("max-width", 400) .text('Minimum threshold of 20 posts per sentiment average cell'); return svg; }3e:[] 32:{"id":"54112b797617ba1a","slug":"part-3-what-are-we-talking-about","title":"Part 3 - What Are We Talking About?","thumbnail":"84bbf3eab52345026eb9735ef4fecb40cee2ef17b92f97f81a519350b8b9328d","access_level":"public","create_time":"$D2021-02-25T22:54:18.863Z","update_time":"$D2021-11-04T06:28:29.525Z","view_time":null,"first_public_time":"$D2021-03-01T10:57:59.308Z","has_unpublished_edits":null,"fork":false,"stars":5,"starred":false,"owner":{"type":"team","id":"46d757b0a7e70394","login":"mtsvelik","name":"Misha Tsvelik","avatar_url":"https://avatars-next.observableusercontent.com/9ef83be769fccaa60175b65abcb8fb329e45720c0290ca484f36e9cbd8d70073","tier":"starter_2024"},"version":504,"publish_version":504,"role":null,"authored":false,"authors":[{"type":"individual","id":"f60395cebced3c6c","login":"mtsvelik","name":"Misha Tsvelik","avatar_url":"https://avatars-next.observableusercontent.com/9ef83be769fccaa60175b65abcb8fb329e45720c0290ca484f36e9cbd8d70073","tier":null}],"members":[{"type":"individual","id":"f60395cebced3c6c","login":"mtsvelik","name":"Misha Tsvelik","avatar_url":"https://avatars-next.observableusercontent.com/9ef83be769fccaa60175b65abcb8fb329e45720c0290ca484f36e9cbd8d70073","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":4,"collection":{"id":"0aa3fad66df8397b","slug":"book-club-what-our-posts-say-about-us","title":"Book Club: What Our Posts Say About Us","owner":{"id":"46d757b0a7e70394","login":"mtsvelik","name":"Misha Tsvelik"}},"type":"notebook","body":{"title":"Part 3 - What Are We Talking About?","cells":[{"id":0,"value":"md `# Part 3 - What Are We Talking About?\n

A visual check on the accuracy of topic classification and exploration of how key world / UK events are shaping our conversations and feelings

The timeline below shows events that I thought likely had an impact on subjects of group discussion. You can play with the filters by brushing the post volume chart to zoom in on time period and selecting categorised topic, then have a look at the top entities detected in post texts and how the filtered messages split by sender.

`\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":64,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":470,"value":"md `Examples of some interesting findings:\n

In 8 Nov, 2016 Trump won the US election, Misha was the main sender of News & Politics posts a week either side of the event? 3 Nov 2020 Biden won and Alan was the main sender of News & Politics category posts. You could a sign of the times in terms of who talks about that topic.

The main sender of Sports category posts during the 2018 World Cup was Henry, he was also the main sender of Sports posts for the year (43% of sports posts for 2018).

When Boris Johnson won the 2020 UK Election, there was a fair bit of chat around the subject, whereas when Teresa May won in 2017 no-one really cared.

During Q1 2018 there was a prolonged period of negative average sentiment. Looking at All topics being talked about then (and setting the entity threshold right down = 3, some typical cultprits of neagtive chatter emerge such Amazon, Cambridge Analytics, Facebook.

\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":106,"value":"vocab_chart = html`

Who is the most verbose? / Has richest post vocabulary?

${viewof vocab_input}

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":401,"value":"md `

Although Henry has the most verbose posts, it is actually will who has the richest vocabulary with respect to how much he writes overall. As can be seen in [Part 2 - Group Dynamics](https://observablehq.com/@mtsvelik/part-2-group-dynamics?collection=@mtsvelik/book-club-what-our-posts-say-about-us) Andy and Jim have lots of conversation with short reply times, it's no surprise then that they are towards the lower end of average word count per post.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":160,"value":"post_count_heatmap_chart = html `

What do we like talking about? A heatmap of sender vs topic post count

${viewof post_count_heatmap_input}

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":484,"value":"md `

Alan posts more than anyone on News and Politics, but Will who has the highest % of their categorised posts as News and Politics

Andy posts a lot on Food & Drink

Mark has the highest % of his classifiable posts as Arts & Entertainment - 70%

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":188,"value":"sender_topic_heatmap_chart = html `

Heatmap: sender vs topic sentiment

${viewof sentiments_heatmap_threshold_input}

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":488,"value":"md `

News and Politics gets very negative average sentiment from everyone

\n
\n

Setting the min observations per cell threshold down to 20, the results are more anecdotal, but you can see Alan's admonishments about the coronavirus being picked up with a very negative sentiment score in Health, Daniel seems to have some positive things to say about People & Society and Henry seems to be negatave about every topic, more on that in the [next](https://observablehq.com/@mtsvelik/part-5-conversations-and-engagement?collection=@mtsvelik/book-club-what-our-posts-say-about-us) section.

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":387,"value":"$35","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":385,"value":"md `### Code: Word cloud / timeline charts`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":67,"value":"timeline_chart = html `

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":51,"value":"word_cloud_chart_gen = Highcharts.chart('wordcloud-container', {\n accessibility: {\n screenReaderSection: {\n beforeChartFormat: '

{chartTitle}

' +\n '

{chartSubtitle}

' +\n '

{chartLongdesc}

' +\n '

{viewTableButton}

'\n }\n },\n chart: {\n height: word_cloud_height\n },\n credits: {\n enabled: false\n },\n series: [{\n type: 'wordcloud',\n data: word_cloud_array,\n name: 'Occurrences'\n }],\n title: {\n text: null\n }\n}); ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":335,"value":"word_cloud_height = 300","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":61,"value":"$36","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":20,"value":"viewof rangesliderdemo_exploded = rangeSlider(exploded_file, \n d=>d['datetime_asdate'],\n value=>d3.count(value.values.map(d=>d['timestamp'])), // We want sum of amount for range\n {\n height:60,\n yTicks:3//,\n //yScale://d3.scalePow().exponent(0.7) // Domain and Range is automatically assigned\n }\n )","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":22,"value":"date_range_selected = `${rangesliderdemo_exploded.range[0].getDate()}/${rangesliderdemo_exploded.range[0].getMonth()+1}/${rangesliderdemo_exploded.range[0].getFullYear().toString().slice(-2)} to ${rangesliderdemo_exploded.range[1].getDate()}/${rangesliderdemo_exploded.range[1].getMonth()+1}/${rangesliderdemo_exploded.range[1].getFullYear().toString().slice(-2)}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":38,"value":"viewof word_cloud_category_input = {\n const word_cloud_category_input = persistentSelect({\n title: \"Category\",\n key:3,\n options: ['All'].concat(post_categories_top15_array),\n value: 'All'\n });\n //dd3.input.style.fontSize = \"30px\";\n word_cloud_category_input.input.style.marginTop = \"8px\";\n return word_cloud_category_input;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":43,"value":"viewof word_cloud_threshold_input = Range([1, d3.max(Array.from(new Set(typeof entities_per_topic_map.get(word_cloud_category_input) == 'undefined'? []:entities_per_topic_map.get(word_cloud_category_input).map(d => d.weight))))], {value: [\"All\", \"Unclassified\"].indexOf(word_cloud_category_input) > -1? 20:2, step:1, label: \"Min per entity occurrence\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":24,"value":"$37","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":318,"value":"$38","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":47,"value":"word_cloud_array = ( \n Highcharts.reduce(entities_per_topic_map.get(word_cloud_category_input), function (arr, word) {\n let obj = Highcharts.find(arr, function (obj) {\n return obj.name === word;\n });\n if (obj) {\n obj.weight += 1;\n } else {\n obj = {\n name: word,\n weight: 1\n };\n arr.push(obj);\n }\n return arr;\n }, []).filter(e => e.weight >= word_cloud_threshold_input)\n );","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":325,"value":"stacked_bar_array = {\n\n const topic_indexes = Array.from( new Set(indexes_per_topic_map.get(word_cloud_category_input)))\n const filtered_main_data = main_file.filter(e => topic_indexes.indexOf(e['index']) > -1);\n \n /*[{\n name: 'John',\n color: '',\n data: [5, 3, 4, 7, 2]\n }, {\n name: 'Jane',\n data: [2, 2, 3, 2, 1]\n }]*/\n \n \n return d3.rollups(filtered_main_data, v => v.length/topic_indexes.length, d => d['sender'])\n .map(e => ({\n name: e[0],\n color: colors[e[0]],\n data: [e[1]]\n }))\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":333,"value":"$39","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":375,"value":"senders_stacked_bar_chart = html `

`\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":100,"value":"md `### Vocab chart`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":104,"value":"$3a","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":108,"value":"viewof vocab_input = Radio(['average word count per post', 'vocabulary richness index (total unique words / total posts)'], {value: 'average word count per post'})\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":99,"value":"$3b","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":150,"value":"md `### Heatmap counts chart`\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":153,"value":"$3c","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":145,"value":"viewof post_count_heatmap_input = Radio([\"as count\",\"as % of member's classifiable posts\"], {value: \"as count\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":121,"value":"post_categories_map = d3.rollup(exploded_file, v => v.length, d => d['Google classification category - top'])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":129,"value":"posts_count = main_file.length","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":133,"value":"//need results returned in standard form of array [person,category,count]\ntopic_posts_array = d3.rollups(exploded_file, v => v.length, d => people_by_post_volume_object[d['sender']], d => topics_by_post_volume_object[d['Google classification category - top']] ).map(function(e){\n let person = e[0]\n return e[1].map(function(el){ \n el.unshift(person)\n return el\n })\n}).reduce(function(arr, elem) {\n return arr.concat(elem);\n}, [])\n.filter(function(el){\n return el[1] != 0;\n});\n\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":135,"value":"//need results returned in as array of objects [{group:,variable:,value:}]\ntopic_posts_objects_array = d3.rollups(exploded_file.filter(e => post_categories_top15_array.indexOf(e['Google classification category - top']) > -1), v => v.length, d => d['sender'], d => d['Google classification category - top']\n).map(function(e){\n let person = e[0]\n return e[1].map(function(el){ \n el.unshift(person)\n return el\n })\n}).reduce(function(arr, elem) {\n return arr.concat(elem);\n}, [])\n.map(function(el){\n return {\n group:el[0],\n variable:el[1],\n value:el[2]\n }\n});","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":147,"value":"post_classifiable_volume_map = {\n let classified_indexes = Array.from( new Set(exploded_file.filter(d => d['Google classification category - top'] != 'Unclassified').map(e => e['index'])) )\n \n let main_file_classified_data = main_file.filter(e => classified_indexes.indexOf(e['index']) > -1)\n \n return d3.rollup(main_file_classified_data, v => v.length, d => d.sender)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":173,"value":"md `### Heatmap sentiment chart`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":180,"value":"viewof sentiments_heatmap_threshold_input = Range([0, 50], {value: 30, step:1, label: \"Minimum posts for cell avg\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":182,"value":"$3d","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":176,"value":"//need results returned in standard form of array [person,category,count]\ntopic_sentiments_array = d3.rollups(exploded_file, v => v.length >= 20? Number(d3.mean(v, d => d['Google sentiment product']).toFixed(2)): '', d => people_by_post_volume_object[d['sender']], d => typeof topics_by_post_volume_object[d['Google classification category - top']] == 'undefined'? 15: topics_by_post_volume_object[d['Google classification category - top']] ).map(function(e){\n let person = e[0]\n return e[1].map(function(el){ \n el.unshift(person)\n return el\n })\n}).reduce(function(arr, elem) {\n return arr.concat(elem);\n}, []);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":178,"value":"//need results returned in as array of objects [{group:,variable:,value:}]\ntopic_sentiments_objects_array = d3.rollups(exploded_file.filter(e => post_categories_top15_array.indexOf(e['Google classification category - top']) > -1),\n v => v.length >= sentiments_heatmap_threshold_input? Number(d3.mean(v, d => d['Google sentiment product']).toFixed(2)): '',\n d => d['sender'],\n d => d['Google classification category - top'])\n.map(function(e){\n let person = e[0]\n return e[1].map(function(el){ \n el.unshift(person)\n return el\n })\n}).reduce(function(arr, elem) {\n return arr.concat(elem);\n}, [])\n.map(function(el){\n return {\n group:el[0],\n variable:el[1],\n value:el[2]\n }\n})\n.filter(e => e.value != \"\");","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"md `### Imports`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":13,"value":"import {Button, Checkbox, Toggle, Radio, Range, Select, Text, Search, Table, Input, bind, html} from \"@observablehq/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":15,"value":"import {rangeSlider, persistentSelect} from \"@bumbeishvili/utils\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":17,"value":"import {d3, Highcharts, main_file, exploded_file, colors, types, post_volume_array, members_by_post_volume_array, people_by_post_volume_object, post_volume_map, post_categories_top15_array, topics_by_post_volume_object} from \"@mtsvelik/book-club-whatsapp-group-what-our-posts-say-about-us-intro\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":273,"value":"import {sentiment_byweekyear_chart} from \"@mtsvelik/d3-change-line-chart-with-positive-negative-fill\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W3e","stdlib":"1","titling":"auto"}} 33:"$32:body"