Visualizing Unstructured Text / Amal Alabdulkarim

Amal Alabdulkarim

Workspace

Published

Edited

Apr 22, 2019

Importers

24 stars

d3 = require('d3')

// Load Text file using d3.text function

Alice_book = d3.text("https://gist.githubusercontent.com/amal994/912d85a113b096ce185e6d5560d3a025/raw/6bd784c2597b5130765616d026bb52df309a7810/alice.txt")

nlp =require('compromise')

alice_nlp = nlp(Alice_book).normalize().sentences() //.normalize.sentences() function helps us clean, standarize and preprocess our dataset and seperate it into sentences.

all_tokens = alice_nlp.terms().out('frequency').slice(0,100) //The terms function extracts all unique words from the text, out('frequency') function counts the occurances of these terms and constructs a bag of word with word counts.

filtered_terms = nlp(Alice_book).normalize().sentences().delete(stop_words).terms().out('frequency').slice(0,100) //Delete function takes a string or a regex experssion and deletes any string that matches the expression from the list of terms, here I initialized the nlp object again because in observable everytime we change a cell everything gets updated and this would make 'filtered_terms' and 'all_tokens' hold the same value.

positive_negative_verbs = alice_nlp.sentences().map(function(m){

return{'sentence':m.out(), //Extract the standarized sentences as pure text.

'negative_verbs': m.verbs().isNegative().length, //.verbs().isNegative().length Count all the negative verbs in the sentence

'positive_verbs':m.verbs().isPositive().length //.verbs().isPositive().length Count all positive verbs in the sentence

}})

sentence_sentiment = alice_nlp.sentences().map(function(m){

var sentiment = 0

if (m.verbs().isNegative().length != 0)

sentiment = -1;

else if(m.verbs().isPositive().length > 0)

sentiment = 1;

return{'sentence':m.out(), //Extract the standarized sentences as pure text.

'sentiment': sentiment //Assign sentiment values to sentences.

}})

people = alice_nlp.people().out('frequency').slice(0, 8)

places = alice_nlp.places().out('frequency')

import {select} from "@jashkenas/inputs"

viewof cloud_dataset_selection = {

const cds = select({

title: "Please choose a dataset to visualize on the word cloud:",

options: [{label:"Unfiltered Words", value:0},

{label:"Filtered Words", value:1},

{label:"Places", value:2},

{label:"People", value:3}],

value:0

});

return cds;

}

html `<svg id="word-cloud"></svg>`

md `We will use the D3-Cloud library for this visualization: https://github.com/jasondavies/d3-cloud`

d3cloud = require('d3-cloud')

md `We create a function to load the words we generated in the previous section and create a word cloud`

createWordCloud = function(words){

document.querySelector('#word-cloud').innerHTML = ""; //Empty the SVG element from any previous elements.

let height = 500; //Specify the height of the Word Cloud

let xScale = d3.scaleLinear() //This scale will help us translate any count range into a range of 10-100,

.domain([0, d3.max(words, function(d) { //we will use this for specifying fonts and colors

return d.count;

}) // The domain is 0 to the maximum count value in the given dataset.

])

.range([10,100]); //The domain is mapped to a [10,100] range.

let svg = d3.select("#word-cloud") //Select the SVG element with the unique ID we specified earlier

let fill = d3.scaleSequential() // Initialize the color scale, here we are using a sequential colorscale

.domain([10, 100]) //It accepts values within the range [10,100] and translate them to the equivelent colors.

.interpolator(d3.interpolateRainbow); // with interpolateRainbow color pallete.

var layout = d3cloud() //Here we are calling the d3cloud library to initialize our layout

.size([width,height]) //we specify the height and width of our word cloud.

.words(words) //Load the dataset

.padding(5) //Add padding 5 between words.

.rotate(function() { return 0; }) //Specify the rotation angle, here it is 0. Hence, no rotation.

.font("Impact") //Fontname

.fontSize(d => xScale(d.count)) //Font size of each word (notice that we use xScale for mapping)

.on("end", draw); //Call the Draw function after the layout is initialized.

layout.start();

function draw(words_d) {

svg.attr("width", layout.size()[0]) //Set the width of the SVG to be equal to the width of the layout object

.attr("height", layout.size()[1]) //Set the height of the SVG to be equal to the height of the layout object

.append("g") //Append a group svg element "g" to contain the words.

.attr(

"transform",

"translate(" + layout.size()[0] / 2 + "," + layout.size()[1] / 2 + ")"

) //Set the center location of the word cloud

.selectAll("text") //Select all text objects (there are none at this point)

.data(words_d) //Load dataset

.enter() //Create a text object for each data point

.append("text") //Append a text object for each word to the "g" object we created earlier

.style("font-size", d => xScale(d.count) + "px") //Specify the font size, similar to what we did earlier.

.style("font-family", "Impact") //Specify font size.

.attr("text-anchor", "middle") //text-anchor:middle => the x,y values will be of the center of the text.

.attr("transform", d => `translate(${[d.x, d.y]})rotate(${d.rotate})`) //Move the word to its x,y location and rotate it if rotation is specified.

.style("fill", function(d) {return fill(xScale(d.count)); }) //Fill the word with a corrosponding color.

.text(d => d.normal); //Add the text of the word.

}

return svg // return the SVG element

}

lists = [all_tokens, filtered_terms, places, people]; // Optional: Initialize an array of datasets

createWordCloud(lists[cloud_dataset_selection]); //This line calls the function to create the word cloud and passes the dataset that we want to use to do that,

viewof barchart_dataset_selection = {

const bds = select({

title: "Please choose a dataset to visualize on the barchart:",

options: [{label:"All Words", value:0},

{label:"Filtered Words", value:1},

{label:"Places", value:2},

{label:"People", value:3}],

value:0

});

return bds;

}

html `<svg id="frequency-barchart"></svg>`

createBarChart = function(words) {

document.querySelector('#frequency-barchart').innerHTML = ""; //Empty the SVG element from any previous elements.

//Adapted from: https://observablehq.com/@d3/horizontal-bar-chart

const format = d3.format(".0f"); //Specify how many decimal places to show on the barchart label, here it is 0 because we are using integer counts.

const margin = ({top: 30, right: 0, bottom: 10, left: 55}); //specify the margins of the graph

const height = words.length * 25 + margin.top + margin.bottom; //specify the height of the chart which will also determine the width of every bar (here the width of the bar is the height because it is a horizontal chart)

const svg = d3.select("#frequency-barchart"); //Select the SVG element by the unique ID we gave it earlier.

svg.attr("width",width) //Specify the width and height of the chart

.attr("height",height);

const x = d3.scaleLinear() //Create the x scale

.domain([0, d3.max(words, d => d.count)])

.range([margin.left, width - margin.right]);

const y = d3.scaleBand() //Ctrate the y scale

.domain(words.map(d => d.normal))

.range([margin.top, height - margin.bottom])

.padding(0.1);

const xAxis = g => g //create the x Axis and attach the x scale to it

.attr("transform", `translate(0,${margin.top})`)

.call(d3.axisTop(x).ticks(width / 80))

.call(g => g.select(".domain").remove());

const yAxis = g => g //create the y Axis and attach the y scale to it

.attr("transform", `translate(${margin.left},0)`)

.call(d3.axisLeft(y).tickSizeOuter(0));

svg.append("g") //Create each bar in the bar chart

.attr("fill", "steelblue")

.selectAll("rect")

.data(words)

.join("rect")

.attr("x", x(0))

.attr("y", d => y(d.normal)) // The word

.attr("width", d => x(d.count) - x(0)) // The frequency of the word

.attr("height", y.bandwidth()); // Get the width of the Band in the Y scale

svg.append("g") // Create the count label on the top of each bar chart

.attr("fill", "white")

.attr("text-anchor", "end")

.style("font", "12px sans-serif")

.selectAll("text")

.data(words)

.join("text")

.attr("x", d => x(d.count) - 4) //The label position on the x axis

.attr("y", d => y(d.normal) + y.bandwidth() / 2) // The label position on the y axis

.attr("dy", "0.35em")

.text(d => format(d.count));

svg.append("g") //Add the x axis to the barchart

.call(xAxis);

svg.append("g") //Add the y axis to the barchart

.call(yAxis);

return svg.node();

}

createBarChart(lists[barchart_dataset_selection]); //This line calls the function to create the bar chart and passes the dataset that we want to use to do that.

viewof method_colors_selection = {

const bds = select({

title: "Please choose a method to visualize the sentiment features",

options: [{label:"Colord Rectangles", value:"rect"},

{label:"Colored Sentences", value:"text"}],

value:"text"

});

return bds;

}

html `<svg id="sentiment-bars"></svg>`

createSentimentChart = function(sentences, method) {

document.querySelector('#sentiment-bars').innerHTML = ""; //Empty the SVG element from any previous elements.

//Adapted from: https://observablehq.com/@d3/horizontal-bar-chart

const format = d3.format(".3f");

const margin = ({top: 30, right: 0, bottom: 10, left: 30});

const svg = d3.select("#sentiment-bars");

const x = d3.scaleLinear() //Specfiy the x scale

.domain([0, 10])

.range([margin.left, width - margin.right]);

const colors = ['red','grey','green']; //Specfiy the colors we will use for our features

const y_ticks = [...Array(sentences.length).keys()]; //generate line numbers to be the ticks of the y scale

const xAxis = g => g //Initialize the xAxis and attach the x scale to it.

.attr("transform", `translate(0,${margin.top})`)

.call(d3.axisTop(x).ticks(width / 80))

.call(g => g.select(".domain").remove());

if(method === "rect") { //Create the colored rectangles version

//Specify the height of the chart which will also determing the height of each of the rectangles

const height = sentences.length * 5 + margin.top + margin.bottom;

const y = d3.scaleBand() //initialize the y scale

.domain(sentences.map((d,i) => y_ticks[i]))

.range([margin.top, height - margin.bottom])

.padding(0.1);

svg.append("g") //Create a rectangle for every sentence in the dataset

.selectAll("rect")

.data(sentences)

.join("rect")

.attr("x", x(0))

.attr("y", (d,i) => y(i))

.attr("fill",d => colors[d.sentiment+1])

.attr("width", d => "150")

.attr("height", y.bandwidth())

.append("title")

.text(function(d) {

return d.sentence;

});

svg.attr("width",width) //Specify the width and height of the chart

.attr("height",height);

}

else if (method === "text"){ //Create the colored text version

const height = sentences.length * 12 + margin.top + margin.bottom;

const y = d3.scaleBand() //initialize the y scale

.domain(sentences.map((d,i) => y_ticks[i]))

.range([margin.top, height - margin.bottom])

.padding(0.1);

const yAxis = g => g //initialize the y axis and attach the y scale to it

.attr("transform", `translate(${margin.left},0)`)

.call(d3.axisLeft(y).tickSizeOuter(0));

svg.append("g")

.style("font", "10px sans-serif")

.selectAll("text")

.data(sentences)

.join("text")

.attr("x", d => x(0)+5) //The sentence position on the x axis

.attr("y", (d,i) => y(i) + y.bandwidth() / 2) // The sentence position on the y axis

.attr("dy", "0.35em")

.attr("fill",d => colors[d.sentiment+1])

.text(d => d.sentence);

svg.append("g") // Add the line numbers next to the lines

.call(yAxis);

svg.attr("width",width) //Specify the width and height of the chart

.attr("height",height);

}

return svg.node();

}

createSentimentChart(sentence_sentiment, method_colors_selection)

Purpose-built for displays of data

Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.

Learn more