The avatar for @enjalot
Ian Johnson
pixel flipper, data sifter. trying to see what I can while I'm here
Public
Edited
Nov 2, 2022
9 forks
21 stars
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
timezone = Intl.DateTimeFormat().resolvedOptions().timeZone //"America/Los_Angeles"
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Plot.plot({
facet: {
data: tidytweets,
y: "month",
marginLeft: 100
},
marks: [
Plot.barY(
tidytweets,
Plot.groupX({
y: "count",
title: v => `${dayf(v[0].created_at)} ${v.length}`
}, {
x: "dayOfMonth",
// y: "month",
fill: d => d.year + "",
order: "ascending"
// fill: "favorite_count"
})
)
],
color: { legend: true },
y: { axis: null },
fy: { label: null },
x: { },
width,
height: 400,
marginLeft: 100
})
Insert cell
Insert cell
db
WITH screen_names as (select unnest(string_to_array(user_mentions, '|')) as screen_name
from tweets)
select screen_name, count(*)::INT as count
from screen_names
group by screen_name
order by count desc
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
tweet(usertweet.id)
Insert cell
usertweet
Insert cell
usertweets = tidytweets.filter(d => d.user_mentions.indexOf(user.screen_name) >= 0)
Insert cell
Insert cell
Insert cell
Insert cell
followers = JSON.parse((await rawfollower.text()).replace(/^.{0,100}.part0 = /, "")).map((d) => d.follower)
Insert cell
followersDerived = followers.map(d => {
let user = interactedMap.get(d.accountId) || { id: d.accountId }
return user
}).sort((a,b) => a.id - b.id)
Insert cell
Insert cell
monthf(new Date())
Insert cell
monthf = d3.utcFormat("%b %Y")
Insert cell
dayf(new Date())
Insert cell
dayf = d3.utcFormat("%b %d %Y")
Insert cell
// get the text value of the loaded file (rawfile is a promise that resolves once you choose a file)
text = rawfile.text()
Insert cell
// the file is a thin javascript wrapper around a big JSON array
tweets = JSON.parse(text.replace(/^.{0,100}.part0 = /, "")).map((d) => d.tweet)
Insert cell
tidytweets = tweets.map(d => {
let retweet = d.full_text.indexOf("RT @") === 0
let reply = d.full_text.indexOf("@") === 0
const tzdate = Temporal.Instant.from(d3.isoFormat(new Date(d.created_at))).toZonedDateTimeISO(timezone)
const hour = tzdate.hour;
const dayOfMonth = tzdate.day;
const day = weekdays[tzdate.dayOfWeek + -1];
const weekend = tzdate.dayOfWeek > 5 ? "Weekend" : "Weekday";
const week = tzdate.weekOfYear;
const month = months[tzdate.month - 1];
const year = tzdate.year;
return {
full_text: d.full_text,
created_at: new Date(d.created_at),
favorite_count: +d.favorite_count,
retweet_count: +d.retweet_count,
user_mentions: d.entities.user_mentions.map(u => u.screen_name).join("|"),
retweet,
reply,
type: retweet ? "retweet" : reply ? "reply" : "tweet",
id: d.id,
lang: d.lang,
// time stuff
hour,
day,
month,
dayOfMonth,
year
}
})
Insert cell
db = DuckDBClient.of({ tweets: tidytweets })
Insert cell
db
-- this will show tweets once you've uploaded your file
select * from tweets
Insert cell
months = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]
Insert cell
weekdays = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
Insert cell
d3.isoFormat(new Date())
Insert cell
// example of finding a specific tweet by id
tweets.filter(d => d.id == "1586885293323460608")
Insert cell
tweets.filter(d => d.entities.user_mentions.length > 0)
Insert cell
interacted = tweets.flatMap(d => d.entities.user_mentions)
Insert cell
interactedMap = d3.rollup(interacted.map(d => ({id: d.id, name: d.name, screen_name: d.screen_name})), v => {
return {
id: v[0].id,
name: v[0].name,
screen_name: v[0].screen_name,
interaction_count: v.length
}
}, d => d.id)
Insert cell
import {DuckDBClient} from "@cmudig/duckdb"
Insert cell
import {Plot} from "@mkfreeman/plot-tooltip"
Insert cell
import {tweet} from "@mbostock/tweet"
Insert cell
// Load the Temporal API using a Polyfill
Temporal = {
const TemporalLib = await require("@js-temporal/polyfill@0.3.0")
return TemporalLib.Temporal
}
Insert cell

Purpose-built for displays of data

Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
Learn more