The avatar for @odbol
Tyler Freeman
Published unlisted
Edited
Mar 13, 2022
2
Insert cell
Insert cell
snitchesAll = FileAttachment("twittysnitches-parsed.json").json()

Insert cell
snitches = snitchesAll.filter(d => !d.error).filter(d => d.tweetTextParsingConfidence > 0);
Insert cell
followers = FileAttachment("followers.json").json()

Insert cell
isFollower = authorHandle => followers.some(f => authorHandle && f.handle === "@" + authorHandle)
Insert cell
snitchesWithFollowers = snitches.map(s => {
const tweet = s.tweet.replaceAll('\\n', '\n');console.log(tweet);
return {
isFromFollowed: isFollower(s.authorHandle),
sentiment: vader.SentimentIntensityAnalyzer.polarity_scores(tweet),
tweet: tweet,
...s
}
});
Insert cell
snitchesFiltered = snitchesWithFollowers.map(d => {
const totalEngagement = d.retweets + d.likes + d.replies + 1;
return {
...d,
retweets: d.retweets + 1,
likes: d.likes + 1,
totalEngagement,
totalEngagementLog: Math.log10(totalEngagement)
};
})
.filter(d => d.totalEngagement > 0); // && d.totalEngagement < 50000
Insert cell
followedColor = d => d.isFromFollowed ? "Followed" : "Stranger";
Insert cell
followedColorSettings = {
return {
domain: ["Stranger", "Followed"],
range: ["rgb(78 121 167 / 50%)", "rgb(242 142 44 / 50%)"],
legend: true
}
};
Insert cell
adColor = 'rgb(214 65 161 / 50%)';
Insert cell
followedColorWithAds = d => d.isPromoted ? "Ad" : followedColor(d)
Insert cell
followedColorSettingsWithAds = {
const newSettings = {
...followedColorSettings
}
newSettings.domain = [...newSettings.domain, 'Ad'];
newSettings.range = [...newSettings.range, adColor];
return newSettings;
}
Insert cell
Inputs.table(snitchesFiltered)
Insert cell
snitchesFiltered.filter(d => d.sentiment.compound == 0)
Insert cell
Plot.rect(snitchesWithFollowers, Plot.bin({fill: "count"}, {x: "likes", y: "retweets"})).plot({
x: {
type: "log",
base: 10,
grid:true
},
y: {
type: "log",
base: 10
}
})
Insert cell
Plot.dot(snitchesWithFollowers, Plot.bin({r: "count", fill:"count"}, {x: d => Math.log10(d.likes + 1), y: d => Math.log10(d.retweets + 1)})).plot({
color: {
domain: [0, 100],
range: ['#a1d99b', '#31a354'],
legend: true
},
})
Insert cell
Plot.dot(snitchesFiltered, Plot.bin({r: "count", fill:followedColor}, {x: d => Math.log10(d.likes + 1), y: d => Math.log10(d.retweets + 1)})).plot({
facet: {
data: snitchesFiltered,
x: followedColor
},
color:
followedColorSettings
,
})
Insert cell
Plot.dot(snitchesWithFollowers.filter((d) => d.likes > 0 && d.retweets > 0), Plot.bin({r: "count", fill:"count"}, {x: d => Math.log10(d.likes), y: d => Math.log10(d.retweets)})).plot({
color: {
domain: [0, 60],
range: ['#a1d99b', '#31a354'],
legend: true
},
})
Insert cell
Plot.dot(snitchesWithFollowers, Plot.bin({r: "count", fill:"count"}, {x: d => d.likes, y: d => d.retweets})).plot({
x: {
type: 'symlog'
},
y: {
type: 'symlog'
},
color: {
domain: [0, 60],
range: ['#a1d99b', '#31a354'],
legend: true
},
})
Insert cell
Plot.dot(snitchesFiltered.filter((d) => d.likes > 0 && d.retweets > 0), Plot.bin({r: "count", fill:"count"}, {x: d => Math.log10(d.likes), y: d => Math.log10(d.totalEngagement)})).plot({
color: {
domain: [0, 60],
range: ['#a1d99b', '#31a354'],
legend: true
},
})
Insert cell
md`### Sentiment Analysis`
Insert cell
Insert cell
Insert cell
md`### Main Character
Tracks the "Ratio", i.e. the proportion of Likes to Replies, in order to spot controversial statements.
`
Insert cell

mainCharacter = Plot.plot({
...chartStyle,
y: {
type: "symlog",
base: 10,
label: "Ratio ( Likes / Replies )",
ticks: 4
},
x: {
type: "symlog",
base: 10,
label: "Total engagement",
ticks: 2,
tickFormat: d3.format('.2s')
},
// facet: {
// data: snitchesFiltered,
// x: followedColor
// },
color:followedColorSettings,
marks: [
Plot.dotY(snitchesFiltered, {
y: d => {
const ratio = d.likes / d.replies;
if (ratio < 1) {
return -d.replies / d.likes;
}
return ratio;
},
x: "totalEngagement",
fill:followedColor,
title: d => d.tweet,
thresholds: 'scott'}),
Plot.ruleY([0]),
Plot.ruleX([0])
]
});

Insert cell
## Proportions

Looks like around 1/3 of my feed is tweets from people I follow, and 2/3s are from strangers and ads. But lets go a bit deeper into why
Insert cell
donut = DonutChart(d3.flatRollup(snitchesFiltered, ar => ar.length, followedColorWithAds), {
name: d => d[0],
value: d => (d[1] / snitchesFiltered.length),
format: '.0%',
names: followedColorSettingsWithAds.domain,
colors: followedColorSettingsWithAds.range,
width,
height: 500
})
Insert cell
[...followedColorSettings.range, 'rgb(242 142 44 / 50%)']
Insert cell
d3.rollup(snitchesFiltered, ar => ar.length, followedColor)

Insert cell
Here we start to see the distribution of followed vs strangers based on engagement: how many times it was retweeted or liked.

We start to see that the less popular tweets are from people I follow, whereas the really popular tweets with lots of retweets and likes are mainly strangers.

Maybe I just don't follow enough popular people who tweet out bangers with hundreds of thousands of likes? I'm curious to see how these numbers compare to someone who primarily follows large accounts, and if it's still an easy 1/3 split between followed/stranger or if the division is really between popularity...
Insert cell
likesVsRetweets = Plot.dot(snitchesWithFollowers, {x: "likes", y: "retweets", fill:followedColorWithAds,
title: d => d.tweet,}).plot({
...chartStyle,
x: {
type: "log",
base: 2,
label: 'Likes',
tickFormat: ".0s"
},
y: {
type: "log",
base: 2,
label: 'Retweets'
},
color:followedColorSettingsWithAds,
})
Insert cell

Plot.dot(snitchesWithFollowers.filter(d => d.retweets < 50000), Plot.bin({r: "count"},{x: "likes", y: "retweets", fill:followedColor})).plot({

color: followedColorSettings,
})
Insert cell
Plot.rectY(snitchesWithFollowers.filter(d => d.retweets < 50000), Plot.binX({y: "count"}, {x: "retweets", fill:followedColor, thresholds: 'scott'})).plot({
color:followedColorSettings,
});
Insert cell

Plot.rectY(snitchesFiltered, Plot.binX({y: "count"}, {x: "retweets", fill:followedColor, thresholds: 'scott'})).plot({
facet: {
data: snitchesFiltered,
x: followedColor
},
color:followedColorSettings,
});
Insert cell

strangerVsFollowed = Plot.plot({
...chartStyle,
y: {
base: 10,
label: 'Number of tweets in timeline',
labelAnchor: 'center',
tickFormat: '~r',
labelOffset: 41
},
x: {
label: "Total engagement",
tickFormat: '.2~s'
},
facet: {
data: snitchesFiltered,
x: followedColor,
labelAnchor: 'right'
},
color:followedColorSettings,
marks: [
Plot.rectY(snitchesFiltered, Plot.binX({y: "count"}, {x: "totalEngagement", fill:followedColor, thresholds: 'scott'})),
Plot.ruleY([0])
]
});
Insert cell

strangerVsFollowedLog = Plot.plot({
...chartStyle,
y: {
type: "symlog",
base: 10,
label: 'Number of tweets in timeline',
labelAnchor: 'center',
tickFormat: '~r',
labelOffset: 41,
ticks: 5
},
x: {
label: "Total engagement",
tickFormat: '.2~s'
},
facet: {
data: snitchesFiltered,
x: followedColor,
labelAnchor: 'right'
},
color:followedColorSettings,
marks: [
Plot.rectY(snitchesFiltered, Plot.binX({y: "count"}, {x: "totalEngagement", fill:followedColor, thresholds: 'scott'})),
Plot.ruleY([0])
]
});
Insert cell
Plot.plot({
y: {
label: 'Number of tweets in timeline'
},
fx: {
domain: ['followed', 'stranger'],
label: null,
tickSize: 2
},
facet: {
data: snitchesFiltered,
x: followedColor
},
color:followedColorSettings,
marks: [
Plot.rectY(snitchesFiltered, Plot.binX({y: "count"}, {x: "totalEngagement", fill:followedColor, thresholds: 'scott'})),
Plot.ruleY([0])
]
});
Insert cell
Plot.areaY(snitchesFiltered, Plot.binX({y: "count"}, {x: "totalEngagement", fill:followedColor, thresholds: 10000})).plot({
x: {
type: "log",
base: 2,
},
y: {
label: 'Number of tweets in timeline'
// domain: [0, 100]
},
color: followedColorSettings,
});
Insert cell
Plot.lineY(snitchesWithFollowers, Plot.binX({y: "count"}, {x: "retweets", stroke:followedColor, thresholds: 10000})).plot({
...chartStyle,
x: {
type: "log",
base: 2,
},
y: {
label: 'Number of tweets in timeline'
// domain: [0, 100]
},
color: followedColorSettings,
});
Insert cell
Plot.lineY(snitchesWithFollowers, Plot.binX({y: "count"}, {x: "retweets", stroke:followedColorWithAds, thresholds: 10000})).plot({
x: {
type: "log",
base: 2,
},
y: {
label: 'Number of tweets in timeline'
// domain: [0, 100]
},
color: followedColorSettingsWithAds,
});
Insert cell
Plot.lineY(snitchesWithFollowers, {x: "retweets", stroke: followedColor}).plot({
x: {
//type: "symlog",
base: 2,
},
y: {
// domain: [0, 100]
},
color: followedColorSettings,
});
Insert cell
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.

Insert cell
Plot.rectY(snitchesWithFollowers.filter(d => d.retweets < 500), Plot.binX({y: "count"}, {x: "retweets", fill:followedColor, thresholds: 'freedman-diaconis'})).plot({
color: followedColorSettings,
});
Insert cell
Plot.rectY(snitchesWithFollowers.filter(d => d.retweets < 500), Plot.binX({y: "count"}, {x: "likes", fill:followedColor, thresholds: 'freedman-diaconis'})).plot({
color: followedColorSettings,
});
Insert cell
import {vader} from "@chrstnbwnkl/vader-sentiment-playground"
Insert cell
sentimentAnalyzer = new vader.SentimentIntensityAnalyzer()
Insert cell
### Tweets shown by friend's recommendations

This shows the distribution of tweets that appeared in my timeline because a person I follow Liked, Retweeted, or replied to it, or follows the account. It looks like the majority of tweets are from strangers recommended to me because someone I follow follows them, and the remaining 2% are ads.
Insert cell
recommendedTweets = Plot.barY(snitchesFiltered, Plot.groupX({y: "count"}, {x: d => d.numWhoLiked + d.numWhoRetweeted + d.whoReplied.length + (d.isReplyTo ? 1 : 0) + d.numWhoFollow, fill:followedColor})).plot({
...chartStyle,
x: {
label: "Number of recommendations",
ticks: 2
},
y: {
label: 'Number of tweets in timeline'
// domain: [0, 100]
},
color: followedColorSettings,
});
Insert cell
d3.schemePastel1
Insert cell
recommendationLabels = (() => {
return {
numWhoLiked:'Liked by followed',
numWhoRetweeted:'Retweeted by followed',
numWhoReplied:'Reply from followed',
isReplyTo:'Reply to followed',
numWhoFollow:'Followed by followed',
}
})()
Insert cell
recommendedRollup = d3.sort(d3.flatRollup(snitchesFiltered, ar => ar.length, d => {
if (d.numWhoLiked > 0) {
return recommendationLabels.numWhoLiked;
} else if (d.numWhoRetweeted > 0) {
return recommendationLabels.numWhoRetweeted;
} else if (d.whoReplied.length > 0) {
return recommendationLabels.numWhoReplied;
} else if (d.isReplyTo) {
return recommendationLabels.isReplyTo;
} else if (d.numWhoFollow > 0) {
return recommendationLabels.numWhoFollow;
}

return followedColorWithAds(d);
}), ar => -ar[1])
Insert cell
recommendedRollupMap = d3.rollup(snitchesFiltered, ar => ar.length, d => {
if (d.numWhoLiked > 0) {
return recommendationLabels.numWhoLiked;
} else if (d.numWhoRetweeted > 0) {
return recommendationLabels.numWhoRetweeted;
} else if (d.whoReplied.length > 0) {
return recommendationLabels.numWhoReplied;
} else if (d.isReplyTo) {
return recommendationLabels.isReplyTo;
} else if (d.numWhoFollow > 0) {
return recommendationLabels.numWhoFollow;
}

return followedColorWithAds(d);
})
Insert cell
d3.union(followedColorSettingsWithAds.range, d3.schemePastel2)
Insert cell
recommendationColorSettings = {
const newSettings = {
...followedColorSettingsWithAds
}
newSettings.domain = [...newSettings.domain, ...Object.values(recommendationLabels)];
newSettings.range = Array.from(d3.union(newSettings.range, d3.schemePastel2.map(c => {
const newColor = d3.color(c);
newColor.opacity = 0.5;
return newColor.formatRgb();
}))).slice(0, newSettings.domain.length);
return newSettings;
}
Insert cell
recommendationColorScale = d3.scaleOrdinal().domain(recommendationColorSettings.domain).range(recommendationColorSettings.range)
Insert cell
recommendedBreakdownDonut = DonutChart(recommendedRollup, {
name: d => d[0],
value: d => (d[1] / snitchesFiltered.length),
format: '.0%',
names: recommendationColorSettings.domain,
colors: recommendationColorSettings.range,
width,
height: 500
})
Insert cell
import {Treemap} from "@d3/treemap"
Insert cell
recommendedBreakdownTreemap = {
const recommendedBreakdownTreemap = Treemap({ children: recommendedRollup }, {
value: d => d[1], // size of each node (file); null for internal nodes (folders)
group: (d, n) => d[0],
label: (d, n) => d[0] + '\n' + Math.round(d[1] / snitchesFiltered.length * 100) + '%',
color: d => recommendationColorSettings.domain[0],
width: 720,
height: 720
});

d3.select(recommendedBreakdownTreemap).selectAll('rect')
.style('fill', d => {
// console.log('nod', d);
return recommendationColorScale(d.data[0]);
})
.style('fill-opacity', 1);
console.log('treemap: ', d3.select(recommendedBreakdownTreemap))
return recommendedBreakdownTreemap
}
Insert cell
recommendedBreakdownTreemapNested = {
const topLevelNames = ['Followed', 'Ad']
const topLevel = recommendedRollup.filter(d => topLevelNames.includes(d[0]));
const recommendedRollupStrangerChildren = recommendedRollup.filter(d => !topLevelNames.includes(d[0]));
const recommendedRollupNested = topLevel;
const strangers = ['Strangers', d3.sum(recommendedRollupStrangerChildren, d => d[1])];
strangers.children = recommendedRollupStrangerChildren
recommendedRollupNested.push(strangers);
const hierarchy = d3.hierarchy({children: recommendedRollupNested})
.sum(d => d.children ? 0 : d[1])
.sort(d => d[1]);
console.log('hierarchy', hierarchy)
const recommendedBreakdownTreemap = Treemap(
hierarchy,
{
value: d => d.data[1], // size of each node (file); null for internal nodes (folders)
// group: (d, n) => d[0],
label: (d, n) => d.data[0] + '\n' + Math.round(d.data[1] / snitchesFiltered.length * 100) + '%',
color: d => recommendationColorSettings.domain[0],
width: 720,
height: 720
});

d3.select(recommendedBreakdownTreemap).selectAll('rect')
.style('fill', d => {
// console.log('nod', d);
return recommendationColorScale(d.data[0]);
})
.style('fill-opacity', 1);
console.log('treemap: ', d3.select(recommendedBreakdownTreemap))
return recommendedBreakdownTreemap
}
Insert cell
recommendedBreakdown = Plot.barY(snitchesFiltered, Plot.groupX({y: "count"}, {x: d => d.numWhoLiked + d.numWhoRetweeted + d.whoReplied.length + (d.isReplyTo ? 1 : 0) + d.numWhoFollow, fill:followedColor})).plot({
...chartStyle,
x: {
label: "Number of recommendations",
ticks: 2
},
y: {
label: 'Number of tweets in timeline'
// domain: [0, 100]
},
color: followedColorSettings,
});
Insert cell
snitchesFiltered.filter( d => d.numWhoLiked + d.numWhoRetweeted + d.whoReplied.length + (d.isReplyTo ? 1 : 0) === 0 && !d.isFromFollowed);
Insert cell
Plot.barY(snitchesFiltered, Plot.groupX({y: "count"}, {x: "isPromoted", fill:followedColorWithAds})).plot({
x: {
label: "Is an ad",
ticks: 2
},
y: {
label: 'Number of tweets in timeline'
// domain: [0, 100]
},
color: followedColorSettingsWithAds,
});
Insert cell
### How old are these things?

It's interesting to note that most the tweets from people I follow is more recent (within a day). Whereas if it's older than a day, it's exclusively from strangers. I guess twitter wants me to catch up on the drama happening in other areas I'm not actively following.

Unfortunately, this means that the majority of tweets I see are relatively old and outdated.
Insert cell
Plot.plot({
...chartStyle,
y: {
domain: [0, 700],
label: 'Number of tweets in timeline'
},
x: {
type: 'log',
ticks: 3,
tickFormat: ms => dayjs().subtract(ms).fromNow()
},
// fx: {
// domain: ['followed', 'stranger'],
// label: null,
// tickSize: 2
// },
// facet: {
// data: snitchesFiltered,
// x: followedColor
// },
color:followedColorSettings,
marks: [
Plot.rectY(snitchesFiltered, Plot.binX({y: "count"}, {x: "postedMillisecondsAgo", fill:followedColor, thresholds: [5, 10, 20, 30, 60, 120, 60 * 3, 60 * 6, 60 * 9,60 * 12,60 * 15,60 * 18, 60 * 24, 60 * 24 * 2, 60 * 24 * 3, 60 * 24 * 4, 60 * 24 *5, 60 * 24 * 6].map(ms => ms * 1000 * 60)})),
Plot.ruleY([0])
]
});
Insert cell
tweetAge = Plot.plot({
...chartStyle,
y: {
domain: [0, 700],
label: 'Number of tweets in timeline'
},
x: {
type: 'log',
ticks: 3,
tickFormat: ms => dayjs().subtract(ms).fromNow(),
label: "Time posted"
},
// fx: {
// domain: ['followed', 'stranger'],
// label: null,
// tickSize: 2
// },
// facet: {
// data: snitchesFiltered,
// x: followedColor
// },
color:followedColorSettingsWithAds,
marks: [
Plot.rectY(snitchesFiltered, Plot.binX({y: "count"}, {x: "postedMillisecondsAgo", fill:followedColorWithAds, thresholds: [5, 10, 20, 30, 60, 120, 60 * 3, 60 * 6, 60 * 9,60 * 12,60 * 15,60 * 18, 60 * 24, 60 * 24 * 2, 60 * 24 * 3, 60 * 24 * 4, 60 * 24 *5, 60 * 24 * 6].map(ms => ms * 1000 * 60)})),
Plot.ruleY([0])
]
});
Insert cell
timeBucketScale = Plot.scale({
x: {
domain: d3.extent(snitchesFiltered.filter(d => d.postedMillisecondsAgo > 0), d => d.postedMillisecondsAgo),
// range: [0, 600],
type: 'log',
// type: "pow", exponent: 1/ 3,
ticks: 3,
tickFormat: ms => dayjs().subtract(ms).fromNow()
},
});
Insert cell
timeBucketThresholds = d3.range(timeBucketScale.domain[0] / 1000 / 60, timeBucketScale.domain[1] / 1000 / 60, 60 * 5).map(d => d * 1000 * 60)
// .map(d => Math.pow(d, 2))
//.map( timeBucketScale.apply).map(d => d * timeBucketScale.domain[1]) //.map(Math.log10)
Insert cell
timeBucketThresholds.map(ms => dayjs().subtract(ms).fromNow())
Insert cell
Plot.plot({
y: {
domain: [0, 700],
label: 'Number of tweets in timeline'
},
x: {
ticks: 3,
tickFormat: ms => dayjs().subtract(ms).fromNow(),
...timeBucketScale
},
// fx: {
// domain: ['followed', 'stranger'],
// label: null,
// tickSize: 2
// },
// facet: {
// data: snitchesFiltered,
// x: followedColor
// },
color:followedColorSettingsWithAds,
marks: [
Plot.rectY(snitchesFiltered, Plot.binX({y: "count"}, {
x: "postedMillisecondsAgo",
fill:followedColorWithAds,
thresholds: timeBucketThresholds
// thresholds: [5, 10, 20, 30, 60, 120, 60 * 3, 60 * 6, 60 * 9,60 * 12,60 * 15,60 * 18, 60 * 24, 60 * 24 * 2, 60 * 24 * 3, 60 * 24 * 4, 60 * 24 *5, 60 * 24 * 6].map(ms => ms * 1000 * 60)
})),
Plot.ruleY([0])
]
});
Insert cell
Plot.plot({
y: {
domain: [0, 80],
label: 'Number of tweets in timeline'
},
x: {
// type: 'log',
ticks: 3,
tickFormat: ms => dayjs().subtract(ms).fromNow()
},
// fx: {
// domain: ['followed', 'stranger'],
// label: null,
// tickSize: 2
// },
// facet: {
// data: snitchesFiltered,
// x: followedColor
// },
color:followedColorSettings,
marks: [
Plot.lineY(snitchesFiltered, {y: "postedMillisecondsAgo", fill:followedColor, }),
Plot.ruleY([0])
]
});
Insert cell
d3.scaleLog().domain(5, 120, 6*24*6)
Insert cell
Plot.plot({
y: {
label: 'Number of tweets in timeline'
},
x: {
type: 'log',
ticks: 5,
tickFormat: ms => dayjs().subtract(ms).fromNow()
},
// fx: {
// domain: ['followed', 'stranger'],
// label: null,
// tickSize: 2
// },
// facet: {
// data: snitchesFiltered,
// x: followedColor
// },
color:followedColorSettings,
marks: [
Plot.rectY(snitchesFiltered, Plot.binX({y: "count"}, {x: "postedMillisecondsAgo", fill:followedColor, thresholds: 'scott'})),
Plot.ruleY([0])
]
});
Insert cell
snitchesByAuthorRollup = d3.flatRollup(snitchesFiltered, g => g.length, d => d.authorHandle).filter(g => g[1] > 1)
Insert cell
snitchesByAuthor = d3.sort(d3.groups(snitchesFiltered, d => d.authorHandle), g => -(g[1].length))
.filter(g => g[0] !== 'odbol')
.filter(g => g[1].length > 1)
.slice(0, 70)
Insert cell
snitchesByAuthorMap = d3.group(snitchesFiltered, d => d.authorHandle)
Insert cell
Plot.plot({
marginLeft: 100,
padding: 0,
x: {
round: true,
grid: true,
},
fy: {
label: null,
domain: d3.groupSort(snitchesFiltered, g => g.length, d => d.authorHandle)
},
color: {
scheme: "YlGnBu"
},
facet: {
data: snitchesFiltered,
marginLeft: 100,
y: d => d.authorHandle
},
marks: [
Plot.barX(snitchesFiltered, Plot.binX({fill: "proportion-facet"}, {x: d => d.totalEngagement, inset: 0.5}))
]
})
Insert cell
Plot.plot({
marginLeft: 100,
padding: 0,
x: {
round: true,
grid: true,
},
fy: {
label: null,
domain: d3.sort(snitchesByAuthor, g => -(g[1].length)).map(d => d[0])
},
color: {
scheme: "YlGnBu"
},
facet: {
data: snitchesByAuthor,
marginLeft: 100,
y: d => d[0]
},
marks: [
Plot.barX(snitchesByAuthor, Plot.binX({fill: "proportion-facet"}, {x: d => d.totalEngagement, inset: 0.5}))
]
})
Insert cell
Plot.plot({
marginLeft: 100,
padding: 0,
width: 2000,
x: {
round: true,
grid: true,
// ticks: 5
},
y: {
label: null,
domain: d3.sort(snitchesByAuthor, g => -(g[1].length)).map(d => d[0])
},
color: {
scheme: "YlGnBu"
},
// facet: {
// data: snitchesFiltered,
// marginLeft: 100,
// y: d => d[0]
// },
marks: [
Plot.cell(snitchesFiltered.filter(tweet => snitchesByAuthor.some(g => g[0] === tweet.authorHandle)), Plot.group({fill: "count"}, {
y: d => d.authorHandle,
x: d => d.retweets,
// title: g => g[1].map((tweet, i) => i),
fill: d => d.retweets,
// inset: 0.5
}))
]
})
Insert cell
Plot.plot({
marginLeft: 100,
padding: 0,
width: 2000,
x: {
round: true,
grid: true,
// ticks: 5
},
y: {
label: null,
domain: d3.sort(snitchesByAuthor, g => -(g[1].length)).map(d => d[0])
},
color: {
scheme: "YlGnBu"
},
// facet: {
// data: snitchesFiltered,
// marginLeft: 100,
// y: d => d[0]
// },
marks: [
Plot.cell(snitchesByAuthor, Plot.group({fill: "count"}, {
y: d => d[0],
x: g => g[1].map((tweet, i) => i)
// fill: d => d[1],
// inset: 0.5
}))
]
})
Insert cell
Plot.plot({
y: {
// domain: [0, 80],
label: 'Number of tweets in timeline'
},
x: {
// type: 'log',
ticks: 3,
tickFormat: ms => dayjs().subtract(ms).fromNow()
},
// fx: {
// domain: ['followed', 'stranger'],
// label: null,
// tickSize: 2
// },
// facet: {
// data: snitchesFiltered,
// x: followedColor
// },
color:followedColorSettings,
marks: [
Plot.rectX(snitchesByAuthor, {x: g => g[1], fill:followedColor}),
// Plot.ruleY([0])
]
});
Insert cell
# Who is in my feed?

Lets see if any strangers are repeatedly showing up in my feed. It's actually quite a lot, which is surprising. I am following over 2000 people so to only see tweets from 10% of them is disconcerting and sad.
Insert cell
allFeaturedAuthorsFollowed = d3.groups(snitchesFiltered.filter(d => d.isFromFollowed), d => d.authorHandle)
Insert cell
### All the people I follow
Broken down by if they are featured in the news feed or not
Insert cell
donutRatio = DonutChart([{
name: 'Appears in Feed',
value: allFeaturedAuthorsFollowed.length
},
{
name: 'Never appears in Feed',
value: followers.length - allFeaturedAuthorsFollowed.length
}], {
name: d => d.name,
value: d => (d.value / followers.length),
format: '.0%',
// names: followedColorSettingsWithAds.domain,
colors: [ '#f4cae4', '#b3e2cd'],
width,
height: 500
})
Insert cell
mostCommonlySeen = Plot.plot({
...chartStyle,
height: 780,
marginLeft: 100,
// grid: true,
x: {
axis: "top",
round: true,
label: "Number of tweets in feed",
// labelAnchor: "center",
// tickFormat: "+",
// transform: d => d * 100
},
y: {
label: null,
domain: d3.sort(snitchesByAuthor, g => -(g[1].length)).map(d => d[0])
},
color: followedColorSettingsWithAds,
marks: [
Plot.barX(snitchesByAuthor, {
y: g => g[0],
x: g => g[1].length,
fill: g => followedColorWithAds(g[1][0])
}),
Plot.ruleX([0])
]
})
Insert cell
Insert cell
Insert cell
authorGridLables = Plot.plot({
...chartStyle,
height: 780,
marginLeft: 100,
// grid: true,
x: {
axis: "top",
round: true,
label: "Number of tweets in feed",
// labelAnchor: "center",
// tickFormat: "+",
// transform: d => d * 100
},
y: {
axis: null,
label: null,
domain: d3.sort(snitchesByAuthor, g => -(g[1].length)).map(d => d[0]),
fontVariant: 'small-caps',
fontWeight: 'bold'
},
color: {
// domain: [0, 10000],
type: 'log',
// scheme: "magma",
scheme: 'YlGnBu',
label: "Total engagement (Retweets + Likes + Replies)",
legend: true
},
marks: [
Plot.cell(snitchesByAuthor
.map(g => {
const tweetsIndexed = d3.sort(g[1], d => d.totalEngagement)
.map((d, index) => {
return {
index,
...d
};
});
return [g[0], tweetsIndexed];
})
.flatMap(g => g[1]), {
y: d => d.authorHandle,
x: d => d.index,
fill: d => d.totalEngagement,
title: d => `${d.index} ${d.retweets} + ${d.likes}`,
// padding:0,
// margin:0
}),
Plot.text(d3.sort(snitchesByAuthor, g => -(g[1].length)).map(d => d[0]), {
// y: "authorHandle",
y: 200,
x: 20,
text: 'lkasjdfkjl',//d => d.authorHandle,
title: "title"
})
]
})
Insert cell
followedColorSequentialScheme = d3.scaleSequentialLog(d3.interpolatePuOr).domain([200000, 1])
Insert cell
followedColorSequentialScheme(10000)

Insert cell
authorGridWColors = Plot.plot({
...chartStyle,
height: 780,
marginLeft: 100,
// grid: true,
x: {
axis: "top",
round: true,
label: "Number of tweets in feed",
// labelAnchor: "center",
// tickFormat: "+",
// transform: d => d * 100
},
y: {
label: null,
domain: d3.sort(snitchesByAuthor, g => -(g[1].length)).map(d => d[0])
},
// color: {
// // domain: [0, 10000],
// type: 'log',
// // scheme: "magma",
// scheme: 'YlGnBu',
// label: "Total engagement (Retweets + Likes + Replies)",
// legend: true
// },
marks: [
Plot.cell(snitchesByAuthor
.map(g => {
const tweetsIndexed = d3.sort(g[1], d => d.totalEngagement)
.map((d, index) => {
return {
index,
...d
};
});
return [g[0], tweetsIndexed];
})
.flatMap(g => g[1]), {
y: d => d.authorHandle,
x: d => d.index,
fill: d => followedColorSequentialScheme(d.totalEngagement),
title: d => `${d.index} ${d.retweets} + ${d.likes}`,
// padding:0,
// margin:0
}),
// Plot.ruleX([0])
// Plot.text(d3.sort(snitchesByAuthor, g => -(g[1].length)).map(d => d[0]), {
// x: "season",
// y: "authorHandle",
// text: d => d.authorHandle,
// title: "title"
// })
]
})
Insert cell
snitchesByRecommendation = d3.sort(d3.groups(snitchesFiltered, d => {
const whoRecommended = d.whoLiked.concat(d.whoRetweeted, d.whoReplied, d.followedBy)
.filter(d => !/\d+ others/.test(d))
.map(d => d.trim());
console.log('who', whoRecommended);
if (!whoRecommended) return null;
return whoRecommended[0];
})
, g => -(g[1].length))
// remove undefined (i.e. tweets that weren't recommended)
.filter(g => g[0])
.map(g => {
// map author name to author handles
const author = followers.find(f => f.name.includes(g[0]) || g[0].includes(f.name));
return [author ? author.handle.substr(1) : g[0], g[1]];
})
// .filter(g => g[0] !== 'odbol')
.filter(g => g[1].length > 1)
.slice(0, 70)
Insert cell
tastemakers = Plot.plot({
...chartStyle,
height: 780,
marginLeft: 100,
// grid: true,
x: {
axis: "top",
round: true,
label: "Number of recommendations in feed",
// labelAnchor: "center",
// tickFormat: "+",
// transform: d => d * 100
},
y: {
label: null,
domain: d3.sort(snitchesByRecommendation, g => -(g[1].length)).map(d => d[0])
},
color: followedColorSettingsWithAds,
marks: [
Plot.barX(snitchesByRecommendation, {
y: g => g[0],
x: g => g[1].length,
fill: g => followedColorWithAds(g[1][0])
}),
Plot.ruleX([0])
]
})
Insert cell
WordCloud(snitchesFiltered.map(d => d.tweet
.split(/\W+/ig))
.reduce((acc, d) => acc.concat(d),[])
.map(w => w.replace(/^[“‘"\-—()\[\]{}]+/g, ""))
.map(w => w.replace(/[;:.!?()\[\]{},"'’”\-—]+$/g, ""))
.map(w => w.replace(/['’]s$/g, ""))
.map(w => w.toLowerCase())
.filter(w => w && !stopwords.has(w)), {
width,
height: 500,
invalidation // a promise to stop the simulation when the cell is re-run
})
Insert cell
chartStyle = (() => {
return {
width: 720,
height: 480,
fontFamily: 'sans-serif'
}
})()
Insert cell
dayjs = require('https://unpkg.com/dayjs@1.8.21/dayjs.min.js')
Insert cell
dayjsRelativeTime = require('https://unpkg.com/dayjs@1.8.21/plugin/relativeTime.js')
Insert cell
dayjs.extend(dayjsRelativeTime)
Insert cell
dayjs(2034809234).fromNow()
Insert cell
import {DonutChart} from "@d3/donut-chart"
Insert cell
import {WordCloud, stopwords} from "@d3/word-cloud"
Insert cell

Purpose-built for displays of data

Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
Learn more