Principal Web Developer at Freedom of the Press Foundation
Beantown Stomp 2023 CO₂ LevelsAssorted CO₂ ReadingsNYC SARS-CoV-2 Wastewater DataAranet4 Data UtilitiesFreedom of the Press Foundation 2022 All Staff CO₂ LevelsCO₂ Measurements for ESCape at Pinewoods Camp 2022Boston-Area SARS-CoV-2 Wastewater LevelsCharting a Case Fatality Rate by WaveCharting a Moving Case Fatality RateWhich states’ worst pandemic days were this week?Covid Cases vs. Deaths and HospitalizationsU.S. Air Travel in a PandemicMassachusetts COVID DataMassachusetts Coronavirus Cases by TownWhere is COVID-19 rising and falling in the U.S.?How many SARS-CoV-2 tests are we running in the U.S.?
COVID-19 Infection and Fatality by Race
COVID_DATA_URL = 'https://corsproxy.harrislapiroff.com/https://docs.google.com/spreadsheets/d/e/2PACX-1vTfUQPxkhP_CRcGmnnpUBihnTNZ9Z8pcizII4_sc2o2n3opOoAJdAM4CRTJBI339tou8LWnQrqbTMgH/pub?gid=902690690&single=true&output=csv'
CENSUS_DATA_URL = FileAttachment("ACSDT1Y2018.B02001_data_with_overlays_2020-05-19T011212.csv").url()
parseDateCol = d3.timeParse('%Y%m%d')
parseUpdateTimeCol = d => {
let returnVal = d3.timeParse('%-m/%-d %-H:%M')(d)
if (!returnVal) return // If date is null, short-circuit
returnVal.setFullYear(2020)
return returnVal
}
n = x => {
// Return null if the empty string, null, or undefined convert to number otherwise
if (x === '' || typeof x === 'undefined' || x === null) return null
return +x
}
parseRaceData = (datum, startIdx) => ({
total: n(datum[startIdx]),
// Race
white: n(datum[startIdx + 1]),
black: n(datum[startIdx + 2]),
latinxHispanic: n(datum[startIdx + 3]),
asian: n(datum[startIdx + 4]),
aian: n(datum[startIdx + 5]), // American Indian/Alaska Native
nhpi: n(datum[startIdx + 6]), // Native Hawaiian/Pacific Islander
multiracial: n(datum[startIdx + 7]),
other: n(datum[startIdx + 8]),
unknownRace: n(datum[startIdx + 9]),
// Ethnicity
hispanic: n(datum[startIdx + 11]),
nonHispanic: n(datum[startIdx + 12]),
unknownEth: n(datum[startIdx + 13]),
})
parseDataCSV = dataStr => {
return d3.csvParseRows(dataStr, (d, i) => {
if (i < 3) return // The first three rows are headers -- skip
return {
date: parseDateCol(d[0]),
state: d[1],
updateTimeET: parseUpdateTimeCol(d[4]),
lastCheck: parseUpdateTimeCol(d[5]),
positives: parseRaceData(d, 8),
deaths: parseRaceData(d, 23),
negatives: parseRaceData(d, 38),
}
})
}
covidData = parseDataCSV(await (await fetch(COVID_DATA_URL)).text())
latestCovidData = Object.keys(US_STATES).map(state => {
// For each state we select all rows that match that state,
// sort in descending update time order, and grab the
// topmost record
return covidData.filter(d => d.state === state).sort((a, b) => b.updateTimeET - a.updateTimeET)[0]
}).filter(x => !!x) // Filter out states which had no entries (mostly actually territories)
statePopData = {
let text = await (await fetch(CENSUS_DATA_URL)).text()
text = text.substring(text.indexOf("\n") + 1) // The first row is useless as headers, so we delete it
return d3.csvParse(text, d => ({
id: d.id,
state: stateNameToAbbr(d['Geographic Area Name']),
total: {
value: +d['Estimate!!Total'],
marginErr: +d['Margin of Error!!Total'],
},
white: {
value: +d['Estimate!!Total!!White alone'],
marginErr: +d['Margin of Error!!Total!!White alone'],
},
black: {
value: +d['Estimate!!Total!!Black or African American alone'],
marginErr: +d['Margin of Error!!Total!!Black or African American alone'],
},
aian: {
value: +d['Estimate!!Total!!American Indian and Alaska Native alone'],
marginErr: +d['Margin of Error!!Total!!American Indian and Alaska Native alone'],
},
asian: {
value: +d['Estimate!!Total!!Asian alone'],
marginErr: +d['Margin of Error!!Total!!Asian alone'],
},
nhpi: {
value: +d['Estimate!!Total!!Native Hawaiian and Other Pacific Islander alone'],
marginErr: +d['Margin of Error!!Total!!Native Hawaiian and Other Pacific Islander alone'],
},
multiracial: {
value: +d['Estimate!!Total!!Two or more races'],
marginErr: +d['Margin of Error!!Total!!Two or more races'],
},
other: {
value: +d['Estimate!!Total!!Some other race alone'],
marginErr: +d['Margin of Error!!Total!!Some other race alone'],
},
}))
}
combinedData = statePopData.map(({
id, state, total, ...categories
}) => {
const covidData = latestCovidData.find(d => d.state === state)
if (!covidData) return;
// Zip the data into a single array
const returnData = Object.entries(categories).reduce((acc, [cat, pop]) => {
const popValue = pop.value
const popTopErr = pop.value + pop.marginErr
const popBottomErr = pop.value - pop.marginErr
acc.positives[cat] = {
percent: covidData.positives[cat] / pop.value,
percentTopErr: covidData.positives[cat] / popBottomErr,
percentBottomErr: covidData.positives[cat] / popTopErr,
count: covidData.positives[cat],
totalPopulation: pop.value,
marginErr: pop.marginErr,
}
acc.deaths[cat] = {
percent: covidData.deaths[cat] / pop.value,
percentTopErr: covidData.deaths[cat] / popBottomErr,
percentBottomErr: covidData.deaths[cat] / popTopErr,
count: covidData.deaths[cat],
totalPopulation: pop.value,
marginErr: pop.marginErr,
}
acc.negatives[cat] = {
percent: covidData.negatives[cat] / pop.value,
percentTopErr: covidData.negatives[cat] / popBottomErr,
percentBottomErr: covidData.negatives[cat] / popTopErr,
count: covidData.negatives[cat],
}
return acc
}, {
state: state,
positives: {},
deaths: {},
negatives: {}
}
)
// Add total and unknown race info
;['positives', 'deaths', 'negatives'].forEach(s => {
returnData[s]._meta = {
total: covidData[s].total,
unknownRace: covidData[s].unknownRace,
}
})
// Add state pop info
returnData.totalPopulation = total.value
return returnData
}).filter(x => !!x)
stateChart = (
data,
chartType = 'deaths',
chartWidth = width / 3,
chartHeight = width * 0.618 / 3,
) => {
const dotWidth = chartWidth * 0.03
const lineWidth = chartWidth * 0.03
// Layout Configuration
const margin = { left: 0, top: dotWidth * 2, bottom: dotWidth * 1.5, right: 0 }
const bottomAxisOffset = 10
const innerWidth = chartWidth - margin.left - margin.right
const innerHeight = chartHeight - margin.top - margin.bottom
const dataAsPairs = Object.entries(data[chartType]).filter((([key, value]) => key !== '_meta'))
// Calculate a good perN value
const minRatio = d3.quantile(
dataAsPairs,
0.5,
([key, details]) => details.percent > 0 ? details.percent : null
)
const perN = reasonableDenominator(minRatio)
const perNFormat = n => {
let returnValue = d3.format(',d')(n * perN)
if (returnValue === '0' && n > 0) returnValue = d3.format('.1r')(n * perN)
return returnValue
}
// Scales
const yMax = Math.max(
d3.max(dataAsPairs, ([eth, d]) => d.percentTopErr),
// Don't show with the max smaller than 0.00001%
1 / 100000
)
const yScale = d3.scaleLinear()
.domain([yMax, 0])
.range([0, innerHeight])
.nice()
const xScale = d3.scalePoint()
.domain(dataAsPairs.map(([eth, d]) => eth))
.range([0, innerWidth])
.padding(1)
// Chart
const chart = d3.select(DOM.svg(chartWidth, chartHeight))
// Axes and Borders
const yAxis = d3.axisLeft(yScale)
.ticks(5)
.tickFormat(perNFormat)
.tickSize(-innerWidth)
chart.append('g')
.attr('transform', `translate(${margin.left}, ${margin.top})`)
.call(yAxis)
.call(g => g.select('.domain').remove())
.call(g => g.selectAll(".tick line").attr('color', '#E5E5E5'))
.call(
// move ticks above the grid lines
g => g.selectAll(".tick text")
.attr('x', 3)
.attr('text-anchor', 'start')
.attr('dy', -5)
.attr('fill', '#666')
)
// Borders
chart.append('rect')
.attr('x', 0.5)
.attr('y', 0.5)
.attr('width', chartWidth - 1)
.attr('height', chartHeight - 1)
.attr('stroke', '#E5E5E5')
.attr('fill', 'none')
// Inner Chart
const innerChart = chart.append('g')
.attr('transform', `translate(${margin.left}, ${margin.top})`)
const dataSelection = innerChart.selectAll()
.data(dataAsPairs)
.enter()
// Margin of Error
dataSelection.append('line')
.attr('x1', ([eth, d]) => xScale(eth))
.attr('x2', ([eth, d]) => xScale(eth))
.attr('y1', ([eth, d]) => yScale(d.percentTopErr))
.attr('y2', ([eth, d]) => yScale(d.percentBottomErr))
.attr('stroke', ([eth, d]) => RACES[eth].color)
.attr('stroke-width', lineWidth)
.attr('stroke-linecap', 'round')
.attr('opacity', 0.15)
// Data Point
dataSelection.append('circle')
.attr('cx', ([eth, d]) => xScale(eth))
.attr('cy', ([eth, d]) => yScale(d.percent))
.attr('r', dotWidth)
.attr('fill', ([eth, d]) => RACES[eth].color)
.attr('stroke', '#FFF')
.attr('stroke-width', 1.5)
.append('title').text(([eth, d]) => RACES[eth].label)
// Data Point Number Label
dataSelection.append('text')
.attr('x', ([eth, d]) => xScale(eth) + dotWidth * 1.5)
.attr('y', ([eth, d]) => yScale(d.percent) + dotWidth * 0.5)
.attr('font-size', 12)
.attr('font-family', 'sans-serif')
.attr('fill', '#777')
.attr('cursor', 'default')
.text(([eth, d]) => perNFormat(d.percent))
// Data Point Label
dataSelection.append('text')
.attr('x', ([eth, d]) => xScale(eth))
.attr('y', ([eth, d]) => yScale(d.percent) + dotWidth * 0.4)
.attr('text-anchor', 'middle')
.attr('text-align', 'center')
.attr('font-size', dotWidth)
.attr('font-family', 'sans-serif')
.attr('fill', '#FFF')
.attr('cursor', 'default')
.text(([eth, d]) => RACES[eth].abbr)
.append('title').text(([eth, d]) => RACES[eth].label)
const unknownPct = data[chartType]._meta.unknownRace / data[chartType]._meta.total
const unknownColor = R.cond([
[R.gt(0.2), R.always('#777')],
[R.gt(0.5), R.always('#980')],
[R.T, R.always('#800')],
])(unknownPct)
return html`
<div class="state-chart">
<div class="state-chart__details">
<div class="state-chart__title">${US_STATES[data.state]}</div>
<div class="state-chart__notes">
${capitalize(chartType)} per ${d3.format(',d')(perN)} •
<strong>Unknown Race:</strong>
<span style="color: ${unknownColor}">
${d3.format('.0%')(unknownPct)}
</span>
<br />
<strong>Pop:</strong> ${d3.format(',d')(data.totalPopulation)} •
<strong>${capitalize(chartType)}:</strong> ${d3.format(',d')(data[chartType]._meta.total)}
</div>
</div>
<div class="state-chart__chart">
${chart.node()}
${stateTable(data, chartType)}
</div>
</div>`
}
stateTable = (data, chartType) => {
const dataAsPairs = Object.entries(data[chartType]).filter((([key, value]) => key !== '_meta'))
return html`
<table class="state-table">
<thead>
<tr>
<th></th>
<th class="state-table__figure-head">Population</th>
<th class="state-table__figure-head"> </th>
<th class="state-table__figure-head">${capitalize(chartType)}</th>
</tr>
</thead>
<tbody>
${dataAsPairs.map(([r, d]) => html`
<tr>
<th class="state-table__race" style="background-color: ${RACES[r].color};" title="${RACES[r].label}">
${RACES[r].abbr}
</th>
<td class="state-table__figure-cell">
${d3.format(',d')(d.totalPopulation)}
</td>
<td class="state-table__figure-cell state-table__figure-cell--error">
±${d3.format(',d')(d.marginErr)}
</td>
<td class="state-table__figure-cell">
${d.count || ''}
</td>
</tr>
`)}
</tbody>
</table>
`
}
html`
<style type="text/css">
.chart-list {
display: flex;
flex-wrap: wrap;
justify-content: space-between;
}
.state-chart {
margin-bottom: 30px
}
.state-chart__details {
margin-bottom: 5px;
}
.state-chart__title {
font-weight: bold;
}
.state-chart__notes {
font-family: sans-serif;
font-size: 12px;
color: #666;
}
.state-table__figure-head {
text-align: right;
}
.state-table__figure-cell {
font-family: sans-serif;
font-variant-numeric: tabular-nums;
text-align: right;
}
.state-table__figure-cell--error {
color: #999;
width: 1px; /* shrink to align with left cell */
text-align: left;
font-weight: 300;
}
.state-table__race {
color: #FFF;
text-align: center;
width: 2em;
font-weight: bold;
}
</style>
`
chartWidth = {
if (width > 1200) {
return (width / 5) - 10
}
if (width > 800) {
return (width / 3) - 10
} else if (width > 600) {
return (width / 2) - 10
} else {
return width
}
}
RACES = ({
white: { abbr: 'W', label: 'White', color: "#158" },
black: { abbr: 'B', label: 'Black', color: "#752" },
aian: { abbr: 'AI', label: 'American Indian/Alaska Native', color: "#364" },
asian: { abbr: 'A', label: 'Asian', color: "#822" },
nhpi: { abbr: 'NH', label: 'Native Hawaiian/Pacific Islander', color: "#638" },
multiracial: { abbr: 'M', label: 'Multiracial', color: "#378" },
other: { abbr: 'O', label: 'Other', color: "#444" },
})
stateNameToAbbr = n => R.invertObj(US_STATES)[n]
capitalize = s => s.charAt(0).toUpperCase() + s.slice(1)
reasonableDenominator = n => {
// Count the number of zeroes before the first digit
const zeroes = Math.ceil(-Math.log10(n))
if (zeroes === Infinity || zeroes === 0) return 10000
// Return 10^{zeroes}
return Math.pow(10, zeroes)
}
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.