Principal Web Developer at Freedom of the Press Foundation
COVID_DATA_URL = 'https://corsproxy.harrislapiroff.com/https://docs.google.com/spreadsheets/d/e/2PACX-1vTfUQPxkhP_CRcGmnnpUBihnTNZ9Z8pcizII4_sc2o2n3opOoAJdAM4CRTJBI339tou8LWnQrqbTMgH/pub?gid=902690690&single=true&output=csv'
CENSUS_DATA_URL = FileAttachment("ACSDT1Y2018.B02001_data_with_overlays_2020-05-19T011212.csv").url()
parseDateCol = d3.timeParse('%Y%m%d')
parseUpdateTimeCol = d => {
let returnVal = d3.timeParse('%-m/%-d %-H:%M')(d)
if (!returnVal) return // If date is null, short-circuit
returnVal.setFullYear(2020)
return returnVal
}
n = x => {
// Return null if the empty string, null, or undefined convert to number otherwise
if (x === '' || typeof x === 'undefined' || x === null) return null
return +x
}
parseRaceData = (datum, startIdx) => ({
total: n(datum[startIdx]),
// Race
white: n(datum[startIdx + 1]),
black: n(datum[startIdx + 2]),
latinxHispanic: n(datum[startIdx + 3]),
asian: n(datum[startIdx + 4]),
aian: n(datum[startIdx + 5]), // American Indian/Alaska Native
nhpi: n(datum[startIdx + 6]), // Native Hawaiian/Pacific Islander
multiracial: n(datum[startIdx + 7]),
other: n(datum[startIdx + 8]),
unknownRace: n(datum[startIdx + 9]),
// Ethnicity
hispanic: n(datum[startIdx + 11]),
nonHispanic: n(datum[startIdx + 12]),
unknownEth: n(datum[startIdx + 13]),
})
parseDataCSV = dataStr => {
return d3.csvParseRows(dataStr, (d, i) => {
if (i < 3) return // The first three rows are headers -- skip
return {
date: parseDateCol(d[0]),
state: d[1],
updateTimeET: parseUpdateTimeCol(d[4]),
lastCheck: parseUpdateTimeCol(d[5]),
positives: parseRaceData(d, 8),
deaths: parseRaceData(d, 23),
negatives: parseRaceData(d, 38),
}
})
}
covidData = parseDataCSV(await (await fetch(COVID_DATA_URL)).text())
latestCovidData = Object.keys(US_STATES).map(state => {
// For each state we select all rows that match that state,
// sort in descending update time order, and grab the
// topmost record
return covidData.filter(d => d.state === state).sort((a, b) => b.updateTimeET - a.updateTimeET)[0]
}).filter(x => !!x) // Filter out states which had no entries (mostly actually territories)
statePopData = {
let text = await (await fetch(CENSUS_DATA_URL)).text()
text = text.substring(text.indexOf("\n") + 1) // The first row is useless as headers, so we delete it
return d3.csvParse(text, d => ({
id: d.id,
state: stateNameToAbbr(d['Geographic Area Name']),
total: {
value: +d['Estimate!!Total'],
marginErr: +d['Margin of Error!!Total'],
},
white: {
value: +d['Estimate!!Total!!White alone'],
marginErr: +d['Margin of Error!!Total!!White alone'],
},
black: {
value: +d['Estimate!!Total!!Black or African American alone'],
marginErr: +d['Margin of Error!!Total!!Black or African American alone'],
},
aian: {
value: +d['Estimate!!Total!!American Indian and Alaska Native alone'],
marginErr: +d['Margin of Error!!Total!!American Indian and Alaska Native alone'],
},
asian: {
value: +d['Estimate!!Total!!Asian alone'],
marginErr: +d['Margin of Error!!Total!!Asian alone'],
},
nhpi: {
value: +d['Estimate!!Total!!Native Hawaiian and Other Pacific Islander alone'],
marginErr: +d['Margin of Error!!Total!!Native Hawaiian and Other Pacific Islander alone'],
},
multiracial: {
value: +d['Estimate!!Total!!Two or more races'],
marginErr: +d['Margin of Error!!Total!!Two or more races'],
},
other: {
value: +d['Estimate!!Total!!Some other race alone'],
marginErr: +d['Margin of Error!!Total!!Some other race alone'],
},
}))
}
combinedData = statePopData.map(({
id, state, total, ...categories
}) => {
const covidData = latestCovidData.find(d => d.state === state)
if (!covidData) return;
// Zip the data into a single array
const returnData = Object.entries(categories).reduce((acc, [cat, pop]) => {
const popValue = pop.value
const popTopErr = pop.value + pop.marginErr
const popBottomErr = pop.value - pop.marginErr
acc.positives[cat] = {
percent: covidData.positives[cat] / pop.value,
percentTopErr: covidData.positives[cat] / popBottomErr,
percentBottomErr: covidData.positives[cat] / popTopErr,
count: covidData.positives[cat],
totalPopulation: pop.value,
marginErr: pop.marginErr,
}
acc.deaths[cat] = {
percent: covidData.deaths[cat] / pop.value,
percentTopErr: covidData.deaths[cat] / popBottomErr,
percentBottomErr: covidData.deaths[cat] / popTopErr,
count: covidData.deaths[cat],
totalPopulation: pop.value,
marginErr: pop.marginErr,
}
acc.negatives[cat] = {
percent: covidData.negatives[cat] / pop.value,
percentTopErr: covidData.negatives[cat] / popBottomErr,
percentBottomErr: covidData.negatives[cat] / popTopErr,
count: covidData.negatives[cat],
}
return acc
}, {
state: state,
positives: {},
deaths: {},
negatives: {}
}
)
// Add total and unknown race info
;['positives', 'deaths', 'negatives'].forEach(s => {
returnData[s]._meta = {
total: covidData[s].total,
unknownRace: covidData[s].unknownRace,
}
})
// Add state pop info
returnData.totalPopulation = total.value
return returnData
}).filter(x => !!x)
stateChart = (
data,
chartType = 'deaths',
chartWidth = width / 3,
chartHeight = width * 0.618 / 3,
) => {
const dotWidth = chartWidth * 0.03
const lineWidth = chartWidth * 0.03
// Layout Configuration
const margin = { left: 0, top: dotWidth * 2, bottom: dotWidth * 1.5, right: 0 }
const bottomAxisOffset = 10
const innerWidth = chartWidth - margin.left - margin.right
const innerHeight = chartHeight - margin.top - margin.bottom
const dataAsPairs = Object.entries(data[chartType]).filter((([key, value]) => key !== '_meta'))
// Calculate a good perN value
const minRatio = d3.quantile(
dataAsPairs,
0.5,
([key, details]) => details.percent > 0 ? details.percent : null
)
const perN = reasonableDenominator(minRatio)
const perNFormat = n => {
let returnValue = d3.format(',d')(n * perN)
if (returnValue === '0' && n > 0) returnValue = d3.format('.1r')(n * perN)
return returnValue
}
// Scales
const yMax = Math.max(
d3.max(dataAsPairs, ([eth, d]) => d.percentTopErr),
// Don't show with the max smaller than 0.00001%
1 / 100000
)
const yScale = d3.scaleLinear()
.domain([yMax, 0])
.range([0, innerHeight])
.nice()
const xScale = d3.scalePoint()
.domain(dataAsPairs.map(([eth, d]) => eth))
.range([0, innerWidth])
.padding(1)
// Chart
const chart = d3.select(DOM.svg(chartWidth, chartHeight))
// Axes and Borders
const yAxis = d3.axisLeft(yScale)
.ticks(5)
.tickFormat(perNFormat)
.tickSize(-innerWidth)
chart.append('g')
.attr('transform', `translate(${margin.left}, ${margin.top})`)
.call(yAxis)
.call(g => g.select('.domain').remove())
.call(g => g.selectAll(".tick line").attr('color', '#E5E5E5'))
.call(
// move ticks above the grid lines
g => g.selectAll(".tick text")
.attr('x', 3)
.attr('text-anchor', 'start')
.attr('dy', -5)
.attr('fill', '#666')
)
// Borders
chart.append('rect')
.attr('x', 0.5)
.attr('y', 0.5)
.attr('width', chartWidth - 1)
.attr('height', chartHeight - 1)
.attr('stroke', '#E5E5E5')
.attr('fill', 'none')
// Inner Chart
const innerChart = chart.append('g')
.attr('transform', `translate(${margin.left}, ${margin.top})`)
const dataSelection = innerChart.selectAll()
.data(dataAsPairs)
.enter()
// Margin of Error
dataSelection.append('line')
.attr('x1', ([eth, d]) => xScale(eth))
.attr('x2', ([eth, d]) => xScale(eth))
.attr('y1', ([eth, d]) => yScale(d.percentTopErr))
.attr('y2', ([eth, d]) => yScale(d.percentBottomErr))
.attr('stroke', ([eth, d]) => RACES[eth].color)
.attr('stroke-width', lineWidth)
.attr('stroke-linecap', 'round')
.attr('opacity', 0.15)
// Data Point
dataSelection.append('circle')
.attr('cx', ([eth, d]) => xScale(eth))
.attr('cy', ([eth, d]) => yScale(d.percent))
.attr('r', dotWidth)
.attr('fill', ([eth, d]) => RACES[eth].color)
.attr('stroke', '#FFF')
.attr('stroke-width', 1.5)
.append('title').text(([eth, d]) => RACES[eth].label)
// Data Point Number Label
dataSelection.append('text')
.attr('x', ([eth, d]) => xScale(eth) + dotWidth * 1.5)
.attr('y', ([eth, d]) => yScale(d.percent) + dotWidth * 0.5)
.attr('font-size', 12)
.attr('font-family', 'sans-serif')
.attr('fill', '#777')
.attr('cursor', 'default')
.text(([eth, d]) => perNFormat(d.percent))
// Data Point Label
dataSelection.append('text')
.attr('x', ([eth, d]) => xScale(eth))
.attr('y', ([eth, d]) => yScale(d.percent) + dotWidth * 0.4)
.attr('text-anchor', 'middle')
.attr('text-align', 'center')
.attr('font-size', dotWidth)
.attr('font-family', 'sans-serif')
.attr('fill', '#FFF')
.attr('cursor', 'default')
.text(([eth, d]) => RACES[eth].abbr)
.append('title').text(([eth, d]) => RACES[eth].label)
const unknownPct = data[chartType]._meta.unknownRace / data[chartType]._meta.total
const unknownColor = R.cond([
[R.gt(0.2), R.always('#777')],
[R.gt(0.5), R.always('#980')],
[R.T, R.always('#800')],
])(unknownPct)
return html`
<div class="state-chart">
<div class="state-chart__details">
<div class="state-chart__title">${US_STATES[data.state]}</div>
<div class="state-chart__notes">
${capitalize(chartType)} per ${d3.format(',d')(perN)} •
<strong>Unknown Race:</strong>
<span style="color: ${unknownColor}">
${d3.format('.0%')(unknownPct)}
</span>
<br />
<strong>Pop:</strong> ${d3.format(',d')(data.totalPopulation)} •
<strong>${capitalize(chartType)}:</strong> ${d3.format(',d')(data[chartType]._meta.total)}
</div>
</div>
<div class="state-chart__chart">
${chart.node()}
${stateTable(data, chartType)}
</div>
</div>`
}
stateTable = (data, chartType) => {
const dataAsPairs = Object.entries(data[chartType]).filter((([key, value]) => key !== '_meta'))
return html`
<table class="state-table">
<thead>
<tr>
<th></th>
<th class="state-table__figure-head">Population</th>
<th class="state-table__figure-head"> </th>
<th class="state-table__figure-head">${capitalize(chartType)}</th>
</tr>
</thead>
<tbody>
${dataAsPairs.map(([r, d]) => html`
<tr>
<th class="state-table__race" style="background-color: ${RACES[r].color};" title="${RACES[r].label}">
${RACES[r].abbr}
</th>
<td class="state-table__figure-cell">
${d3.format(',d')(d.totalPopulation)}
</td>
<td class="state-table__figure-cell state-table__figure-cell--error">
±${d3.format(',d')(d.marginErr)}
</td>
<td class="state-table__figure-cell">
${d.count || ''}
</td>
</tr>
`)}
</tbody>
</table>
`
}
html`
<style type="text/css">
.chart-list {
display: flex;
flex-wrap: wrap;
justify-content: space-between;
}
.state-chart {
margin-bottom: 30px
}
.state-chart__details {
margin-bottom: 5px;
}
.state-chart__title {
font-weight: bold;
}
.state-chart__notes {
font-family: sans-serif;
font-size: 12px;
color: #666;
}
.state-table__figure-head {
text-align: right;
}
.state-table__figure-cell {
font-family: sans-serif;
font-variant-numeric: tabular-nums;
text-align: right;
}
.state-table__figure-cell--error {
color: #999;
width: 1px; /* shrink to align with left cell */
text-align: left;
font-weight: 300;
}
.state-table__race {
color: #FFF;
text-align: center;
width: 2em;
font-weight: bold;
}
</style>
`
chartWidth = {
if (width > 1200) {
return (width / 5) - 10
}
if (width > 800) {
return (width / 3) - 10
} else if (width > 600) {
return (width / 2) - 10
} else {
return width
}
}
RACES = ({
white: { abbr: 'W', label: 'White', color: "#158" },
black: { abbr: 'B', label: 'Black', color: "#752" },
aian: { abbr: 'AI', label: 'American Indian/Alaska Native', color: "#364" },
asian: { abbr: 'A', label: 'Asian', color: "#822" },
nhpi: { abbr: 'NH', label: 'Native Hawaiian/Pacific Islander', color: "#638" },
multiracial: { abbr: 'M', label: 'Multiracial', color: "#378" },
other: { abbr: 'O', label: 'Other', color: "#444" },
})
stateNameToAbbr = n => R.invertObj(US_STATES)[n]
capitalize = s => s.charAt(0).toUpperCase() + s.slice(1)
reasonableDenominator = n => {
// Count the number of zeroes before the first digit
const zeroes = Math.ceil(-Math.log10(n))
if (zeroes === Infinity || zeroes === 0) return 10000
// Return 10^{zeroes}
return Math.pow(10, zeroes)
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
