Principal Web Developer at Freedom of the Press Foundation
Published
Edited
Aug 25, 2020
6 stars
Beantown Stomp 2023 CO₂ LevelsAssorted CO₂ ReadingsNYC SARS-CoV-2 Wastewater DataAranet4 Data UtilitiesFreedom of the Press Foundation 2022 All Staff CO₂ LevelsCO₂ Measurements for ESCape at Pinewoods Camp 2022Boston-Area SARS-CoV-2 Wastewater LevelsCharting a Case Fatality Rate by WaveCharting a Moving Case Fatality RateWhich states’ worst pandemic days were this week?Covid Cases vs. Deaths and HospitalizationsU.S. Air Travel in a PandemicMassachusetts COVID DataMassachusetts Coronavirus Cases by TownCOVID-19 Infection and Fatality by RaceWhere is COVID-19 rising and falling in the U.S.?
How many SARS-CoV-2 tests are we running in the U.S.?
Also listed in…
Coronavirus
usData = (await fetch('https://corsproxy.harrislapiroff.com/https://covidtracking.com/api/v1/us/daily.json')).json()
statesData = (await fetch('https://corsproxy.harrislapiroff.com/https://covidtracking.com/api/v1/states/daily.json')).json()
annotateData = (data, movingAverageN) => {
data.sort((a, b) => a.date - b.date)
return data.map((d, i, a) => {
const startIdx = i < movingAverageN ? 0 : i + 1 - movingAverageN
const dataToAverage = a.slice(startIdx, i + 1)
return {
...d,
positiveIncreaseSMA: average(dataToAverage.map(getPosTestCount)),
negativeIncreaseSMA: average(dataToAverage.map(getNegTestCount)),
totalIncreaseSMA: average(dataToAverage.map(getTestCount)),
}
})
}
activeData = annotateData(
dataFilter === 'ALL' ? usData : statesData.filter(d => d.state === dataFilter),
MOVING_AVERAGE_N
).filter(d => getDate(d) >= DATA_START_DATE)
height = width * .618
xScale = d3.scaleTime()
.domain([DATA_START_DATE, d3.max(usData, getDate)])
.range([margin.left, width - margin.right])
yScale = d3.scaleLinear()
.domain([0, d3.max(activeData, getTestCount)])
.range([height - margin.bottom, margin.top])
.nice()
xAxis = g => g.attr('transform', `translate(0, ${height - margin.bottom})`)
.call(
d3.axisBottom(xScale)
.ticks(width > 400 ? 15 : 5) // Fewer ticks for smaller screen
.tickFormat(d3.timeFormat(width > 400 ? '%b %d' : '%m/%d')) // Concise dates for smaller screen
)
yAxis = g => g.attr('transform', `translate(${margin.left}, 0)`)
.attr('class', 'grid')
.call(
d3.axisLeft(yScale)
.ticks(5)
.tickSize(-width + margin.left + margin.right)
)
.call(g => g.select('.domain').remove())
totalTestsSMALine = d3.area()
.defined(d => !isNaN(getPosSMA(d)))
.x(d => xScale(getDate(d)))
.y1(d => yScale(getTotalSMA(d)))
.y0(d => yScale(getPosSMA(d)) - 1)
posTestsSMALine = d3.area()
.defined(d => !isNaN(getPosSMA(d)))
.x(d => xScale(getDate(d)))
.y1(d => yScale(getPosSMA(d)))
.y0(d => yScale(0))
addInteractivity = (
xScale, // d3 scale
yScale, // d3 scale
g // d3 svg
) => {
const line = g.append('g')
.append('line')
.attr('stroke', 'rgba(0, 0, 0, 0.5)')
.attr('stroke-width', 1)
.attr('x1', 100)
.attr('y1', yScale.range()[0])
.attr('x2', 100)
.attr('y2', yScale.range()[1])
.attr('opacity', 0)
// We're "cheat" a little at interactivity by leveraging Obervable's views functionality
// We start by setting the value of the DOM node to null and then update it according to mouse events
const node = g.node()
node.value = null
const inputEvent = new CustomEvent("input")
g.on('mouseover', function() {
line.attr('opacity', 1)
})
g.on('mouseout', function() {
line.attr('opacity', 0)
node.value = null
node.dispatchEvent(inputEvent)
})
g.on('mousemove', function() {
const mouse = d3.mouse(this)
const xDomainVal = xScale.invert(mouse[0])
// Add 12 hours and clip off the time to round to the nearest midnight
const xDomainValRounded = (new Date(+xDomainVal + 43200000)).setHours(0, 0, 0, 0)
const xRangeVal = xScale.clamp(true)(xDomainValRounded)
// Offset position by 0.5 to get a pixel aligned line
line.attr('x1', xRangeVal + 0.5)
.attr('x2', xRangeVal + 0.5)
node.value = new Date(xDomainValRounded)
node.dispatchEvent(inputEvent)
})
}
testChart = {
const svg = d3.select(DOM.svg(width, height))
svg.append('g').call(yAxis)
const lineConfig = [
{
d: totalTestsSMALine,
fill: TOTAL_COLOR,
stroke: 'none',
},
{
d: posTestsSMALine,
fill: POS_COLOR,
stroke: 'none',
},
]
lineConfig.forEach(conf => {
const line = svg.append('path').datum(activeData)
Object.entries(conf).forEach(([k, v]) => line.attr(k, v))
})
// x-axis has to go after lines so it stacks on top of them
svg.append('g').call(xAxis)
.style('text-anchor', 'end')
.selectAll('text')
.attr('y', 8)
.attr('x', -6)
.attr('transform', 'rotate(-45)')
svg.call(
addInteractivity.bind(
this,
xScale,
yScale,
)
)
return svg
}
posPerYScale = d3.scaleLinear()
.domain([0, 1]).nice()
.range([height - margin.bottom, margin.top])
posPerYAxis = g => g.attr('transform', `translate(${margin.left}, 0)`)
.attr('class', 'grid')
.call(
d3.axisLeft(posPerYScale)
.ticks(10)
.tickFormat(d3.format(".0%"))
.tickSize(-width + margin.left + margin.right)
)
.call(g => g.select('.domain').remove())
posPerLine = d3.line()
.defined(d => !isNaN(getPosPerTest(d)))
.x(d => xScale(getDate(d)))
.y(d => posPerYScale(getPosPerTest(d)))
posPerSMALine = d3.line()
.defined(d => !isNaN(getPosPerTestSMA(d)))
.x(d => xScale(getDate(d)))
.y(d => posPerYScale(getPosPerTestSMA(d)))
posPerTestChart = {
const svg = d3.select(DOM.svg(width, height))
svg.append('g').call(xAxis)
.style('text-anchor', 'end')
.selectAll('text')
.attr('y', 8)
.attr('x', -6)
.attr('transform', 'rotate(-45)')
svg.append('g').call(posPerYAxis)
svg.append('path')
.datum(activeData)
.attr('d', posPerSMALine)
.attr('fill', 'none')
.attr('stroke', POS_COLOR)
.attr('stroke-width', FOREGROUND_WIDTH)
.attr('stroke-dasharray', FOREGROUND_DASH)
svg.call(
addInteractivity.bind(
this,
xScale,
yScale,
)
)
return svg
}
STATE_CHART_WIDTH = width > 600 ? '33%' : '100%'
html`<style type="text/css">
.state-charts-grid {
display: flex;
flex-direction: ${width > 600 ? 'row' : 'column'};
flex-wrap: wrap;
}
.state-charts-grid > * {
flex-basis: ${STATE_CHART_WIDTH};
margin-bottom: 15px;
}
.chart-label {
font-family: sans-serif;
font-weight: bold;
font-size: 12px;
text-transform: uppercase;
letter-spacing: .05em;
text-align: center;
}
</style>`
stateChart = (stateAbbr, width, highlight = 'total') => {
const height = Math.floor(width / 1.618)
const margin = {left: 40, right: 40, top: 10, bottom: 20}
const stateData = annotateData(
statesData.filter(d => d.state === stateAbbr),
MOVING_AVERAGE_N
)
if (stateData.length === 0) return null
const xScale = d3.scaleTime()
.domain([DATA_START_DATE, d3.max(usData, getDate)])
.range([margin.left, width - margin.right])
// Let's make a line for number of tests
const yScaleTotals = d3.scaleLinear()
.domain([0, d3.max(stateData, getTotalSMA)])
.range([height - margin.bottom, margin.top])
.nice()
const testLine = d3.line()
.defined(d => !isNaN(getTotalSMA(d)))
.x(d => xScale(getDate(d)))
.y(d => yScaleTotals(getTotalSMA(d)))
// Let's make a line for positives as a percentage of tsts
const yScalePercentage = d3.scaleLinear()
.domain([0, 1])
.range([height - margin.bottom, margin.top])
.nice()
const posPerTestLine = d3.line()
.defined(d => !isNaN(getPosPerTestSMA(d)))
.x(d => xScale(getDate(d)))
.y(d => yScalePercentage(getPosPerTestSMA(d)))
// Now generate the chart HTML and SVG
const wrapper = d3.select(html`<div class="state-chart"></div>`)
const svg = wrapper.append('svg')
.attr('width', width)
.attr('height', height)
// Add the name underneath the chart
wrapper.append('div')
.attr('class', 'chart-label')
.html(US_STATES[stateAbbr])
// Add the x-axis
svg.append('g')
.attr('transform', `translate(0, ${height - margin.bottom})`)
.call(
d3.axisBottom(xScale)
.ticks(width > 400 ? 5 : 3) // Fewer ticks for smaller screen
.tickFormat(d3.timeFormat(width > 400 ? '%b %d' : '%m/%d')) // Concise dates for smaller screen
.tickSize(5)
)
// Add y-axes
svg.append('g')
.attr('class', 'grid')
.attr('transform', `translate(${margin.left}, 0)`)
.call(
d3.axisLeft(highlight == 'total' ? yScaleTotals : yScalePercentage)
.ticks(5)
.tickFormat(highlight == 'total' ? d3.format('.2s') : d3.format('.0%'))
.tickSize(-width + margin.left + margin.right)
)
.call(g => g.select('.domain').remove())
// Add the test total data
svg.append('path')
.datum(stateData)
.attr('d', testLine)
.attr('fill', 'none')
.attr('stroke', TOTAL_COLOR)
.attr('stroke-width', highlight === 'total' ? 2 : 1)
.attr('stroke-dasharray', highlight === 'total' ? FOREGROUND_DASH : BACKGROUND_DASH)
// Add the pos per test data
svg.append('path')
.datum(stateData)
.attr('d', posPerTestLine)
.attr('fill', 'none')
.attr('stroke', POS_COLOR)
.attr('stroke-width', highlight === 'percentage' ? 2 : 1)
.attr('stroke-dasharray', highlight === 'percentage' ? FOREGROUND_DASH : BACKGROUND_DASH)
const finalPoint = stateData[stateData.length - 1]
const finalPointTotal = getTotalSMA(finalPoint)
svg.append('text')
.attr('x', width - margin.right + 5)
.attr('y', yScaleTotals(finalPointTotal))
.attr('font-family', 'sans-serif')
.attr('font-size', 12)
.attr('font-weight', 'bold')
.attr('fill', TOTAL_COLOR)
.attr('dominant-baseline', 'middle')
.attr('opacity', highlight === 'total' ? 1 : 0.25)
.text(d3.format('.2s')(finalPointTotal))
const finalPointPosPercentage = getPosPerTestSMA(finalPoint)
svg.append('text')
.attr('x', width - margin.right + 5)
.attr('y', yScalePercentage(finalPointPosPercentage))
.attr('font-family', 'sans-serif')
.attr('font-size', 12)
.attr('font-weight', 'bold')
.attr('fill', POS_COLOR)
.attr('dominant-baseline', 'middle')
.attr('opacity', highlight === 'percentage' ? 1 : 0.25)
.text(d3.format('.0%')(finalPointPosPercentage))
return wrapper.node()
}
html`
<style type="text/css">
.grid line {
stroke: #E5E5E5;
}
.grid .tick:first-of-type {
display: none;
}
</style>
`
getPosPerTest = d => {
const testCount = getTestCount(d)
if (testCount < MIN_TESTS_FOR_PERCENTAGE) return NaN // If test count is too low to return a useful %age
return getPosTestCount(d) / testCount
}
getPosPerTestSMA = d => {
const testCountSMA = getTotalSMA(d)
if (testCountSMA < MIN_TESTS_FOR_PERCENTAGE) return NaN // If test count is too low to return a useful %age
return getPosSMA(d) / testCountSMA
}
getPointByDate = (data, date) => data.filter(d => date.getTime() == getDate(d).getTime())[0]
FOREGROUND_WIDTH = width > 600 ? 4 : 1
BACKGROUND_WIDTH = width > 600 ? 1.5 : 1
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.