Principal Web Developer at Freedom of the Press Foundation
Published
Edited
Aug 17, 2021
12 stars
parseDate = d3.timeParse("%Y-%m-%d")
dateToKey = d3.timeFormat("%Y-%m-%d")
formatDateVerbose = d3.timeFormat("%B %d, %Y")
dataFiles = {
const domparser = new DOMParser()
const rawHtml = await (await fetch(CORS_PROXY_PREFIX + 'https://www.jefftk.com/apartment_prices/data-listing')).text()
const htmlDoc = domparser.parseFromString(rawHtml, 'text/html')
return Array.from(htmlDoc.querySelectorAll('a')).map(a => ({
date: parseDate(a.text),
url: 'https://www.jefftk.com' + a.getAttribute('href')
}))
}
RentFileParser = d3.dsvFormat(' ')
parseRentFile = (text, date) => RentFileParser.parseRows(
text,
d => ({
price: +d[0],
bedrooms: +d[1],
aptId: +d[2],
lon: +d[3],
lat: +d[4],
date: date,
})
)
dataSets = Promise.all(selectedDates.filter(d => d).map(async dateKey => {
const {date, url} = dataFiles.find(d => dateToKey(d.date) === dateKey)
const rawData = await (await fetch(CORS_PROXY_PREFIX + url)).text()
return parseRentFile(rawData, date)
}))
annotateAndFilter = data => data.filter(d => d.bedrooms === +brCountFilter)
.sort((a, b) => a.price - b.price)
.map((d, i, arr) => ({
...d,
percentile: (i + 1)/arr.length
}))
joinedData = dataSets.reduce((acc, data) => acc.concat(annotateAndFilter(data)), [])
margin = ({ top: 20, bottom: 80, left: 40, right: 20 })
chartWidth = width
chartHeight = width > 600 ? chartWidth / 1.6 : chartWidth * 1.6
xScaleRange = {
const min = quantileRange[0] < 0.05 ? 0 : quantileRange[0]
const max = quantileRange[1] > 0.95 ? 1 : quantileRange[1]
return [min, max]
}
xScale = d3.scaleLinear()
.domain(xScaleRange)
.range([margin.left, chartWidth - margin.right])
yScale = d3.scaleLinear()
.domain([
d3.min(joinedData.filter(d => d.percentile > quantileRange[0]).map(d => d.price)),
d3.max(joinedData.filter(d => d.percentile < quantileRange[1]).map(d => d.price)),
])
.range([chartHeight - margin.bottom, margin.top])
xAxis = d3.axisBottom(xScale)
.tickFormat(d3.format('.0%'))
.tickSize(-chartHeight - margin.top - margin.bottom)
yAxis = d3.axisLeft(yScale)
.tickFormat(d => '$' + d3.format('~s')(d))
.tickSize(-chartWidth - margin.left - margin.right)
colorScaleNonCircular = (date, dates) => d3.schemeSet2[dates.indexOf(typeof date === 'string' ? date : dateToKey(date))]
colorScale = date => colorScaleNonCircular(date, selectedDates)
// Filter out data that's above or below our scale range
// and then render it as data points
chartDataPoints = (data, xScale, yScale, colorScale) => data.filter(
d => d.price > yScale.domain()[0] && d.price < yScale.domain()[1] &&
d.percentile > xScale.domain()[0] && d.percentile < xScale.domain()[1]
).map(d => svg.fragment`
<circle
cx=${xScale(d.percentile)}
cy=${yScale(d.price)}
r="2"
fill=${colorScale(d.date)}
/>
`)
legend = () => svg.fragment`<g transform="translate(${margin.left + 5} ${margin.top - 5})">
<rect x="0" y="0" width="150" height=${10 + 20 * selectedDates.filter(d => d).length} fill="#FFF" stroke="#CCC" />
${selectedDates.filter(d => d).map((dateKey, i) => svg.fragment`
<circle cx="15" cy=${15 + 20 * i} r="5" fill=${colorScale(dateKey)} />
<text
x="25"
y=${19 + 20 * i}
text-anchor="start"
font-size="12"
font-family="var(--sans-serif)"
>${formatDateVerbose(parseDate(dateKey))}</text>
`)}
</g>`
closest = (
data,
accessor,
value,
cmp = (a, b) => Math.abs(a - b)
) => {
const idx = d3.bisectLeft(data.map(accessor), value)
if (idx === data.length - 1) return data[idx]
return cmp(accessor(data[idx]), value) < cmp(accessor(data[idx + 1]), value) ? data[idx] : data[idx + 1]
}
CORS_PROXY_PREFIX = 'https://corsproxy.harrislapiroff.com/'
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
