The end-to-end solution for building and hosting better data apps, dashboards, and reports.
Public
Edited
Jan 20, 2023
26 forks
Importers
39 stars
cityMap
// Fetch the data files for each of the above selected sensors
// The related functions for fetching and processing are detailed further down in the notebook
cityCollection = Promise.all(
citySelected.map((d) => fetchISDLite(d, year))
).then((results) =>
results.flatMap((d, i) => processISDLite(d, citySelected[i]))
)
stationRollup = d3
.rollups(
cityCollection,
(v) => v.length,
(d) => d.station
)
.map((d) => ({
station: cityMap.sensors.find((s) => s.id == d[0]),
id: d[0],
datapoints: d[1]
}))
viewof isds = Inputs.table(isdUSCurrent, {
multiple: false,
columns: ["USAF", "WBAN", "name", "state", "begin", "end"]
})
isdHistory = d3.csv("https://www1.ncdc.noaa.gov/pub/data/noaa/isd-history.csv")
isdStations = isdHistory.map((d) => {
return {
id: d.USAF + d.WBAN,
USAF: d.USAF,
WBAN: d.WBAN,
name: d["STATION NAME"],
country: d.CTRY,
state: d.STATE,
lat: +d.LAT,
lon: +d.LON,
elevation: +d["ELEV(M)"],
begin: parseNumDate(d.BEGIN),
end: parseNumDate(d.END)
};
})
isdUS = isdStations.filter((d) => d.country == "US" && d.state)
isdUSCurrent = isdUS.filter((d) => {
// d.end >= d3.timeMonth(new Date())
return (
d.begin.getFullYear() <= year &&
d.end.getFullYear() >= year &&
d.WBAN !== "99999"
);
})
sensorpoints = isdUSCurrent.map((d) => projection([d.lon, d.lat]) || [-1, -1])
sensortree = d3
.quadtree()
.x((d) => d[0])
.y((d) => d[1])
.addAll(sensorpoints)
isdliteHeader = [
"year",
"month",
"day",
"hour",
"air_temp",
"dew_temp",
"sea_level_pressure",
"wind_direction",
"wind_speed_rate",
"sky_condition",
"precip_1hour",
"precip_6hour"
]
processISDLite = (isd, station) => {
return d3.tsvParse(isd, d3.autoType).map((d) => {
return {
station: station.id,
// The data is stored in UTC
date: parseDate(`${d.year}-${d.month}-${d.day} ${d.hour}`),
// we set null values to null, adjust by scaling
air_temp: d.air_temp == -9999 ? null : d.air_temp / 10,
dew_temp: d.dew_temp == -9999 ? null : d.dew_temp / 10,
sea_level_pressure:
d.sea_level_pressure == -9999 ? null : d.sea_level_pressure / 10,
wind_direction: d.wind_direction == -9999 ? null : d.wind_direction,
wind_speed_rate:
d.wind_speed_rate == -9999 ? null : d.wind_speed_rate / 10,
sky_condition: d.sky_condition == -9999 ? null : d.sky_condition,
precip_1hour: d.precip_1hour == -9999 ? null : d.precip_1hour / 10,
precip_6hour: d.precip_6hour == -9999 ? null : d.precip_6hour / 10
};
});
}
parseDate = d3.utcParse("%Y-%-m-%-d %H")
isdurl = (d, year) =>
`https://www.ncei.noaa.gov/pub/data/noaa/isd-lite/${year}/${d.USAF}-${d.WBAN}-${year}.gz`
fetchISDLite = (station, year = 2021) => {
return fetch(isdurl(station, year)).then((d) => {
return d.arrayBuffer().then((u) => {
let head = isdliteHeader.join("\t") + "\n";
let body;
try {
body = fflate
.strFromU8(fflate.decompressSync(new Uint8Array(u)))
.replace(/[ ]+/g, "\t");
} catch (e) {
let str = fflate.strFromU8(new Uint8Array(u));
if (str.indexOf("404 Not Found")) return "";
// TODO: throw error?
}
return head + body;
});
});
}
Inputs.table(usCitiesMinPop)
populatedPlaces = FileAttachment("ne_10m_populated_places.csv").csv()
usCitiesPopulated = populatedPlaces.filter((d) => d["ADM0NAME"] == "United States of America")
// simplify the fields
usCities = usCitiesPopulated.map((d) => ({
name: d.NAME,
state: d.ADM1NAME,
population: +d.POP_MAX,
lat: +d.LATITUDE,
lon: +d.LONGITUDE
}))
minimumPopulation = 70000
// turns out Wyoming doesn't have that many heavily populated cities.
usCities.filter((d) => d.state == "Wyoming")
usCitiesMinPop = usCities.filter((d) => d.population > minimumPopulation)
// quick way to see if we have at least one city in each state / territory
d3.group(usCitiesMinPop, (d) => d.state)
findCity = {
const quadtree = d3
.quadtree()
.x((d) => projection([d.lon, d.lat])[0])
.y((d) => projection([d.lon, d.lat])[1])
.addAll(filteredCities);
return (p) => quadtree.find(...p);
}
function findSensors(city) {
const [x, y] = projection([city.lon, city.lat]);
const sensorps = findInCircle(sensortree, x, y, radius);
const sensors = sensorps.map((p) => isdUSCurrent[sensorpoints.indexOf(p)]);
return { city, sensors };
}
defaultHighlight = findSensors(filteredCities.find((d) => d.name === "New York"))
stateShapes = topojson.feature(us, us.objects.states)
us = d3.json("https://unpkg.com/us-atlas@3/counties-10m.json")
projection = d3.geoAlbersUsa().scale(1237).translate([464, 290]) // fit to 928x581
radius = 10
parseNumDate = d3.timeParse("%Y%m%d")
intFormat = d3.format(",d")
// A function that quickly returns the array of elements in the quadtree
// that are below a certain distance of a given point
// https://observablehq.com/@d3/quadtree-findincircle
function findInCircle(quadtree, x, y, radius, filter) {
const result = [],
radius2 = radius * radius,
accept = filter
? d => filter(d) && result.push(d)
: d => result.push(d);
quadtree.visit(function(node, x1, y1, x2, y2) {
if (node.length) {
return x1 >= x + radius || y1 >= y + radius || x2 < x - radius || y2 < y - radius;
}
const dx = +quadtree._x.call(null, node.data) - x,
dy = +quadtree._y.call(null, node.data) - y;
if (dx * dx + dy * dy < radius2) {
do { accept(node.data); } while (node = node.next);
}
});
return result;
}
// a handy way to create baseline "rules" in the preview plot
marksByMetric = ({
"air_temp": setRuleMarks(0,-40,40),
"dew_temp": setRuleMarks(0,-40,40),
"sea_level_pressure": setRuleMarks(1013,980,1030), // standard at sea level, cat 1 hurricane, strong high pressure system
// https://www.theweatherprediction.com/habyhints2/410/
"wind_direction": setRuleMarks(0,0,360),
"wind_speed_rate": setRuleMarks(0,10,20),
"sky_condition": setRuleMarks(0,0,9),
"precip_1hour": setRuleMarks(0,5,10),
"precip_6hour": setRuleMarks(0,5,10),
})
// convenience function for creating the marks in the preview plot
setRuleMarks = function(baseline, min, max){
return [
Plot.ruleY([baseline], { stroke: "lightgray", strokeDasharray: "4,2"}),
Plot.ruleY([min], { stroke: "lightgray" }),
Plot.ruleY([max], { stroke: "lightgray" })
]
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
