The end-to-end solution for building and hosting better data apps, dashboards, and reports.
Public
Edited
Jan 20, 2023
26 forks
Importers
39 stars
cityMap
// Fetch the data files for each of the above selected sensors
// The related functions for fetching and processing are detailed further down in the notebook
cityCollection = Promise.all(
citySelected.map((d) => fetchISDLite(d, year))
).then((results) =>
results.flatMap((d, i) => processISDLite(d, citySelected[i]))
)
stationRollup = d3
.rollups(
cityCollection,
(v) => v.length,
(d) => d.station
)
.map((d) => ({
station: cityMap.sensors.find((s) => s.id == d[0]),
id: d[0],
datapoints: d[1]
}))
viewof isds = Inputs.table(isdUSCurrent, {
multiple: false,
columns: ["USAF", "WBAN", "name", "state", "begin", "end"]
})
isdHistory = d3.csv("https://www1.ncdc.noaa.gov/pub/data/noaa/isd-history.csv")
isdStations = isdHistory.map((d) => {
return {
id: d.USAF + d.WBAN,
USAF: d.USAF,
WBAN: d.WBAN,
name: d["STATION NAME"],
country: d.CTRY,
state: d.STATE,
lat: +d.LAT,
lon: +d.LON,
elevation: +d["ELEV(M)"],
begin: parseNumDate(d.BEGIN),
end: parseNumDate(d.END)
};
})
isdUS = isdStations.filter((d) => d.country == "US" && d.state)
isdUSCurrent = isdUS.filter((d) => {
// d.end >= d3.timeMonth(new Date())
return (
d.begin.getFullYear() <= year &&
d.end.getFullYear() >= year &&
d.WBAN !== "99999"
);
})
sensorpoints = isdUSCurrent.map((d) => projection([d.lon, d.lat]) || [-1, -1])
sensortree = d3
.quadtree()
.x((d) => d[0])
.y((d) => d[1])
.addAll(sensorpoints)
isdliteHeader = [
"year",
"month",
"day",
"hour",
"air_temp",
"dew_temp",
"sea_level_pressure",
"wind_direction",
"wind_speed_rate",
"sky_condition",
"precip_1hour",
"precip_6hour"
]
processISDLite = (isd, station) => {
return d3.tsvParse(isd, d3.autoType).map((d) => {
return {
station: station.id,
// The data is stored in UTC
date: parseDate(`${d.year}-${d.month}-${d.day} ${d.hour}`),
// we set null values to null, adjust by scaling
air_temp: d.air_temp == -9999 ? null : d.air_temp / 10,
dew_temp: d.dew_temp == -9999 ? null : d.dew_temp / 10,
sea_level_pressure:
d.sea_level_pressure == -9999 ? null : d.sea_level_pressure / 10,
wind_direction: d.wind_direction == -9999 ? null : d.wind_direction,
wind_speed_rate:
d.wind_speed_rate == -9999 ? null : d.wind_speed_rate / 10,
sky_condition: d.sky_condition == -9999 ? null : d.sky_condition,
precip_1hour: d.precip_1hour == -9999 ? null : d.precip_1hour / 10,
precip_6hour: d.precip_6hour == -9999 ? null : d.precip_6hour / 10
};
});
}
parseDate = d3.utcParse("%Y-%-m-%-d %H")
isdurl = (d, year) =>
`https://www.ncei.noaa.gov/pub/data/noaa/isd-lite/${year}/${d.USAF}-${d.WBAN}-${year}.gz`
fetchISDLite = (station, year = 2021) => {
return fetch(isdurl(station, year)).then((d) => {
return d.arrayBuffer().then((u) => {
let head = isdliteHeader.join("\t") + "\n";
let body;
try {
body = fflate
.strFromU8(fflate.decompressSync(new Uint8Array(u)))
.replace(/[ ]+/g, "\t");
} catch (e) {
let str = fflate.strFromU8(new Uint8Array(u));
if (str.indexOf("404 Not Found")) return "";
// TODO: throw error?
}
return head + body;
});
});
}
Inputs.table(usCitiesMinPop)
populatedPlaces = FileAttachment("ne_10m_populated_places.csv").csv()
usCitiesPopulated = populatedPlaces.filter((d) => d["ADM0NAME"] == "United States of America")
// simplify the fields
usCities = usCitiesPopulated.map((d) => ({
name: d.NAME,
state: d.ADM1NAME,
population: +d.POP_MAX,
lat: +d.LATITUDE,
lon: +d.LONGITUDE
}))
minimumPopulation = 70000
// turns out Wyoming doesn't have that many heavily populated cities.
usCities.filter((d) => d.state == "Wyoming")
usCitiesMinPop = usCities.filter((d) => d.population > minimumPopulation)
// quick way to see if we have at least one city in each state / territory
d3.group(usCitiesMinPop, (d) => d.state)
findCity = {
const quadtree = d3
.quadtree()
.x((d) => projection([d.lon, d.lat])[0])
.y((d) => projection([d.lon, d.lat])[1])
.addAll(filteredCities);
return (p) => quadtree.find(...p);
}
function findSensors(city) {
const [x, y] = projection([city.lon, city.lat]);
const sensorps = findInCircle(sensortree, x, y, radius);
const sensors = sensorps.map((p) => isdUSCurrent[sensorpoints.indexOf(p)]);
return { city, sensors };
}
defaultHighlight = findSensors(filteredCities.find((d) => d.name === "New York"))
stateShapes = topojson.feature(us, us.objects.states)
us = d3.json("https://unpkg.com/us-atlas@3/counties-10m.json")
projection = d3.geoAlbersUsa().scale(1237).translate([464, 290]) // fit to 928x581
radius = 10
parseNumDate = d3.timeParse("%Y%m%d")
intFormat = d3.format(",d")
// A function that quickly returns the array of elements in the quadtree
// that are below a certain distance of a given point
// https://observablehq.com/@d3/quadtree-findincircle
function findInCircle(quadtree, x, y, radius, filter) {
const result = [],
radius2 = radius * radius,
accept = filter
? d => filter(d) && result.push(d)
: d => result.push(d);
quadtree.visit(function(node, x1, y1, x2, y2) {
if (node.length) {
return x1 >= x + radius || y1 >= y + radius || x2 < x - radius || y2 < y - radius;
}
const dx = +quadtree._x.call(null, node.data) - x,
dy = +quadtree._y.call(null, node.data) - y;
if (dx * dx + dy * dy < radius2) {
do { accept(node.data); } while (node = node.next);
}
});
return result;
}
// a handy way to create baseline "rules" in the preview plot
marksByMetric = ({
"air_temp": setRuleMarks(0,-40,40),
"dew_temp": setRuleMarks(0,-40,40),
"sea_level_pressure": setRuleMarks(1013,980,1030), // standard at sea level, cat 1 hurricane, strong high pressure system
// https://www.theweatherprediction.com/habyhints2/410/
"wind_direction": setRuleMarks(0,0,360),
"wind_speed_rate": setRuleMarks(0,10,20),
"sky_condition": setRuleMarks(0,0,9),
"precip_1hour": setRuleMarks(0,5,10),
"precip_6hour": setRuleMarks(0,5,10),
})
// convenience function for creating the marks in the preview plot
setRuleMarks = function(baseline, min, max){
return [
Plot.ruleY([baseline], { stroke: "lightgray", strokeDasharray: "4,2"}),
Plot.ruleY([min], { stroke: "lightgray" }),
Plot.ruleY([max], { stroke: "lightgray" })
]
}
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.