Personal Profile.
Head of AI Skunkworks @NHSx, formerly Head of Data @departmentfortransport, formerly known as "The Open Data Rottweiler". Always a techie.
// Fetch the data files for each of the above selected sensors
// The related functions for fetching and processing are detailed further down in the notebook
cityCollection = Promise.all(
citySelected.map((d) => fetchISDLite(d, year))
).then((results) =>
results.flatMap((d, i) => processISDLite(d, citySelected[i]))
)
viewof isds = Inputs.table(isdUSCurrent, {
multiple: false,
columns: ["USAF", "WBAN", "name", "state", "begin", "end"]
})
isdHistory = d3.csv("https://www1.ncdc.noaa.gov/pub/data/noaa/isd-history.csv")
isdStations = isdHistory.map((d) => {
return {
id: d.USAF + d.WBAN,
USAF: d.USAF,
WBAN: d.WBAN,
name: d["STATION NAME"],
country: d.CTRY,
state: d.STATE,
lat: +d.LAT,
lon: +d.LON,
elevation: +d["ELEV(M)"],
begin: parseNumDate(d.BEGIN),
end: parseNumDate(d.END)
};
})
// isdUS = isdStations.filter((d) => d.country == "US" && d.state)
isdUS = isdStations.filter((d) => d.country == "UK")
isdUSCurrent = isdUS.filter((d) => {
// d.end >= d3.timeMonth(new Date())
// return (
// d.begin.getFullYear() <= year &&
// d.end.getFullYear() >= year &&
// d.WBAN !== "99999" // this worked for the US
// );
return (
d.begin.getFullYear() <= year &&
d.end.getFullYear() >= year
);
})
sensorpoints = isdUSCurrent.map((d) => projection([d.lon, d.lat]) || [-1, -1])
sensortree = d3
.quadtree()
.x((d) => d[0])
.y((d) => d[1])
.addAll(sensorpoints)
isdliteHeader = [
"year",
"month",
"day",
"hour",
"air_temp",
"dew_temp",
"sea_level_pressure",
"wind_direction",
"wind_speed_rate",
"sky_condition",
"precip_1hour",
"precip_6hour"
]
processISDLite = (isd, station) => {
return d3.tsvParse(isd, d3.autoType).map((d) => {
return {
station: station.id,
// The data is stored in UTC
date: parseDate(`${d.year}-${d.month}-${d.day} ${d.hour}`),
// we set null values to null, adjust by scaling
air_temp: d.air_temp == -9999 ? null : d.air_temp / 10,
dew_temp: d.dew_temp == -9999 ? null : d.dew_temp / 10,
sea_level_pressure:
d.sea_level_pressure == -9999 ? null : d.sea_level_pressure / 10,
wind_direction: d.wind_direction == -9999 ? null : d.wind_direction,
wind_speed_rate:
d.wind_speed_rate == -9999 ? null : d.wind_speed_rate / 10,
sky_condition: d.sky_condition == -9999 ? null : d.sky_condition,
precip_1hour: d.precip_1hour == -9999 ? null : d.precip_1hour / 10,
precip_6hour: d.precip_6hour == -9999 ? null : d.precip_6hour / 10
};
});
}
parseDate = d3.utcParse("%Y-%-m-%-d %H")
isdurl = (d, year) =>
`https://www.ncei.noaa.gov/pub/data/noaa/isd-lite/${year}/${d.USAF}-${d.WBAN}-${year}.gz`
fetchISDLite = (station, year = 2021) => {
return fetch(isdurl(station, year)).then((d) => {
return d.arrayBuffer().then((u) => {
let head = isdliteHeader.join("\t") + "\n";
let body;
try {
body = fflate
.strFromU8(fflate.decompressSync(new Uint8Array(u)))
.replace(/[ ]+/g, "\t");
} catch (e) {
let str = fflate.strFromU8(new Uint8Array(u));
if (str.indexOf("404 Not Found")) return "";
// TODO: throw error?
}
return head + body;
});
});
}
Inputs.table(usCitiesMinPop)
populatedPlaces = FileAttachment("ne_10m_populated_places.csv").csv()
//usCitiesPopulated = populatedPlaces.filter(
// (d) => d["ADM0NAME"] == "United States of America"
//)
usCitiesPopulated = populatedPlaces.filter(
(d) => d["ADM0NAME"] == "United Kingdom"
)
usCities = usCitiesPopulated.map((d) => {
// let's simplify the fields
return {
name: d.NAME,
state: d.ADM1NAME,
population: +d.POP_MAX,
lat: +d.LATITUDE,
lon: +d.LONGITUDE
};
})
minimumPopulation = 20000
// turns out Wyoming doesn't have that many heavily populated cities.
// usCities.filter((d) => d.state == "Wyoming")
usCitiesMinPop = usCities.filter((d) => d.population > minimumPopulation)
// quick way to see if we have at least one city in each state / territory
// d3.group(usCitiesMinPop, (d) => d.state)
// citypoints = usCitiesMinPop.map((d) => projection([d.lon, d.lat]))
citypoints = filteredCities.map((d) => projection([d.lon, d.lat]))
filteredCities.map((d) => projection([d.lon, d.lat]))
citytree = d3
.quadtree()
.x((d) => d[0])
.y((d) => d[1])
.addAll(citypoints)
stateShapes = topojson.feature(us, us.objects.states)
localShapes = topojson.feature(gb, gb.objects.eer)
us = d3.json("https://unpkg.com/us-atlas@3/counties-10m.json")
gb = d3.json("https://martinjc.github.io/UK-GeoJSON/json/eng/topo_eer.json")
//gb = d3.json("https://raw.githubusercontent.com/martinjc/UK-GeoJSON/master/json/administrative/gb/lad.json")
// projection = d3.geoAlbersUsa().fitSize([mapWidth - 10, mapHeight], stateShapes)
// projection = d3.geoAlbersUsa().fitSize([width - 10, mapHeight * widthRatio], stateShapes)
// projection = d3.geoMercator().fitSize([width-10,mapHeight * widthRatio], stateShapes);
// projection = d3.geoMercator();
projection = d3.geoMercator().fitSize([width-10,mapHeight * widthRatio], localShapes);
mapWidth = 960
mapHeight = 500
widthRatio = width / mapWidth
parseNumDate = d3.timeParse("%Y%m%d")
intFormat = d3.format(",d")
// https://observablehq.com/@d3/quadtree-findincircle
function findInCircle(quadtree, x, y, radius, filter) {
const result = [],
radius2 = radius * radius,
accept = filter
? d => filter(d) && result.push(d)
: d => result.push(d);
quadtree.visit(function(node, x1, y1, x2, y2) {
if (node.length) {
return x1 >= x + radius || y1 >= y + radius || x2 < x - radius || y2 < y - radius;
}
const dx = +quadtree._x.call(null, node.data) - x,
dy = +quadtree._y.call(null, node.data) - y;
if (dx * dx + dy * dy < radius2) {
do { accept(node.data); } while (node = node.next);
}
});
return result;
}
// a handy way to create baseline "rules" in the preview plot
marksByMetric = ({
"air_temp": setRuleMarks(0,-40,40),
"dew_temp": setRuleMarks(0,-40,40),
"sea_level_pressure": setRuleMarks(1013,980,1030), // standard at sea level, cat 1 hurricane, strong high pressure system
// https://www.theweatherprediction.com/habyhints2/410/
"wind_direction": setRuleMarks(0,0,360),
"wind_speed_rate": setRuleMarks(0,10,20),
"sky_condition": setRuleMarks(0,0,9),
"precip_1hour": setRuleMarks(0,5,10),
"precip_6hour": setRuleMarks(0,5,10),
})
// convenience function for creating the marks in the preview plot
setRuleMarks = function(baseline, min, max){
return [
Plot.ruleY([baseline], { stroke: "lightgray", strokeDasharray: "4,2"}),
Plot.ruleY([min], { stroke: "lightgray" }),
Plot.ruleY([max], { stroke: "lightgray" })
]
}
// https://observablehq.com/@fil/hello-fflate
fflate = import("https://cdn.skypack.dev/fflate@0.7.1")
import { button as downloadButton } from "@jeremiak/download-data-button"
import {addTooltips} from "@mkfreeman/plot-tooltip"
import {authorship, readMore, navigation, workshop} from "@observablehq/timeseries-assets"
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
