pixel flipper, data sifter. trying to see what I can while I'm here
air = d3
.json(
"https://www.purpleair.com/data.json?opt=1/mAQI/a10/cC0&fetch=true&nwlat=43.167493434353474&selat=32.25997597364069&nwlng=-127.87343156694496&selng=-114.15416006614362&fields=pm_1"
)
.then(d =>
Object.assign(
d.data.map(b => Object.fromEntries(d.fields.map((f, i) => [f, b[i]]))),
{ columns: d.fields }
)
)
station = d3.json(`https://www.purpleair.com/json?show=${airfile[0].ID}`)
thing = station.results[0]
pm1 = d3.json(`https://api.thingspeak.com/channels/${thing.THINGSPEAK_PRIMARY_ID}/fields/8.json?start=2020-08-10%2023:12:35&offset=0&round=2&average=60&api_key=${thing.THINGSPEAK_PRIMARY_ID_READ_KEY}`)
p03 = d3.json(`https://api.thingspeak.com/channels/${thing.THINGSPEAK_PRIMARY_ID}/fields/1.json?start=2020-08-10%2023:12:35&offset=0&round=2&average=60&api_key=${thing.THINGSPEAK_PRIMARY_ID_READ_KEY}`)
p03csv = d3.csvFormat(p03.feeds.map(d => {
return {
id: thing.ID,
thingId: thing.THINGSPEAK_PRIMARY_ID,
created_at: +new Date(d.created_at)/1000,
value: d.field1
}
}))
airfile = FileAttachment("air.json").json()
sensors = airfile.filter(d => d.Type == 0)
day = "2020-08-23"
projection = d3.geoAlbersUsa()
.fitSize([width, height], countyExtent)
projection.scale()
projection.translate()
map1 = {
const canvas = d3.create("canvas").node()
canvas.width = width
canvas.height = height
const ctx = canvas.getContext("2d")
let path = d3.geoPath(projection)
.context(ctx)
ctx.strokeStyle = "#111"
ctx.lineWidth = 1
ctx.fillStyle = "darkslategray"
ctx.beginPath()
path(california)
ctx.stroke()
ctx.fill()
ctx.globalAlpha = 0.9
ctx.fillStyle = "#999"
sensors.forEach(d => {
let p = projection([d.Lon, d.Lat])
if(p) {
ctx.fillStyle = color2(d.pm_1)
ctx.fillRect(p[0], p[1], 3,3)
}
})
// let copy = V1_08_2020_CA
// .filter(d => new Date(d.acq_date) < new Date(day3_2020) && d.bright_ti4 > 320)
// .forEach(d => {
// let p = projection([d.longitude, d.latitude])
// ctx.fillRect(p[0], p[1], 1,1)
// })
// daily_V1_08_2020_CA.get(day3_2020).forEach(d => {
// let p = projection([d.longitude, d.latitude])
// ctx.fillStyle = ti4Color(d.bright_ti4)
// ctx.fillRect(p[0], p[1], 1,1)
// })
ctx.fillStyle = "#111"
ctx.font = "bold 32px Courier New"
ctx.fillText(day, 20, 50)
return canvas
}
color = d3.scaleSequential(d3.interpolateSpectral)
// .domain(d3.extent(airfile, d => d.pm_1))
.domain([250,0])
legend({ color: color})
color2 = d3.scaleThreshold()
.domain([50, 100, 150, 200, 250])
.range(["green", "yellow", "orange", "red", "maroon"])
legend({ color: color2})
colorPM1 = d3.scaleLinear()
.domain([0,50, 100, 150, 200, 250])
.range(["green", "yellow", "orange", "red", "maroon", "maroon"])
opacityPM1 = d3.scaleLinear()
.domain([0,50, 100, 150, 200, 250])
.range([0.1, 0.5, 0.75, 0.9, 0.9, 0.9])
legend({ color: colorPM1})
d3.max(fullhexbin, d => d3.max(d.days, day => day.p03))
d3.min(fullhexbin, d => d3.min(d.days, day => day.p03))
colorP03 = d3.scaleLinear()
.domain([1,1000, 5000, 10000, 20000, 25000])
.range(["green", "yellow", "orange", "red", "maroon", "maroon"])
legend({ color: colorP03})
opacityP03 = d3.scaleLinear()
.domain([1,1000, 5000, 10000, 20000, 25000])
.range([0.1, 0.5, 0.75, 0.9, 0.9, 0.9])
import {legend} from "@d3/color-legend"
hexdata = {
const data = sensors.map(d => {
const p = projection([d.Lon, d.Lat]);
if(p) {
p.pm_1 = d.pm_1
p.ID = d.ID
p.Label = d.Label
if(p[0] < 0 || p[0] > width) return
if(p[1] < 0 || p[1] > height) return
return p
}
return p;
}).filter(d => !!d);
return Object.assign(
hexbin(data)
.map(d => (d.pm_1 = d3.median(d, d => d.pm_1), d))
.sort((a, b) => b.length - a.length),
{title: "foo"}
);
}
chart = {
const svg = d3.create("svg")
.attr("viewBox", [0, 0, width, height]);
svg.append("path")
.datum(california)
.attr("fill", "none")
.attr("stroke", "#777")
.attr("stroke-width", 0.5)
.attr("stroke-linejoin", "round")
.attr("d", d3.geoPath(projection));
svg.append("g")
.selectAll("path")
.data(hexdata)
.join("path")
.attr("transform", d => `translate(${d.x},${d.y})`)
.attr("d", d => hexbin.hexagon()) // radius(d.length)
.attr("fill", d => colorPM1(d.pm_1))
.attr("opacity", 0.7)
// .attr("stroke", d => d3.lab(color(d.date)).darker())
.append("title")
// .text(d => `${d.length.toLocaleString()} stores
// ${d.date.getFullYear()} median opening`);
return svg.node();
}
hexdata2 = hexdata.map(d => {
let a = d.map(d => Math.floor(d.pm_1))
let med = Math.floor(d3.median(a))
let i = a.indexOf(med)
let o = d[i] || d[0]
let ret = [o[0], o[1]]
ret.sensors = d
ret.x = d.x
ret.y = d.y
ret.pm_1 = o.pm_1
ret.median = med
ret.sensor = o
ret.count = d.length
return ret
})
hexdata2.map(d => d.sensor.ID)
chart2 = {
const svg = d3.create("svg")
.attr("viewBox", [0, 0, width, height]);
svg.append("path")
.datum(california)
.attr("fill", "none")
.attr("stroke", "#777")
.attr("stroke-width", 0.5)
.attr("stroke-linejoin", "round")
.attr("d", d3.geoPath(projection));
svg.append("g")
.selectAll("path")
.data(hexdata2)
.join("path")
.attr("transform", d => `translate(${d.x},${d.y})`)
.attr("d", d => hexbin.hexagon()) // radius(d.length)
.attr("fill", d => colorPM1(d.pm_1))
// .attr("stroke", d => d3.lab(color(d.date)).darker())
.append("title")
// .text(d => `${d.length.toLocaleString()} stores
// ${d.date.getFullYear()} median opening`);
return svg.node();
}
// historical = d3.csvParse(await FileAttachment("ids-out@1.csv").text(), d3.autoType)
historical = d3.csvParse(await FileAttachment("ids3-out.csv").text(), d3.autoType)
dayExtent = d3.extent(historical, d => new Date(d.created_at*1000))
// TODO: Generate the days
// have a slider over the days
// lookup the
grouped = d3.group(historical, d => d.id, d => d.type, d => d.created_at)
byhour = d3.group(historical, d => d.created_at, d => d.id, d => d.type)
byidbyhour = d3.group(historical, d => d.id, d => d.created_at, d => d.type)
Array.from(byidbyhour).filter(d => d[1].size > 260)
// time = 1597381200
new Date(time * 1000)
days =
d3.timeHour
.range(new Date(dayExtent[0]), new Date(dayExtent[1]))
.map(d => +d/1000)
viewof time = Scrubber(days, { delay: 150 , format: d => new Date(d*1000)})
// The ids that do not have a value at the current time, so we can choose something else...
hexdata2.filter(d => {
let id = d.sensor.ID
let group = grouped.get(id)
if(!group) return true
let t = group.get("pm1").get(time)
// console.log(t)
if(!t) return true
return false
}).map(d => d.sensor.ID)
hexdata2
// The ids that do not have a value at the current time, so we can choose something else...
hexdata2.map(d => {
let id = d.sensor.ID
let group = grouped.get(id)
let ci = d.sensors.indexOf(d.sensor) // current index
// we want to choose something else
let newsensor = d.sensors[ci -1] || d.sensors[ci + 1] || d.sensors[ci]
if(!group) return newsensor.ID
let t = group.get("pm1").get(time)
// console.log(t)
if(!t) return newsensor.ID
return id
})
// .map(d => d.sensor.ID)
hexbin = d3.hexbin().extent([[0, 0], [width, height]]).radius(10 * height/500)
fullhexbin = FileAttachment("ids-full-hexbins.json").json()
fullhexbinmapped = fullhexbin.map(d => {
let ret = {...d}
// let p = projection([d.lng, d.lat])
// ret.x = p[0]
// ret.y = p[1]
ret.days = new Map(ret.days.map(day => [day.day, day]))
return ret
})
days.indexOf(1597647600)
viewof time2 = Scrubber(days.slice(152), { delay: 150 , format: d => new Date(d*1000)})
viewof chart4 = {
const svg = d3.create("svg")
.attr("viewBox", [0, 0, width, height]);
let node = svg.node()
node.value = ""
svg.append("path")
.datum(california)
.attr("fill", "none")
.attr("stroke", "#777")
.attr("stroke-width", 0.5)
.attr("stroke-linejoin", "round")
.attr("d", d3.geoPath(projection));
svg.append("g")
.selectAll("path")
.data(fullhexbinmapped)
.join("path")
.attr("transform", d => `translate(${d.x},${d.y})`)
.attr("d", d => hexbin.hexagon()) // radius(d.length)
.style("fill", d => {
let t = d.days.get(time2)
if(t)
return colorPM1(t.pm1)
return "#aaa"
})
.style("opacity", d => {
let t = d.days.get(time2)
if(t)
return opacityPM1(t.pm1)
return 0.05
})
.on("mouseover", (e,d) => {
let t = d.days.get(time2)
node.value = t
node.dispatchEvent(new CustomEvent("input", { bubbles: true }));
})
// .attr("stroke", d => d3.lab(color(d.date)).darker())
.append("title")
// .text(d => `${d.length.toLocaleString()} stores
// ${d.date.getFullYear()} median opening`);
return node
}
width
// Choose a couple counties that define the extent I care about
countyExtent = ({
type: "FeatureCollection",
// features: countyShapes.features.filter(d => (d.properties.name == "Tehama" || d.properties.name == "San Benito") && d.id.slice(0,2) == "06")
features: countyShapes.features.filter(d => (d.properties.name == "Marin" || d.properties.name == "Alameda") && d.id.slice(0,2) == "06")
})
us = d3.json("https://unpkg.com/us-atlas@3/counties-10m.json")
california = statesByName.get("California")
stateShapes = topojson.feature(us, us.objects.states)//.features
statesByName = new Map(stateShapes.features.map(d => [d.properties.name, d]))
countyShapes = topojson.feature(us, us.objects.counties)//.features
import {Scrubber} from "@mbostock/scrubber"
d3 = require("d3@6.0.0-rc.3", "d3-hexbin@0.2")
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
