pixel flipper, data sifter. trying to see what I can while I'm here
// this can take a while to respond, it gives us a list of URLs where the first is the latest
bsky = d3.json(cors("https://tools.airfire.org/websky/v2/api/runs/standard/CANSAC-1.33km/?"+new Date()))
forecastUrl = bsky.run_urls[1] // for some reason the latest run often doesn't load, so we use the second most recent
// this gives us important metadata about the model and its output
metadata = d3.json(cors(`https://tools.airfire.org/websky/v2/api/run/?url=${forecastUrl}`))
region = metadata.region
hourly = metadata.overlays["100m"].hourly.GrayColorBar
hourly.root_url + "/" + hourly.images[0]
testImageUrl = hourly.root_url + "/" + hourly.images[36]
nw = mercator([region.west_lon, region.north_lat])
ne = mercator([region.east_lon, region.north_lat])
se = mercator([region.east_lon, region.south_lat])
sw = mercator([region.west_lon, region.south_lat])
testPixels = {
let img = await d3.image(cors(testImageUrl), {crossOrigin: "anonymous"})
let w = img.width
let h = img.height
let canvas = DOM.canvas(w,h)
canvas.width = w
canvas.height = h
let ctx = canvas.getContext("2d")
ctx.drawImage(img,0,0,w,h)
let imgData = ctx.getImageData(0,0,w,h).data
let pixels = []
console.log(w,h, w*h, imgData.length)
let i = 0;
for (let y = 0; y < h; y++) {
for (let x = 0; x < w; x++) {
// x + y*width is the position for point (x, y) in a flat array of pixel data
// multiply by 4 for imageData position as each pixel is 4 (rgba) values
i = (x+y*w)*4
// we only need one channel, since R,G and B should all be the same value at each pixel
if(imgData[i+3]) { // we check the alpha channel, since that will be 0 if no smoke
pixels.push({x, y, v:imgData[i]})
}
}
}
// lets keep the original image size around (though we should know this for all images ahead of time anyway...
pixels.width = w
pixels.height = h
return pixels
}
md`Let's scale our image down to the region in our current map projection, and then we can invert the projection to get back lat,lon coordinates.
`
pixScale = d3.scaleLinear()
.domain([0, testPixels.width])
.range([0, ne[0] - nw[0]])
smallWidth = ne[0] - nw[0]
sx = smallWidth / testPixels.width
sy = (se[1] - ne[1]) / testPixels.height
smallTestPixels = testPixels.map(d => {
// we use the same x and y scale since the image should be preserving its aspect ratio when scaled down
return {v:d.v, x: sx * d.x, y: sy * d.y }
})
testGeoPoints = smallTestPixels.map(d => {
// invert after translating the image to the north west corner
let lnglat = mercator.invert([d.x + nw[0],d.y + nw[1]])
return {v: d.v, lng: lnglat[0], lat: lnglat[1] }
})
values = testGeoPoints.map(d => d.v)
binner = d3.bin()
binner(values)
pmLookup = new Map([
[201, 12], // the first value is the grayscale color, the second is the PM2.5
[200, 12],
[175, 35],
[150, 55],
[125, 150],
[100, 250],
[74, 350],
[75, 350],
[24, 500],
[25, 500]
])
testGeoPM = testGeoPoints.map(d => {
let ret = {...d}
ret.v = pmLookup.get(d.v)
return ret
})
colorPM1 = d3.scaleLinear()
.domain([0,50, 100, 150, 200, 250])
.range(["green", "yellow", "orange", "red", "maroon", "maroon"])
legend({color: colorPM1})
hexdata = d3.csvParse(await FileAttachment("hexbin-2020082812.csv").text(), d3.autoType)
hexbyhour = d3.group(hexdata, d => d.img)
hexhours = Array.from(hexbyhour.keys())
mapbayhexhourcanvas = {
const canvas = d3.create("canvas").node()
canvas.width = width
canvas.height = height
const ctx = canvas.getContext("2d")
let projection = d3.geoAlbersUsa()
.fitSize([width, height], california)
let path = d3.geoPath(projection)
.context(ctx)
ctx.strokeStyle = "#111"
ctx.lineWidth = 1
ctx.beginPath()
path(westCoast)
ctx.stroke()
// testGeoPoints.forEach(d => {
hexbyhour.get(hexhour).forEach(d => {
// ctx.fillStyle = `rgb(${[d.v,d.v,d.v]})`
ctx.fillStyle = colorPM1(d.v)
let p = projection([d.lng, d.lat])
ctx.fillRect(p[0], p[1], 1,1)
})
return canvas
}
testdata = d3.csvParse(await FileAttachment("100m_hourly_202008281200.csv").text(), d3.autoType)
binner(testdata.map(d => d.v))
mapbaytestcanvas = {
const canvas = d3.create("canvas").node()
canvas.width = width
canvas.height = height
const ctx = canvas.getContext("2d")
let projection = d3.geoAlbersUsa()
.fitSize([width, height], california)
// let path = d3.geoPath(projection)
let path = d3.geoPath(mercator)
.context(ctx)
ctx.strokeStyle = "#111"
ctx.lineWidth = 1
ctx.beginPath()
path(westCoast)
ctx.stroke()
let sx = (ne[0] - nw[0]) / testPixels.width
let sy = (sw[1] - ne[1]) / testPixels.height
// testGeoPoints.forEach(d => {
testdata.forEach(d => {
// ctx.fillStyle = `rgb(${[d.v,d.v,d.v]})`
ctx.fillStyle = colorPM1(pmLookup.get(d.v))
// let p = projection([d.lng, d.lat])
let lnglat = mercator.invert([d.x * sx + nw[0], d.y * sy + nw[1]])
// let p = projection([lnglat[0], lnglat[1]])
let p = mercator([lnglat[0], lnglat[1]])
if(p)
ctx.fillRect(p[0], p[1], 1, 1)
// ctx.fillRect(d.x, d.y, 1,1)
})
return canvas
}
sxx = (ne[0] - nw[0]) / width
syy = (sw[1] - ne[1]) / height
hexbin2 ={
let sx = (ne[1] - nw[1]) / width
let sy = (se[1] - ne[1]) / height
let hexbin2 = d3.hexbin()
.extent([[0, 0], [width, height]])
.radius(1)
.x(d => d.x)
.y(d => d.y)
return hexbin2
}
hexdata2 = hexbin2(testdata).map(d => ({x:d.x, y: d.y, v: d3.median(d, a => a.v)}))
round = function(n) {
let prec = 1000
return Math.floor(n * prec) / prec
}
mapbaytest2canvas = {
const canvas = d3.create("canvas").node()
canvas.width = width
canvas.height = height
const ctx = canvas.getContext("2d")
let projection = d3.geoAlbersUsa()
.fitSize([width, height], california)
let path = d3.geoPath(projection)
.context(ctx)
ctx.strokeStyle = "#111"
ctx.lineWidth = 1
ctx.beginPath()
path(westCoast)
ctx.stroke()
let sx = (ne[0] - nw[0]) / testPixels.width
let sy = (sw[1] - ne[1]) / testPixels.height
// testGeoPoints.forEach(d => {
hexdata2.forEach(d => {
// ctx.fillStyle = `rgb(${[d.v,d.v,d.v]})`
ctx.fillStyle = colorPM1(d.v)
// let p = projection([d.lng, d.lat])
let lnglat = mercator.invert([d.x * sx + nw[0], d.y * sy + nw[1]])
// let p = projection([lnglat[0], lnglat[1]])
let a = projection([round(lnglat[0]), round(lnglat[1])])
let p = alb([round(lnglat[0]), round(lnglat[1])])
// if(p)
if(p[0] > 0 && p[0] < width && p[1] > 0 && p[1] < height)
ctx.fillRect(a[0], a[1], 5, 5)
// ctx.fillRect(d.x, d.y, 1,1)
})
return canvas
}
// files = zip(await FileAttachment("hexbin-2020082812.csv@2.zip").arrayBuffer(), {
// base64: false,
// checkCRC32: true
// }).files
// hexdata3 = d3.csvParse(files["data/hexbin-2020082812.csv"].asText(), d3.autoType)
hexdata3 = d3.csvParse(await FileAttachment("hexbin-2020083000@2.csv").text(), d3.autoType)
hexbyhour3 = d3.group(inside3, d => d.i)
inside3 = hexdata3.filter(d => {
let p = alb([d.lng, d.lat])
return p[0] > 0 && p[0] < width && p[1] > 0 && p[1] < height
})
alb = d3.geoAlbersUsa()
.fitSize([width, height], countyExtent)
viewof hexhour3 = Scrubber(hexbyhour3.keys(), { delay: 200, format: d => d })
mapbayhex2canvas = {
const canvas = d3.create("canvas").node()
canvas.width = width
canvas.height = height
const ctx = canvas.getContext("2d")
let projection = alb
// let projection = d3.geoAlbersUsa()
// .fitSize([width, height], california)
let path = d3.geoPath(projection)
.context(ctx)
ctx.strokeStyle = "#111"
ctx.lineWidth = 1
ctx.beginPath()
path(westCoast)
ctx.stroke()
hexbyhour3.get(hexhour3).forEach(d => {
ctx.globalAlpha = 0.5
ctx.fillStyle = colorPM1(d.pm1)
let p = projection([d.lng, d.lat])
// if(p[0] > 0 && p[0] < width && p[1] > 0 && p[1] < height)
ctx.fillRect(p[0], p[1], 5, 5)
})
return canvas
}
mapbayhex3canvas = {
const canvas = d3.create("canvas").node()
canvas.width = width
canvas.height = height
const ctx = canvas.getContext("2d")
let projection = alb
let path = d3.geoPath(projection)
.context(ctx)
ctx.strokeStyle = "#111"
ctx.lineWidth = 1
ctx.beginPath()
path(westCoast)
ctx.stroke()
Array.from(hexbyhour3.keys()).slice(0,10).forEach(i => {
ctx.globalAlpha = 0.1
ctx.fillStyle = "white"
ctx.fillRect(0,0,width,height)
hexbyhour3.get(i).forEach(d => {
ctx.globalAlpha = 0.5
ctx.fillStyle = colorPM1(d.pm1)
let p = projection([d.lng, d.lat])
// if(p[0] > 0 && p[0] < width && p[1] > 0 && p[1] < height)
ctx.fillRect(p[0] + i*5, p[1] + i*5, 5, 5)
})
})
return canvas
}
alb.scale()
alb.translate()
import {fullhexbin} from "@enjalot/purpleair-historical-data"
mercator = d3.geoMercator()
.fitSize([width, height], california)
// .fitSize([width, height], [[region.west_lon, region.north_lat], [region.east_lon, region.south_lat]])
mercator.scale()
mercator.translate()
california = statesByName.get("California")
westCoast = ({type: "FeatureCollection", features: [
statesByName.get("California"),
statesByName.get("Oregon"),
statesByName.get("Arizona"),
statesByName.get("Nevada")
]})
// Choose a couple counties that define the extent I care about
countyExtent = ({
type: "FeatureCollection",
features: countyShapes.features.filter(d => (
d.properties.name == "Marin" ||
d.properties.name == "Santa Clara") && d.id.slice(0,2) == "06")
})
countyShapes = topojson.feature(us, us.objects.counties)//.features
stateShapes = topojson.feature(us, us.objects.states)//.features
statesByName = new Map(stateShapes.features.map(d => [d.properties.name, d]))
us = d3.json("https://unpkg.com/us-atlas@3/counties-10m.json")
height = 500
function cors(url) {
return "https://cors-anywhere.herokuapp.com/" + url
}
import {Scrubber} from "@mbostock/scrubber"
turf = require('@turf/turf')
topojson = require("topojson-client@3")
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
