pixel flipper, data sifter. trying to see what I can while I'm here
// this can take a while to respond, it gives us a list of URLs where the first is the latest
bsky = d3.json(cors("https://tools.airfire.org/websky/v2/api/runs/standard/CANSAC-1.33km/?"+new Date()))
forecastUrl = bsky.run_urls[1] // for some reason the latest run often doesn't load, so we use the second most recent
// this gives us important metadata about the model and its output
metadata = d3.json(cors(`https://tools.airfire.org/websky/v2/api/run/?url=${forecastUrl}`))
region = metadata.region
hourly = metadata.overlays["100m"].hourly.GrayColorBar
hourly.root_url + "/" + hourly.images[0]
testImageUrl = hourly.root_url + "/" + hourly.images[36]
nw = mercator([region.west_lon, region.north_lat])
ne = mercator([region.east_lon, region.north_lat])
se = mercator([region.east_lon, region.south_lat])
sw = mercator([region.west_lon, region.south_lat])
testPixels = {
let img = await d3.image(cors(testImageUrl), {crossOrigin: "anonymous"})
let w = img.width
let h = img.height
let canvas = DOM.canvas(w,h)
canvas.width = w
canvas.height = h
let ctx = canvas.getContext("2d")
ctx.drawImage(img,0,0,w,h)
let imgData = ctx.getImageData(0,0,w,h).data
let pixels = []
console.log(w,h, w*h, imgData.length)
let i = 0;
for (let y = 0; y < h; y++) {
for (let x = 0; x < w; x++) {
// x + y*width is the position for point (x, y) in a flat array of pixel data
// multiply by 4 for imageData position as each pixel is 4 (rgba) values
i = (x+y*w)*4
// we only need one channel, since R,G and B should all be the same value at each pixel
if(imgData[i+3]) { // we check the alpha channel, since that will be 0 if no smoke
pixels.push({x, y, v:imgData[i]})
}
}
}
// lets keep the original image size around (though we should know this for all images ahead of time anyway...
pixels.width = w
pixels.height = h
return pixels
}
md`Let's scale our image down to the region in our current map projection, and then we can invert the projection to get back lat,lon coordinates.
`
pixScale = d3.scaleLinear()
.domain([0, testPixels.width])
.range([0, ne[0] - nw[0]])
smallWidth = ne[0] - nw[0]
sx = smallWidth / testPixels.width
sy = (se[1] - ne[1]) / testPixels.height
smallTestPixels = testPixels.map(d => {
// we use the same x and y scale since the image should be preserving its aspect ratio when scaled down
return {v:d.v, x: sx * d.x, y: sy * d.y }
})
testGeoPoints = smallTestPixels.map(d => {
// invert after translating the image to the north west corner
let lnglat = mercator.invert([d.x + nw[0],d.y + nw[1]])
return {v: d.v, lng: lnglat[0], lat: lnglat[1] }
})
values = testGeoPoints.map(d => d.v)
binner = d3.bin()
binner(values)
pmLookup = new Map([
[201, 12], // the first value is the grayscale color, the second is the PM2.5
[200, 12],
[175, 35],
[150, 55],
[125, 150],
[100, 250],
[74, 350],
[75, 350],
[24, 500],
[25, 500]
])
testGeoPM = testGeoPoints.map(d => {
let ret = {...d}
ret.v = pmLookup.get(d.v)
return ret
})
colorPM1 = d3.scaleLinear()
.domain([0,50, 100, 150, 200, 250])
.range(["green", "yellow", "orange", "red", "maroon", "maroon"])
legend({color: colorPM1})
hexdata = d3.csvParse(await FileAttachment("hexbin-2020082812.csv").text(), d3.autoType)
hexbyhour = d3.group(hexdata, d => d.img)
hexhours = Array.from(hexbyhour.keys())
mapbayhexhourcanvas = {
const canvas = d3.create("canvas").node()
canvas.width = width
canvas.height = height
const ctx = canvas.getContext("2d")
let projection = d3.geoAlbersUsa()
.fitSize([width, height], california)
let path = d3.geoPath(projection)
.context(ctx)
ctx.strokeStyle = "#111"
ctx.lineWidth = 1
ctx.beginPath()
path(westCoast)
ctx.stroke()
// testGeoPoints.forEach(d => {
hexbyhour.get(hexhour).forEach(d => {
// ctx.fillStyle = `rgb(${[d.v,d.v,d.v]})`
ctx.fillStyle = colorPM1(d.v)
let p = projection([d.lng, d.lat])
ctx.fillRect(p[0], p[1], 1,1)
})
return canvas
}
testdata = d3.csvParse(await FileAttachment("100m_hourly_202008281200.csv").text(), d3.autoType)
binner(testdata.map(d => d.v))
mapbaytestcanvas = {
const canvas = d3.create("canvas").node()
canvas.width = width
canvas.height = height
const ctx = canvas.getContext("2d")
let projection = d3.geoAlbersUsa()
.fitSize([width, height], california)
// let path = d3.geoPath(projection)
let path = d3.geoPath(mercator)
.context(ctx)
ctx.strokeStyle = "#111"
ctx.lineWidth = 1
ctx.beginPath()
path(westCoast)
ctx.stroke()
let sx = (ne[0] - nw[0]) / testPixels.width
let sy = (sw[1] - ne[1]) / testPixels.height
// testGeoPoints.forEach(d => {
testdata.forEach(d => {
// ctx.fillStyle = `rgb(${[d.v,d.v,d.v]})`
ctx.fillStyle = colorPM1(pmLookup.get(d.v))
// let p = projection([d.lng, d.lat])
let lnglat = mercator.invert([d.x * sx + nw[0], d.y * sy + nw[1]])
// let p = projection([lnglat[0], lnglat[1]])
let p = mercator([lnglat[0], lnglat[1]])
if(p)
ctx.fillRect(p[0], p[1], 1, 1)
// ctx.fillRect(d.x, d.y, 1,1)
})
return canvas
}
sxx = (ne[0] - nw[0]) / width
syy = (sw[1] - ne[1]) / height
hexbin2 ={
let sx = (ne[1] - nw[1]) / width
let sy = (se[1] - ne[1]) / height
let hexbin2 = d3.hexbin()
.extent([[0, 0], [width, height]])
.radius(1)
.x(d => d.x)
.y(d => d.y)
return hexbin2
}
hexdata2 = hexbin2(testdata).map(d => ({x:d.x, y: d.y, v: d3.median(d, a => a.v)}))
round = function(n) {
let prec = 1000
return Math.floor(n * prec) / prec
}
mapbaytest2canvas = {
const canvas = d3.create("canvas").node()
canvas.width = width
canvas.height = height
const ctx = canvas.getContext("2d")
let projection = d3.geoAlbersUsa()
.fitSize([width, height], california)
let path = d3.geoPath(projection)
.context(ctx)
ctx.strokeStyle = "#111"
ctx.lineWidth = 1
ctx.beginPath()
path(westCoast)
ctx.stroke()
let sx = (ne[0] - nw[0]) / testPixels.width
let sy = (sw[1] - ne[1]) / testPixels.height
// testGeoPoints.forEach(d => {
hexdata2.forEach(d => {
// ctx.fillStyle = `rgb(${[d.v,d.v,d.v]})`
ctx.fillStyle = colorPM1(d.v)
// let p = projection([d.lng, d.lat])
let lnglat = mercator.invert([d.x * sx + nw[0], d.y * sy + nw[1]])
// let p = projection([lnglat[0], lnglat[1]])
let a = projection([round(lnglat[0]), round(lnglat[1])])
let p = alb([round(lnglat[0]), round(lnglat[1])])
// if(p)
if(p[0] > 0 && p[0] < width && p[1] > 0 && p[1] < height)
ctx.fillRect(a[0], a[1], 5, 5)
// ctx.fillRect(d.x, d.y, 1,1)
})
return canvas
}
// files = zip(await FileAttachment("hexbin-2020082812.csv@2.zip").arrayBuffer(), {
// base64: false,
// checkCRC32: true
// }).files
// hexdata3 = d3.csvParse(files["data/hexbin-2020082812.csv"].asText(), d3.autoType)
hexdata3 = d3.csvParse(await FileAttachment("hexbin-2020083000@2.csv").text(), d3.autoType)
hexbyhour3 = d3.group(inside3, d => d.i)
inside3 = hexdata3.filter(d => {
let p = alb([d.lng, d.lat])
return p[0] > 0 && p[0] < width && p[1] > 0 && p[1] < height
})
alb = d3.geoAlbersUsa()
.fitSize([width, height], countyExtent)
viewof hexhour3 = Scrubber(hexbyhour3.keys(), { delay: 200, format: d => d })
mapbayhex2canvas = {
const canvas = d3.create("canvas").node()
canvas.width = width
canvas.height = height
const ctx = canvas.getContext("2d")
let projection = alb
// let projection = d3.geoAlbersUsa()
// .fitSize([width, height], california)
let path = d3.geoPath(projection)
.context(ctx)
ctx.strokeStyle = "#111"
ctx.lineWidth = 1
ctx.beginPath()
path(westCoast)
ctx.stroke()
hexbyhour3.get(hexhour3).forEach(d => {
ctx.globalAlpha = 0.5
ctx.fillStyle = colorPM1(d.pm1)
let p = projection([d.lng, d.lat])
// if(p[0] > 0 && p[0] < width && p[1] > 0 && p[1] < height)
ctx.fillRect(p[0], p[1], 5, 5)
})
return canvas
}
mapbayhex3canvas = {
const canvas = d3.create("canvas").node()
canvas.width = width
canvas.height = height
const ctx = canvas.getContext("2d")
let projection = alb
let path = d3.geoPath(projection)
.context(ctx)
ctx.strokeStyle = "#111"
ctx.lineWidth = 1
ctx.beginPath()
path(westCoast)
ctx.stroke()
Array.from(hexbyhour3.keys()).slice(0,10).forEach(i => {
ctx.globalAlpha = 0.1
ctx.fillStyle = "white"
ctx.fillRect(0,0,width,height)
hexbyhour3.get(i).forEach(d => {
ctx.globalAlpha = 0.5
ctx.fillStyle = colorPM1(d.pm1)
let p = projection([d.lng, d.lat])
// if(p[0] > 0 && p[0] < width && p[1] > 0 && p[1] < height)
ctx.fillRect(p[0] + i*5, p[1] + i*5, 5, 5)
})
})
return canvas
}
alb.scale()
alb.translate()
import {fullhexbin} from "@enjalot/purpleair-historical-data"
mercator = d3.geoMercator()
.fitSize([width, height], california)
// .fitSize([width, height], [[region.west_lon, region.north_lat], [region.east_lon, region.south_lat]])
mercator.scale()
mercator.translate()
california = statesByName.get("California")
westCoast = ({type: "FeatureCollection", features: [
statesByName.get("California"),
statesByName.get("Oregon"),
statesByName.get("Arizona"),
statesByName.get("Nevada")
]})
// Choose a couple counties that define the extent I care about
countyExtent = ({
type: "FeatureCollection",
features: countyShapes.features.filter(d => (
d.properties.name == "Marin" ||
d.properties.name == "Santa Clara") && d.id.slice(0,2) == "06")
})
countyShapes = topojson.feature(us, us.objects.counties)//.features
stateShapes = topojson.feature(us, us.objects.states)//.features
statesByName = new Map(stateShapes.features.map(d => [d.properties.name, d]))
us = d3.json("https://unpkg.com/us-atlas@3/counties-10m.json")
height = 500
function cors(url) {
return "https://cors-anywhere.herokuapp.com/" + url
}
import {Scrubber} from "@mbostock/scrubber"
turf = require('@turf/turf')
topojson = require("topojson-client@3")
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.