Published
Edited
Nov 24, 2020
14 stars
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
draw_loop = {while(true) {
yield(draw_frame(date))
}}
Insert cell
dates = date_names.map(d => new Date(Date.parse(d)))
Insert cell
viewof zoom = slider({title: "zoom (square root)", min: .5, max: 15, value: 1, step: .1})
Insert cell
Insert cell
Insert cell
md`## Some scales

Not sure if I still use them all!
`
Insert cell
rate_of_change = d3.scaleDivergingLog().domain([5,1, .2]).interpolator(d3.interpolatePiYG).clamp(true)
Insert cell
incidence_color = d3.scaleDivergingLog().domain([1e-2, 1e-3,1e-4 ]).interpolator(d3.interpolateRdBu).clamp(true)
Insert cell
date = ddate.toISOString().slice(0, 10)
Insert cell
distortion = d3.scaleSqrt().domain([0, 1/1000]).range([0, 1])
Insert cell
size_scale = d3.scaleSqrt().domain([0, 50]).range([0, 10])
Insert cell
delta_scale = d3.scaleDivergingLog().domain([.2, 1, 10]).clamp(true).range([0, 0, 1])
Insert cell
md`## Draw command`
Insert cell
hi_res_counties.get(0).vertices
Insert cell
draw_frame = function(date) {
const overall_radius = jitter_radius / 100 * (Math.sin(Date.now()/1000) + 1)/2
const prop_list = []
let ix = -1

for (let feature of hi_res_counties) {
ix += 1
if (feature.vertices === null) {continue}
if (feature.coordinates === null) {continue}
if (handler.element_handler.get(ix) === undefined) {
handler.element_handler.set(ix, gl.elements({
primitive: 'triangles',
usage: 'static',
data: feature.vertices,
type: "uint" + feature.coord_resolution,
length: feature.vertices.length, // in bytes
count: feature.vertices.length / feature.coord_resolution * 8
}))
handler.coord_handler.post_data(ix, feature.coordinates)
}
if (handler.get_prop("angle", ix) === undefined) handler.set_prop("angle", ix, 0)
let rand = handler.get_prop("rand", ix) || handler.set_prop("random", ix, 0)
const { STATE, COUNTY } = feature
let c_stats = long_term_corona_counts.get(STATE + COUNTY + "-" + date)
const pop = pops.get(`${STATE}${COUNTY}`)
if (c_stats == undefined) {
c_stats = {delta: 1, rolling: 0};
}
let { delta, rolling } = c_stats;
const rate = rolling/pop

// Increment the current angle by an amount.
const pop_per_area = pop/(feature.CENSUSAREA + 1e-10)
const magnitude = distortion(rate)
if (isNaN(delta)) {
delta = 1
} else if (delta <= 0) {
delta = .1
}
handler.set_prop("angle", ix, handler.get_prop("angle", ix) + delta_scale(delta))
const t = Math.sin(Date.now()/(600+rand[1]*10))
let color = incidence_color(rate) || 'rgb(0, 0, 0)'
let [r, g, b] = color.split(/\(|\)/)[1].split(",").map(d => +d)
//r = 128; g = 128; b = 128;
let x_shift = -feature.centroid_x
let y_shift = -feature.centroid_y
const v = {
color: [r/255, g/255, b/255],
zoom,
centroid: [feature.centroid_x, feature.centroid_y],
angle: handler.get_prop("angle", ix),
scale: scale_by_votes == "yes" ? size_scale(rolling/feature.CENSUSAREA) : 1,
elements: handler.element_handler.get(ix),
coords: handler.coord_handler.get(ix),
translate: [0, 0],
incidence: incidence

}
prop_list.push(v)
}
render(prop_list)
}

Insert cell
import { ReglBufferHandler } from '@bmschmidt/pre-triangulated-binary-maps-for-webgl'
Insert cell
handler = new ReglBufferHandler(gl)
Insert cell
gl = regl({canvas:myCanvas});
Insert cell
start_time = Date.now()

Insert cell
import { radio, slider } from '@jashkenas/inputs'
Insert cell
render = gl(renderer);
Insert cell
colorscheme = d3.scaleLinear().domain([0, 100]).range(d3.schemeBuGn)
Insert cell
groups = {return {36005: "New York", 36047: "New York", 36061: "New York", 36081: "New York", 36085: "New York"}}
Insert cell
function tri_merge(triangles, coords, int_sizes) {
}
Insert cell
feathered
Insert cell
renderer = (
//Starting point: https://observablehq.com/@marcom13/mesh-rendering-using-webgl-regl
{
vert: `
precision mediump float;
attribute vec2 position;
uniform float aspect;
uniform float u_zoom;
uniform float u_scale;

uniform float u_incidence;
uniform float u_theta;
uniform vec3 color;
uniform vec2 translate;
uniform vec2 u_centroid;
varying vec3 fragColor;

mat2 rotate2d(float _angle){
return mat2(cos(_angle),-sin(_angle),
sin(_angle),cos(_angle));
}


void main () {
gl_PointSize = 2.;
vec2 rot;
fragColor = color;
vec2 p_from_centroid = position - u_centroid;
if (u_incidence == 0.) {
rot=p_from_centroid*rotate2d(u_theta);
} else {

float angle = atan(p_from_centroid.x, p_from_centroid.y);
float magnitude = sqrt(dot(p_from_centroid, p_from_centroid));
float adjust = magnitude * (2. + sin(angle - u_theta)) / 2.;
if (u_scale != 1. && adjust > magnitude) {
adjust = magnitude;
}
rot = vec2(adjust * sin(angle), adjust * cos(angle));

//rot = vec2(.03 * sin(angle), .03 * cos(angle));
}
vec2 rot_adj = rot * u_scale + u_centroid;
gl_Position = vec4(
rot_adj.x *u_zoom + translate.x * u_zoom,
-rot_adj.y * u_zoom * aspect - translate.y*u_zoom*aspect, 0., 1.);
}
`,
frag: `
precision mediump float;
varying vec3 fragColor;
void main () {
gl_FragColor = vec4(fragColor, 1.);
}
`,
attributes: {
position: (state, props) => props.coords
},
elements: function (state, props) {return props.elements},
uniforms: {
u_centroid: gl.prop('centroid'),
u_theta: (_, {angle}) => angle !== undefined ? angle : 0,
u_zoom: (_, {zoom}) => zoom * zoom,
u_scale: (_, {scale}) => scale,
u_incidence: (_, {incidence}) => incidence == "distortion" ? 1 : 0,
aspect: width/height,
translate: (state, props) => props.translate,
color: (state, props) => props.color
},
primitive: "triangle",
}

)
Insert cell
height = width * .75
Insert cell
{
return "in process-merging multiple triangulated features together."
const g = {}
for (let f of ["coordinates", "vertices", "coord_resolution", "pixel_area", "centroid_x", "centroid_y"]) {
g[f] = [hi_res_counties.getColumn(f).get(1), hi_res_counties.getColumn(f).get(2), hi_res_counties.getColumn(f).get(16)]
}
const l = d3.sum(g.coordinates.map(d => d.length))
const coordinates = new Uint8Array(l)
let offset = 0
for (let element of g.coordinates) {
coordinates.set(element, offset)
offset += element.length
}
const n_points = coordinates.length * 4 / 2; // (i.e.: 4 byte floats, two floats per point).
const coord_resolution = n_points < 2**8 ? 8 : n_points < 2**16 ? 16 : 32
const lengths = g.vertices.map((d, i) => d.length/g.coord_resolution[i] * 8)
const Buffer = getInt(coord_resolution)
const vertices = new Buffer(d3.sum(lengths))
let buffer_offset = 0;
let index_offset = 0;
g.vertices.forEach((d, i) => {
const Res = getInt(g.coord_resolution[i])
const indices = new Res(g.vertices[i].buffer)
// PROBLEM--ACCESSING THE BUFFER THIS WAY RETURNS THE WHOLE UNDERLYING BUFFER, NOT JUST THE SLICE
// REPRESENTING THE DATA WE'RE INTERESTING IN. NEED TO INCLUDE OFFSETS SOMEHOW.
console.log(indices, g.coord_resolution[i])
indices.forEach((index, i) => {
vertices[buffer_offset] = indices[i] + index_offset
buffer_offset += 1
})
index_offset += lengths[i]
})
const pixel_area = d3.sum(g.pixel_area)
const centroid_x = d3.sum(d3.zip(g.centroid_x, g.pixel_area).map(([a, b]) => a * b/pixel_area))
const centroid_y = d3.sum(d3.zip(g.centroid_y, g.pixel_area).map(([a, b]) => a * b/pixel_area))
return {
coordinates,
vertices, coord_resolution, pixel_area, centroid_x, centroid_y}
}
Insert cell
function getInt(n) {
if (n == 8) {return Uint8Array}
if (n == 16) {return Uint16Array}
if (n == 32) {return Uint32Array}
throw "No such buffer size"
}
Insert cell
new Uint32Array((new Uint8Array([1, 2, 3, 4]).buffer))
Insert cell
hi_res_counties.schema.fields.map(d => d.name)
Insert cell
import {Scrubber} from "@mbostock/scrubber"
Insert cell
arrow = require("apache-arrow")
Insert cell
raw_data = FileAttachment("full@2.feather").arrayBuffer()
Insert cell
feathered = arrow.Table.from(raw_data)
Insert cell
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.

Insert cell
hi_res_counties = arrow.Table.from(await FileAttachment("countries (3).gleofeather").arrayBuffer())
Insert cell
hi_res_counties.schema
Insert cell
import { pops as raw_pops } from '@codingwithfire/cmu-covidcast-api-bubbles-export'
Insert cell
pops = {
// raw_pops.set("36061", 8.5e06)
return raw_pops
}
Insert cell
import {aq, op} from '@uwdata/arquero'
Insert cell
regl = require("regl") // Use the latest Version
Insert cell
hopkins = aq.fromCSV(await fetch("https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_US.csv").then(d => d.text()))
Insert cell
t2 = hopkins
.select(aq.not(["Lat", "Long", "Long_", "code3", "iso3", "iso2", "UID", "Admin2", "Province_State", "Country_Region"]))
.derive({FIPS: d => op.padstart(d.FIPS, 5, "0")})
.fold(aq.range(2, hopkins.numCols() - 1))
.groupby(["FIPS", "Combined_Key"])
.orderby("key")
.rollup({values: d => op.values(d.value)})


Insert cell
date_names = hopkins.columnNames().map(d => [d, new Date(Date.parse(d))]).filter(d => Date.parse(d[0])).map(d => d[1].toISOString().slice(0, 10))
Insert cell
long_term_corona_counts = {
const counts = new Float32Array(date_names.length)
const rolling = new Float32Array(date_names.length)
const delta = new Float32Array(date_names.length)
const datecounts = new Map()
for (let row of t2) {
// return row
const val = row.values
counts[0] = val[0]

for (let i = 1; i < val.length; i++) {
counts[i] = val[i] > val[i-1] ? val[i] - val[i-1] : 0
}
for (let i = 0; i < 14; i++) {
rolling[i] = d3.sum(counts.slice(0, i))
}
for (let i = 14; i < counts.length; i ++) {
rolling[i] = d3.sum(counts.slice(i-7, i))
}

delta.fill(0)
// NB this must be even or else counts and delta have to be switched.
blur(rolling, delta, 4)
for (let i=0; i < rolling.length; i++) {
if (i < 7) {delta[i] = 1} else {
delta[i] = rolling[i]/rolling[i-7]
}
}
for (let i = 0; i < delta.length; i++) {
const date = date_names[i]
datecounts.set(`${row.FIPS}-${date_names[i]}`,
{rolling: rolling[i],
delta: delta[i]})
}
}
return datecounts
}
Insert cell
function blur(ping, pong, passes=1) {
if (passes == 0) {
return
}
for (let i=1; i < ping.length-1; i++) {
pong[i] = (ping[i-1] + ping[i] + ping[i+1])/3
}
pong[0] = (ping[0] + ping[1])/2
pong[pong.length-1] = (ping[pong.length-1] + ping[pong.length-2])/2
blur(pong, ping, passes-1)
}
Insert cell
d3 = require("d3@v6", "d3-geo-projection")
Insert cell

One platform to build and deploy the best data apps

Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.
Learn more