Digital Humanist. Manhattan-based.
Published
Edited
Feb 3, 2021
13 stars
Dot Density election maps of America
viewof base_point_size = slider({title: "Point size at full zoom", min: .5, max: 5, value: 1})
current_scheme = [[66, 167, 245, 255], [219, 64, 74, 255]]
md`There's still a bit of cruft in this notebook; but coupled with the builder library at [here](https://observablehq.com/@bmschmidt/trifeather-builder), it's probably possible to make these for any other geojson with quantity values in the properties.`
import {fullscreen} from "@fil/fullscreen"
md`I only show 20% of the generated points at a time.`
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
map.point_opacity
update_colors(current_scheme)
viewof pixels = slider({title: "Number of pixels to spread", min: .1, max: 60, value: .1})
trifeather_cache = new Map()
state_info.get("NE")
lines = {
const trifeathers = []
for (let state of selected_states) {
const fips = state_info.get(state).fips
if (fips == "11" || fips == "31") {continue} // DC and Nebraska don't have lower houses
const url = `https://benschmidt.org/trifeather/tl_2019_${fips}_sldl.trifeather`;
let features;
if (trifeather_cache.get(url)) {
features = trifeather_cache.get(url)
} else {
const lines = await fetch(url).then(d=>d.arrayBuffer())
features = new TriFeather(lines)
trifeather_cache.set(url, features)
}
trifeathers.push(features)
}
return trifeathers
}
import { linear_us_state_order } from '@bmschmidt/useful-linear-orders-for-countries-and-states'
md`# Point Data.
Also stored as trifeather! But we'll present these as points, not as polygons.`
map.discard_share = discard_share
point_data = {
const point_data = []
for (let state_abbr of selected_states) {
const state = state_info.get(state_abbr).fips
const url = `https://benschmidt.org/trifeather/${state}-votes.trifeather`
let features
if (trifeather_cache.get(url)) {
features = trifeather_cache.get(url)
} else {
const bytes = await fetch(url).then(d=>d.arrayBuffer())
features = new TriFeather(bytes)
trifeather_cache.set(url, features)
}
point_data.push(features)
}
return point_data
}
md`# Some other lines.
`
congressional_districts = new TriFeather(await FileAttachment("congressional@5.trifeather").arrayBuffer())
TriFeather = trifeather.TriFeather
trifeather = require('trifeather@1.0.4')
timings = []
timings
{
const [points, time] = [d3.sum(timings, d => d[0]), d3.sum(timings, d => d[1])]
return md`${points} points in ${time} milliseconds, ${points/time} points per ms`
}
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
map.layers[0]
function random_points(frame, fields, n_represented = 1, index_function) {
if (index_function === undefined) {
index_function = c => fields.indexOf(c)
}
const field_index_array = fields.map(field => index_function(field))
let counts_by_field = []
for (let field of fields) {
const ix = index_function(field)
counts_by_field[ix] = counts_by_field[ix] || 0
console.log(field, frame.t.get(1))
counts_by_field[ix] += d3.sum(frame.t.getColumn(field).toArray())/n_represented
}
counts_by_field = counts_by_field.map(d => Math.round(d))
const total_counts = d3.sum(counts_by_field)
// Keep track of both the points we *do* need to allocate
// and the points we *should* need to allocate; this will
// allow slight adjustments to correct the direction.
const points_we_should_have_left = [...counts_by_field]
const points_we_do_have_left = [...counts_by_field]
const x_array = new Float32Array(total_counts)
const y_array = new Float32Array(total_counts)
const f_num_array = new Float32Array(total_counts)
const ix_array = d3.range(total_counts)
// We are going to place these points at random points along the way.
d3.shuffle(ix_array)
//let coord_positions = all_coords.map(() => -2);
// The overall position
let overall_position = -1;
for (let ix of d3.range(frame.t.length)) {
const f = frame.t.get(ix);
if (f.coord_resolution === null) {continue}
const vert_buffer = new DataView(f.vertices.buffer, f.vertices.byteOffset, f.vertices.byteLength)
let double_areas = new Array(counts_by_field.length).fill(f.pixel_area * 2)// save an op later by doubling here.
let number_neededs = new Array(counts_by_field.length).fill(0)
fields.forEach((field, i) => {
const ix = field_index_array[i]
const actual_number_needed = f[field]/n_represented
const adjustment_factor = points_we_do_have_left[ix]/points_we_should_have_left[ix]
number_neededs[ix] += actual_number_needed * adjustment_factor
})
// round in a random direction.
number_neededs = number_neededs.map(d => randround(d))
const metadata = []
const stride = f.coord_resolution / 8
const offset = f.coord_buffer_offset;
for (let i = 0; i < f.vertices.byteLength; i += stride * 3) {
let a, b, c;
[a, b, c] = ([0, 1, 2]).map(
ix => vert_buffer[`getUint${f.coord_resolution}`](i + ix*stride, true))
.map(n => frame.coord(n + offset))
// Ax(By - Cy) + Bx(Cy - Ay) + Cx(Ay - By)
const area = Math.abs(
a[0] * (b[1] - c[1]) + b[0] * (c[1] - a[1]) + c[0] * (a[1] - b[1])
)
// earcut seems to always return triangles in a form where the absolute
// value isn't necessary.
for (let f_num of d3.range(number_neededs.length)) {
const share = area/double_areas[f_num]
double_areas[f_num] -= area
let how_many_points_do_i_get = randround(number_neededs[f_num] * share)
points_we_should_have_left[f_num] -= number_neededs[f_num] * share;
points_we_do_have_left[f_num] -= how_many_points_do_i_get;
for (let i = 0; i < how_many_points_do_i_get; i++) {
// Assign no more if we're at the cap.
if ((number_neededs[f_num] -= 1) <= 0) {break}
const [x, y] = random_point(a, b, c)
overall_position++;
const writing_to = ix_array[overall_position]
x_array[writing_to] = x;
y_array[writing_to] = y;
f_num_array[writing_to] = f_num;
}
}
}
}
return {x_array, y_array, f_num_array}
}
1 + true
function randround(how_many_points_do_i_get) {
const leftover = how_many_points_do_i_get % 1;
// Random round to decide if you get a fractional point.
if (Math.random() > leftover) {
how_many_points_do_i_get -= leftover
} else {
how_many_points_do_i_get += (1 - leftover)
}
return how_many_points_do_i_get
}
function random_point([ax, ay], [bx, by], [cx, cy]) {
const a = [bx - ax, by - ay]
const b = [cx - ax, cy - ay]
let [u1, u2] = [Math.random(), Math.random()]
if (u1 + u2 > 1) {u1 = 1 - u1; u2 = 1 - u2}
const w = [u1 * a[0] + u2 * b[0], u1 * a[1] + u2 * b[1]]
return [w[0] + ax, w[1] + ay]
}
fullscreen(div, {center: true})
toGlColor = input => {const {r, g, b} = d3.rgb(input); return !isNaN(r) ? [r/255, g/255, b/255]: [.5, .5, .5]}
colorscale = d3.scaleSequential(d3.interpolateRdBu).domain([0.05, .95])
map = {
const map = new TriMap(div, lines)
invalidation.then(() => {
map.cleanup()
map.regl.destroy()
})
return map
}
md`# TriMap
A handler class to deal with interactions between regl contexts and feather feature sets. I'll use this more later.`
md`## Sentinels
To listen for changes in the sliders without rebuilding the whole webgl context.`
candidates
{
// map.add_layer(pop_data)
let f = undefined;
map.generate_random_points(candidates, n_represented, point_data, true)
}
n_represented
rgb2glcolor = col => {
const {r, g, b} = d3.rgb(col)
return [r, g, b, 255]
}
md`# TriMap class
This is living in this notebook for now--eventually, it will go into the trifeather package or another one like it.
`
map.layers[0].features[1].properties
map.filter = function(d) {
if (d.holc_grade === undefined) {return true}
return selected_states.indexOf(d.state) > -1
}
class TriMap {
constructor(div, layers) {
this.div = div
this.regl = wrapRegl({canvas: div, extensions: ["OES_element_index_uint"]})
for (let layer of layers) {
layer.bind_to_regl(this.regl)
}
this.layers = layers;
const {width, height} = div
this.width = width
this.height = height
this.set_magic_numbers()
this.prepare_div(width, height)
this.color_map = this.regl.texture( {
width: 128,
format: "rgba",
height: 1,
data: d3.range(128*4)})
this.set_renderer()
this.random_points = []
}
add_layer(layer) {
layer.bind_to_regl(this.regl)
this.layers.push(layer)
}
get filter() {
return this._filter ? this._filter : function (d) {return true}
}
set filter(f) {
this._filter = f
}
cleanup() {
this.cleanup_point_buffers()
this.cleanup_frame_buffers()
this.cleanup_poly_buffers()
}
cleanup_poly_buffers() {
// pass
}
cleanup_frame_buffers() {
if (this.buffers) {
for (let buffer of this.buffers.values()) {
buffer.destroy()
}
}
}
fbo(name) {
this.buffers = this.buffers || new Map()
if (this.buffers.get(name)) {
return this.buffers.get(name)
}
const fbo = this.regl.framebuffer({
width: this.width,
height: this.height,
stencil: false
})
this.buffers.set(name, fbo)
return this.buffers.get(name)
}
set_magic_numbers() {
// It's a major pain to align regl with d3 scales.
const { layers, width, height } = this;
const extent = JSON.parse(JSON.stringify(layers[0].bbox));
for (let layer of layers) {
if (layer.t.get(0).get("holc_id")) {
continue
}
const { bbox } = layer
extent.x = d3.extent([...extent.x, ...bbox.x])
extent.y = d3.extent([...extent.y, ...bbox.y])
}
const scales = {};
const scale_dat = {'x': {}, 'y': {}}
for (let [name, dim] of [['x', width], ['y', height]]) {
const limits = extent[name]
scale_dat[name].limits = limits;
scale_dat[name].size_range = limits[1] - limits[0]
scale_dat[name].pixels_per_unit = dim / scale_dat[name].size_range
}
const data_aspect_ratio =
scale_dat.x.pixels_per_unit / scale_dat.y.pixels_per_unit
let x_buffer_size = 0, y_buffer_size = 0,
x_target_size = width, y_target_size = height;
if (data_aspect_ratio > 1) {
// There are more pixels in the x dimension, so we need a buffer
// around it.
x_target_size = width / data_aspect_ratio;
x_buffer_size = (width - x_target_size)/2
} else {
y_target_size = height * data_aspect_ratio;
y_buffer_size = (height - y_target_size)/2
}
scales.x =
d3.scaleLinear()
.domain(scale_dat.x.limits)
.range([x_buffer_size, width-x_buffer_size])
scales.y =
d3.scaleLinear()
.domain(scale_dat.y.limits)
.range([y_buffer_size, height-y_buffer_size])
this.magic_numbers = window_transform(
scales.x,
scales.y, width, height)
.map(d => d.flat())
}
prepare_div(width, height) {
this.zoom = {transform: {k: 1, x: 0, y:0}}
d3.select(this.div)
.call(d3.zoom().extent([[0, 0], [width, height]]).on("zoom", (event, g) => {
this.zoom.transform = event.transform
}));
return this.div;
}
get size_func() {
return this._size_function ? this._size_function : () => 1
}
set size_func(f) {
this._size_function = f
}
set color_func(f) {
this._color_function = f
}
get index_color() {
return function(f) {
if (f._index_color) {return f._index_color}
f._index_color = [0, 1, 2].map(d => 1 / 255 * Math.floor(Math.random() * 255))
return f._index_color
}
}
get color_func() {
return this._color_function ? this._color_function : p => greyscale(p.ix).slice(0, 3).map(c => c/255)
}
draw_edges(layer) {
const {regl} = this;
const colors = this.fbo("colorpicker")
const edges = this.fbo("edges")
colors.use(d => {
this.regl.clear({color: [0, 0, 0, 0]})
this.poly_tick(layer)
})
edges.use(() => {
this.regl.clear({color: [1, 1, 1, 1]})
const shader = simple_shader(this.regl, edge_detection)
shader({layer: colors})
})
// Copy the edges to a ping-pong shader to be blurred.
const pingpong = [this.fbo("ping"), this.fbo("pong")]
const copier = simple_shader(this.regl, copy_shader)
let alpha = hidden_state.decay;
const { decay } = hidden_state;
pingpong[0].use(() => {
regl.clear({color: [0, 0, 0, 0]})
copier({layer: edges})
})
const edge_propagator = simple_shader(this.regl, edge_propagation)
while (alpha > 1/255) {
pingpong[1].use(() => {
regl.clear({color: [0, 0, 0, 0]})
edge_propagator({layer: pingpong[0], decay: decay})
})
// swap the buffers.
alpha *= decay
pingpong.reverse()
}
const final_shade = simple_shader(this.regl, alpha_color_merge, true)
// First copy the blur
final_shade({alpha: pingpong[0], color: colors})
}
cleanup_point_buffers() {
this.random_points.map(d => {
d.x.destroy()
d.y.destroy()
d.f_num.destroy()
d.ix.destroy()
})
}
generate_random_points(fields, represented=1, layers, clear = true, index_function) {
if (clear) {
this.cleanup_point_buffers()
this._number_of_points = 0
this.random_points = []
}
for (let layer of layers) {
const { regl } = this;
const {x_array, y_array, f_num_array} = random_points(layer, fields, represented, index_function);
this._number_of_points += x_array.length
let this_item = {
x: regl.buffer(x_array),
y: regl.buffer(y_array),
f_num: regl.buffer(f_num_array),
ix: regl.buffer(d3.range(x_array.length)),
count: x_array.length,
};
this.random_points.push(this_item)
}
}
point_tick() {
const { regl } = this;
const calls = []
// multiple interleaved tranches prevent Trump or Biden from always being on top. This is
// an issue with Williamson's maps, which over-represent the Hispanic population of DC because it
// gets plotted last.
const alpha_scale = d3.scaleSqrt().domain([0, 500]).range([0, 1])
for (let pointset of this.random_points) {
calls.push({
x: pointset.x,
y: pointset.y,
ix: pointset.ix,
f_num: pointset.f_num,
transform: this.zoom.transform,
// Drops the last point in each tranch--needs a modulo operation to know how
// many to expect.
count: pointset.count,
centroid: [0, 0],
size: this.point_size ? this.point_size : 1,
alpha: this.point_opacity > 1/255 ? this.point_opacity : 1/255
})
}
this.render_points(calls)
}
tick(wut) {
const { regl } = this
regl.clear({
color: [1, 1, 1, 1],
})
const alpha = 1
if (wut === "points") {
this.point_tick()
} else {
for (let layer of this.layers) {
this.draw_edges(layer)
}
this.fbo("points").use(d => {
regl.clear({
color: [0, 0, 0, 0]
})
this.point_tick()
})
const copier = simple_shader(this.regl, copy_shader, true)
copier({layer: this.fbo("points")})
}
}
poly_tick(layer) {
const { regl } = this;
const calls = []
for (let feature of layer) {
//if (feature.properties['2020_tot'] === null) {continue}
const {vertices, coords} = feature;
calls.push({
transform: this.zoom.transform,
color: this.color_func(feature),
centroid: [feature.properties.centroid_x, feature.properties.centroid_y],
size: this.size_func(feature),
alpha: 1,
vertices: vertices,
coords: coords
})
}
this.render_polygons(calls)
}
get point_vertex_shader() {return `
precision mediump float;
attribute float a_x;
attribute float a_y;
attribute float a_ix;
attribute float a_f_num;
uniform sampler2D u_color_map;
uniform float u_discard_prob;
uniform float u_size;
uniform vec2 u_centroid;
varying vec4 fragColor;
uniform float u_k;
uniform float u_time;
varying vec4 fill;
// Transform from data space to the open window.
uniform mat3 u_window_scale;
// Transform from the open window to the d3-zoom.
uniform mat3 u_zoom;
uniform mat3 u_untransform;
uniform float u_scale_factor;
float distortion_factor = exp(log(u_k)*u_scale_factor);
vec4 discard_me = vec4(-100., -100., 0., 1.);
float tau = 3.14159265358 * 2.;
highp float ix_to_random(in float ix, in float seed) {
// For high numbers, taking the log avoids coincidence.
highp float seed2 = log(ix) + 1.;
vec2 co = vec2(seed2, seed);
highp float a = 12.9898;
highp float b = 78.233;
highp float c = 43758.5453;
highp float dt = dot(co.xy, vec2(a, b));
highp float sn = mod(dt, 3.14);
return fract(sin(sn) * c);
}
vec2 box_muller(in float ix, in float seed) {
// Box-Muller transform gives you two gaussian randoms for two uniforms.
highp float U = ix_to_random(ix, seed);
highp float V = ix_to_random(ix, seed + 17.123123);
return vec2(sqrt(-2. * log(U)) * cos(tau * V),
sqrt(-2. * log(U)) * sin(tau * V));
}
// From another project
vec2 circle_jitter(in float ix, in float aspect_ratio, in float time,
in float radius, in float speed) {
float rand1 = ix_to_random(ix, 3.0);
float rand2 = ix_to_random(ix, 4.0);
float stagger_time = rand1 * tau;
// How long does a circuit take?
float units_per_period = radius * radius * tau / 2.;
float units_per_second = speed / 100.;
float seconds_per_period = units_per_period / units_per_second;
seconds_per_period = tau / speed;
float time_period = seconds_per_period;
if (time_period > 1e4) {
return vec2(0., 0.);
}
// Adjust time from the clock to our current spot.
float varying_time = time + stagger_time * time_period;
// Where are we from 0 to 1 relative to the time period
float relative_time = 1. - mod(varying_time, time_period) / time_period;
float theta = relative_time * tau;
return vec2(cos(theta), aspect_ratio * sin(theta)) *
radius * rand2;
}
vec2 jitter(in float ix, in float radius) {
return circle_jitter(ix, 1.2, u_time, radius, .5);
}
// We can bundle the three matrices together here for all shaders.
mat3 from_coord_to_gl = u_window_scale * u_zoom * u_untransform;
void main () {
vec2 position = vec2(a_x, a_y);
vec3 p = vec3(position, 1.) * from_coord_to_gl;
// vec2 jittered = jitter(a_ix, .0004 * distortion_factor) * distortion_factor;
// p = p + vec3(jittered.xy, 0.);
float my_offset = ix_to_random(a_ix, 3.2);
float keep_prob = (1. - u_discard_prob);
// always stay on screen 10 seconds.
float time_period = 10./(keep_prob);
float fraction_of_time = fract(u_time / time_period);
float size_dilate = 0.;
float my_fract = fract(fraction_of_time + my_offset);
if (my_fract >= keep_prob) {
gl_Position = discard_me;
gl_PointSize = 0.;
return;
} else {
float fraction_within = my_fract / keep_prob;
size_dilate = abs(1. - 4.*pow((.5 - fraction_within), 2.));
size_dilate = clamp(size_dilate, 0., 1.);
}
gl_Position = vec4(p, 1.0);
gl_PointSize = u_size * distortion_factor * size_dilate;
//gl_PointSize += exp(sin(u_time / 2. + a_f_num/6. * 2. * 3.1415));
fragColor = texture2D(u_color_map, vec2(a_f_num / 128., .5));
}
`}
get vertex_shader() {return `
precision mediump float;
attribute vec2 position;
uniform float u_size;
uniform vec2 u_centroid;
varying vec4 fragColor;
uniform float u_k;
uniform float u_time;
uniform vec3 u_color;
varying vec4 fill;
// Transform from data space to the open window.
uniform mat3 u_window_scale;
// Transform from the open window to the d3-zoom.
uniform mat3 u_zoom;
uniform mat3 u_untransform;
uniform float u_scale_factor;
// We can bundle the three matrices together here for all shaders.
mat3 from_coord_to_gl = u_window_scale * u_zoom * u_untransform;
void main () {
// scale to normalized device coordinates
// gl_Position is a special variable that holds the position
// of a vertex
vec2 from_center = position-u_centroid;
vec3 p = vec3(from_center * u_size + u_centroid, 1.) * from_coord_to_gl;
gl_Position = vec4(p, 1.0);
gl_PointSize = u_size * (exp(log(u_k)*u_scale_factor));
fragColor = vec4(u_color.rgb, 1.);
//gl_Position = vec4(position / vec2(1., u_aspect), 1., 1.);
}
`}
set_renderer() {
this.render_polygons = this.regl(this.renderer())
this.render_points = this.regl(this.renderer("points"))
}
get point_frag() { return `
precision highp float;
uniform float u_alpha;
varying vec4 fragColor;
void main() {
vec2 coord = gl_PointCoord;
vec2 cxy = 2.0 * coord - 1.0;
float r_sq = dot(cxy, cxy);
if (r_sq > 1.0) {discard;}
gl_FragColor = fragColor * u_alpha;
}`}
get triangle_frag() { return `
precision highp float;
uniform float u_alpha;
varying vec4 fragColor;
void main() {
gl_FragColor = fragColor * u_alpha;
}`}
renderer(wut = "polygons") {
const { regl, magic_numbers } = this;
const definition = {
depth: {
enable: false
},
blend: {enable: true, func: {
srcRGB: 'one',
srcAlpha: 'one',
dstRGB: 'one minus src alpha',
dstAlpha: 'one minus src alpha',
}
},
vert: wut == 'polygons' ? this.vertex_shader : this.point_vertex_shader,
frag: wut == 'polygons' ? this.triangle_frag : this.point_frag,
attributes: {
a_x: regl.prop("x"),
a_y: regl.prop("y"),
a_ix: regl.prop("ix"),
a_f_num: regl.prop("f_num"),
position: wut == "polygons" ?
(_, {coords}) => coords:
(_, {position, stride, offset}) => {return {buffer: position, offset , stride}}
},
count: regl.prop("count"),
elements: wut == "polygons" ? (_, {vertices}) => vertices : undefined,
uniforms: {
u_time: (context, _) => performance.now()/500,
u_scale_factor: () => this.scale_factor ? this.scale_factor : .5,
u_color_map: () => this.color_map,
u_k: function(context, props) {
return props.transform.k
},
u_discard_prob: () => this.discard_share,
u_centroid: propd("centroid", [0, 0]),
u_color: (_, {color}) => color ? color : [.8, .9, .2],
u_window_scale: magic_numbers[0].flat(),
u_untransform: magic_numbers[1].flat(),
u_zoom: function(context, props) {
const g = [
// This is how you build a transform matrix from d3 zoom.
[props.transform.k, 0, props.transform.x],
[0, props.transform.k, props.transform.y],
[0, 0, 1],
].flat()
return g
},
u_alpha: (_, {alpha}) => alpha ? alpha : 1,
u_size: (_, {size}) => size || 1,
},
primitive: wut == "polygons" ? "triangles" : "points"
}
if (wut === "polygons") {
delete definition['count']
}
return definition
}
}
m = new Uint16Array([1, 2, 3, 4, 5, 6])
greyscale = d3.scaleOrdinal().range(greys)
greyscale.range()
function propd(string, def) {
return (_, props) => {
if (props[string] !== undefined) {return props[string]}
return def
}
}
wrapRegl = require("regl")
d3 = require("d3@v6")
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.