Digital Humanist. Manhattan-based.
Published
Edited
Dec 5, 2020
Importers
1 star
{
if (display == "yes") {
render(true)
return ctx.canvas}
else {
//render(false)
}
return "Done"
}
import { radio } from '@jashkenas/inputs'
md`# Data Converter
You can convert any geojson file into this format and download it using the button below; just change the source.
You can also live-transform by importing the feature_collection_to_feather_frame function below.
`
url = 'https://raw.githubusercontent.com/martynafford/natural-earth-geojson/master/110m/cultural/ne_110m_admin_0_countries_lakes.json'
raw_geojson = d3.json(url)
raw_geojson.features.filter(d => d.properties.SOVEREIGNT !== "Antarctica")
geojson = {
const geojson = raw_geojson
// For the mapping I'm doing of the world with distortion,
// it is preferable to remove exclaves like Alaska, the portions of Russia past -180 longitude,
// and French Guinea
geojson.features = geojson.features.filter(d => d.properties.SOVEREIGNT !== "Antarctica")
return geojson
for (let feature of geojson.features) {
if (false) {//feature.properties.GEOUNIT.match(/United States of America|United Kingdom|France|Russia/)) {
console.log(feature.properties.GEOUNIT)
reduce_to_largest_elements(feature)
}
}
return geojson
}
function reduce_to_largest_elements(feature, n = 1) {
const coords = feature.geometry.coordinates.sort((a, b) => +b[0].length - +a[0].length)
feature.geometry.coordinates = feature.geometry.coordinates.slice(0, n)
}
projection = d3.geoGingery().scale(300).lobes(6).rotate([98, -40, 0])//d3.geoInterruptedMollweide().scale(5000)//d3.geoMercator().scale(2).translate([-0, 0])//d3.geoInterruptedHomolosine().translate([0, 0]).scale(.2)
country_feather = feature_collection_to_feather_frame(geojson, projection, {clip_to_sphere: clip_to_sphere == "yes"})
arrow = require('apache-arrow@2.0.0')
function download_button() {return html`
${DOM.download(new Blob([country_feather.serialize().buffer], {type: "application/octet-stream"}), "countries.gleofeather", `Download ${country_feather.length} row gleofeather`)}
`}
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
ctx = DOM.context2d(width, width/2)
render = function(val) {
if (!val) {return}
const [[xmin, ymin], [xmax, ymax]] = secret.bounds;
const scale = (xmax-xmin)/width
ctx.save()
ctx.clearRect(0, 0, width, width)
ctx.scale(scale, scale)
// ctx.translate(width/2, ctx.canvas.height/2)
for (let feature of country_feather) {
ctx.fillStyle = "red"
const coords = new Float32Array(new Uint8Array(feature.coordinates).buffer)
const vertices = new window[`Uint${feature.coord_resolution}Array`](new Uint8Array(feature.vertices).buffer)
for (let i = 0; i < vertices.length; i += 3) {
if (i > vertices.length/2) {ctx.fillStyle="blue" }
ctx.beginPath();
ctx.moveTo(coords[2*vertices[i]], coords[2*vertices[i]+1]);
ctx.lineTo(coords[2*vertices[i+1]], coords[2*vertices[i+1]+1]);
ctx.lineTo(coords[2*vertices[i+2]], coords[2*vertices[i+2]+1]);
ctx.fill();
}
}
ctx.restore()
}
country_feather.get(1)
md`# Functions to turn geoJSON into feather frames.`
function polygon_to_triangles(polygon) {
// Actually perform the earcut work on a polygon.
const el_pos = []
const coords = polygon.flat(2)
const vertices = earcut(...Object.values(earcut.flatten(polygon)))
return { coords, vertices }
}
clip = require('polygon-clipping')
secret = new Object()
d3.geoPath().bounds(geojson)
yield secret.bounds
feature_collection_to_feather_frame = function(
feature_collection, projection, options = {dictionary_threshold: .75, clip_to_sphere: false}) {
if (projection === undefined) {throw "Must define a projection"}
// feature_collections: a (parsed) geoJSON object.
// projection: a d3.geoProjection instance; eg, d3.geoMollweide().transform([10, 20])
// options:
const properties = new Map()
properties.set("id", [])
const vertices = []
const coordinates = []
// Stores the number of bytes used for the coordinates.
const coord_resolutions = []
// centroids let you have fun with shapes. Store x and y separately.
const centroids = [[], []]
// Storing areas makes it possible to weight centroids.
const areas = []
let i = -1;
const path = d3.geoPath()
let clip_shape;
if (options.clip_to_sphere) {
clip_shape = d3.geoProject({"type": "Sphere"}, projection)
}
const bounds = [[Infinity, Infinity], [-Infinity, -Infinity]]
for (let feature of feature_collection.features) {
i++;
properties.get("id")[i] = feature.id;
for (let [k, v] of Object.entries(feature.properties)) {
if (!properties.get(k)) {properties.set(k, [])}
properties.get(k)[i] = v
}
/* Project and get new coords */
let projected = d3.geoProject(feature.geometry, projection);
if (options.clip_to_sphere) {
const new_coords = clip.intersection(projected.coordinates, clip_shape.coordinates)
if (projected.type == "Polygon" && typeof(new_coords[0][0][0] != "numeric")) {
projected.type = "MultiPolygon"
}
projected.coordinates = new_coords
}
[centroids[0][i], centroids[1][i]] = path.centroid(projected)
areas[i] = path.area(projected)
const loc_bounds = d3.geoPath().bounds(projected)
for (let dim of [0, 1]) {
if (loc_bounds[0][dim] < bounds[0][dim]) {bounds[0][dim] = loc_bounds[0][dim]}
if (loc_bounds[1][dim] > bounds[1][dim]) {bounds[1][dim] = loc_bounds[1][dim]}
}
let loc_coordinates;
if (projected === null) {
coord_resolutions[i] = null
coordinates[i] = null
vertices[i] = null
continue
} else
if (projected.type == "Polygon") {
loc_coordinates = [projected.coordinates]
} else if (projected.type == "MultiPolygon") {
loc_coordinates = projected.coordinates
} else {
throw "All elements must be polygons or multipolgyons."
}
let all_coords = []
let all_vertices = []
for (let polygon of loc_coordinates) {
const current_vertex = all_coords.length/2
const { coords, vertices } = polygon_to_triangles(polygon);
all_coords.push(...coords)
// If need to shift because we may be storing multiple triangle sets on a feature.
all_vertices.push(...vertices.map(d => d + current_vertex))
}
// Flatten to a silly degree just in case?
all_vertices = all_vertices.flat(10)
coordinates[i] = Float32Array.from(all_coords.flat(20))
// The type of the vertex array can be smaller if there
// are fewer coordinates to reference.
let MyArray
if (all_coords.length < 2**8) {
coord_resolutions[i] = 8
MyArray = Uint8Array
} else if (all_coords.length < 2**16) {
coord_resolutions[i] = 16
MyArray = Uint16Array
} else {
// Will not allow more than 4 billion points on a single feature,
// should be fine.
coord_resolutions[i] = 32
MyArray = Uint32Array
}
vertices[i] = MyArray.from(all_vertices.flat(10))
}
const cols = {
"coordinates": pack_binary(coordinates),
"vertices": pack_binary(vertices),
"coord_resolution": arrow.Uint8Vector.from(coord_resolutions),
"pixel_area": arrow.Float32Vector.from(areas),
"centroid_x": arrow.Float32Vector.from(centroids[0]),
"centroid_y": arrow.Float32Vector.from(centroids[1])
}
for (const [k, v] of properties.entries()) {
if (k in cols) {
// silently ignore.
//throw `Duplicate column names--rename ${k} `;
}
cols[k] = arrow.Vector.from({nullable: true, values: v, type: infer_type(v, options.dictionary_threshold)})
}
const named_columns = []
for (const [k, v] of Object.entries(cols)) {
// console.log(k, v)
named_columns.push(arrow.Column.new(k, v))
}
const tab = arrow.Table.new(...named_columns)
secret.bounds = bounds
return tab
}
//arrow_gl_map = feature_collection_to_feather_frame(k500_counties, projection)
function infer_type(array, dictionary_threshold = .75) {
// Certainly reinventing the wheel here--
// determine the most likely type of something based on a number of examples.
// Dictionary threshold: a number between 0 and one. Character strings will be cast
// as a dictionary if the unique values of the array are less than dictionary_threshold
// times as long as the length of all (not null) values.
const seen = new Set()
let strings = 0
let floats = 0
let max_int = 0
for (let el of array) {
if (Math.random() > 200/array.length) {continue} // Only check a subsample for speed. Try
// to get about 200 instances for each row.
if (el === undefined || el === null) {
continue
}
if (typeof(el) === "string") {
strings += 1
seen.add(el)
} else if (typeof(el) === "number") {
if (el % 1 > 0) {
floats += 1
} else if (isFinite(el)) {
max_int = Math.max(Math.abs(el), max_int)
} else {
}
} else {
throw `No behavior defined for type ${typeof(el)}`
}
}
if ( strings > 0 ) {
// moderate overlap
if (seen.length < strings.length * .75) {
return new arrow.Dictionary(new arrow.Utf8(), new arrow.Int32())
} else {
return new arrow.Utf8()
}
}
if (floats > 0) {
return new arrow.Float32()
}
if (Math.abs(max_int) < 2**8) {
return new arrow.Int8()
}
if (Math.abs(max_int) < 2**16) {
return new arrow.Int16()
}
if (Math.abs(max_int) < 2**32) {
return new arrow.Int32()
} else {
return new arrow.Int64()
}
}
function pack_binary(els) {
const { Builder, Binary } = arrow;
const binaryBuilder = Builder.new({
type: new Binary(),
nullValues: [null, undefined],
highWaterMark: 2**16
});
for (let el of els) { binaryBuilder.append(el) }
return binaryBuilder.finish().toVector()
}
earcut = require("earcut")
d3 = require("d3-geo-projection", "d3-fetch", "d3-geo")
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
