GeoParquet on the WebGeoArrow and GeoParquet in deck.glZero-copy Apache Arrow with WebAssembly
Prototyping GeoRust + GeoArrow in WebAssembly
import { geometryColumn } from "@kylebarron/geoarrow-and-geoparquet-in-deck-gl"
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
flatCoordinateArray = {
const flatCoordinateVector = geometryColumn
.getChildAt(0)
.getChildAt(0)
.getChildAt(0);
return flatCoordinateVector.data[0].values;
}
polygonOffsets = geometryColumn.getChildAt(0).data[0].valueOffsets
geomOffsets = geometryColumn.data[0].valueOffsets
areaArray = {
// Copy `flatCoordinateArray` into WebAssembly memory and construct a
// `geoarrow.CoordBuffer` object. In JavaScript, this object is just a
// pointer to the array in the Wasm memory space.
const coordBuffer = geoarrow.CoordBuffer.from_interleaved_coords(
new geoarrow.InterleavedCoordBuffer(flatCoordinateArray)
);
// Construct a `geoarrow.PolygonArray` from the coordinates and offsets
const polygonArray = new geoarrow.PolygonArray(
coordBuffer,
geomOffsets,
polygonOffsets
);
// Reproject the array of polygons from WGS84 to UTM
// ~3000ms on my machine for 1 million buildings.
// Check your browser console for the timing on your machine.
console.time("reproject");
const utmPolygonArray = polygonArray.reproject_rs(
"utm zone=12",
geoarrow.ReprojectDirection.Fwd
);
console.timeEnd("reproject");
// Compute the area
// ~290ms (on full 1M buildings)
console.time("area");
const areaWasmArray = utmPolygonArray.area();
console.timeEnd("area");
// "Export" the array from Wasm memory
const areaFFIArray = areaWasmArray.to_ffi();
// Use arrow-js-ffi to parse the field metadata from Wasm memory
const areaField = arrowJsFFI.parseField(
geoarrowMemory.buffer,
areaFFIArray.field_addr()
);
// Use arrow-js-ffi to copy the array back to JavaScript
// <2ms even for 1M buildings
console.time("parse vector");
const areaArray = arrowJsFFI.parseVector(
geoarrowMemory.buffer,
areaFFIArray.array_addr(),
areaField.type,
true
);
console.timeEnd("parse vector");
// Manually free the memory from WebAssembly to avoid memory leaks
//
// This sucks but WebAssembly does not have a garbage collector,
// so a certain level of manual memory management is required.
//
// Freeing the `arrowJsFFI` array is safe because we passed `copy=true`
// into `parseVector`, so we're no longer using the WebAssembly memory.
//
// `coordinateBuffer.free()` does not need to be run (and will in fact error!)
// because passing the coordinateBuffer into `geoarrow.PolygonArray` takes
// ownership of the coordinateBuffer, so `wasmPolygonArray.free()` also frees
// the coordinate buffer. This is indeed complex but hopefully will get easier
// to use over time.
polygonArray.free();
utmPolygonArray.free();
areaWasmArray.free();
areaFFIArray.free();
return areaArray;
}
areaArray.toArray()
colorScale = {
// Convert from value to RGB values
const lowerBound = 10;
const upperBound = 5000;
const linearScale = d3
.scaleLinear()
.domain([parseFloat(lowerBound), parseFloat(upperBound)])
.range([0, 1]);
// This is a fast vectorized approach: we store all colors in a single Float32Array instead of many small JS buffers
// Indices maps from geometry to vertex index
return (values, indices) => {
const vertexArrayLength = indices[indices.length - 1];
const outputArray = new Float32Array(vertexArrayLength * 3);
let lastIndex = 0;
for (let i = 0; i < values.length; ++i) {
let nextIndex = indices[i + 1];
const value = values[i];
const color = d3.color(d3.interpolateViridis(linearScale(Math.sqrt(value))));
const r = color.r / 255;
const g = color.g / 255;
const b = color.b / 255;
for (let j = lastIndex; j < nextIndex; ++j) {
outputArray[j * 3] = r;
outputArray[j * 3 + 1] = g;
outputArray[j * 3 + 2] = b;
}
lastIndex = nextIndex;
}
return outputArray;
};
}
colorAttribute = {
// On my computer, generating the color scale from the area array takes ~130ms.
// Check your browser console for your timing.
console.time("colorScale");
const result = colorScale(areaArray.toArray(), resolvedPolygonIndices);
console.timeEnd("colorScale");
return result;
}
resolvedPolygonIndices = {
const resolvedIndices = new Int32Array(geomOffsets.length);
for (let i = 0; i < resolvedIndices.length; ++i) {
// Perform the lookup into the polygonOffsets array using the geomOffsets array
resolvedIndices[i] = polygonOffsets[geomOffsets[i]];
}
return resolvedIndices;
}
deckglLayer = {
// Refer to https://deck.gl/docs/api-reference/layers/solid-polygon-layer#use-binary-attributes
const data = {
// Number of geometries
length: geometryColumn.length,
// Indices into coordinateArray where each polygon starts
startIndices: resolvedPolygonIndices,
attributes: {
// Flat coordinates array
getPolygon: { value: flatCoordinateArray, size: 2 },
// Pass in the color values per coordinate vertex
getFillColor: { value: colorAttribute, size: 3 }
}
};
const layer = new deck.SolidPolygonLayer({
// This is an Observable hack - changing the id will force the layer to refresh when the cell reevaluates
id: `layer-${Date.now()}`,
data,
// Skip normalization for binary data
_normalize: false,
// Counter-clockwise winding order
_windingOrder: "CCW"
});
deckglMap.setProps({ layers: [layer] });
return layer;
}
// Load the parquet-wasm library
geoarrowModule = {
const geoarrowModule = await import(
"https://unpkg.com/geoarrow-wasm@0.1.0/esm/index.js"
);
// Need to await the default export first to initialize the WebAssembly code
const { memory } = await geoarrowModule.default();
return [geoarrowModule, memory];
}
geoarrow = geoarrowModule[0]
geoarrowMemory = geoarrowModule[1]
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
