Public
Edited
Feb 1, 2022
4 stars
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
// This is a generic function provding transformations of binary blocks to Json objects.
function compressedBinaryToJson(delimiter = ';') {
return agen.compose(
fromStream(), // Read binary blocks from the input stream
agen.inflate(), // Inflate the compressed stream
agen.decode(), // Decode binary blocks and transform them to text blocks
agen.lines(), // Split text blocks to individual lines
agen.arraysFromDsv(delimiter),// Split lines by the specified delimiter and transforms them to arrays
toJson(), // Transform arrays to JSON objects;
// Note: This function will be added in one of @agen modules
);
}
Insert cell
// This function returns a filter transforming binary blocks to GeoJson objects.
// It takes into account specificities of the attached file:
// - it knows that the delimiter is the ";" symbol
// - it knows also how to transform individual fields to get GeoJson
function compressedBinaryToGeoJson() {
return agen.compose(
compressedBinaryToJson(';'), // Decompress, decode, split lines, transforms lines to arrays...
// ...and arrays to objects

agen.map(obj => { // Transforms simple JSON objects to GeoJson
let geometry;
if (obj.coordonnees_ban) {
geometry = { // Fix geometry - split and parse point coordinates
type : 'Point',
coordinates : obj.coordonnees_ban.split(',').map(v => parseFloat(v))
}
}
return { // Returns the resulting GeoJSON object
id : obj.ref,
type : 'Feature',
properties : obj,
geometry
}
}),
agen.filter(obj => !!obj.geometry), // Filter out objects without geometries
);
}
Insert cell
Insert cell
{
const f = agen.compose(
compressedBinaryToGeoJson(';'), // This transformation chain converts compressed binaries to GeoJSON
agen.batch(20) // This step packs returned objects in batches of 20 objects
);
for await (let batch of f(await openFileStream())) {
yield batch;
break; // Shows just the first batch
}
}
Insert cell
async function* readData(stream, delimiter = ';') {
const f = compressedBinaryToGeoJson(delimiter);
yield* f(stream);
}
Insert cell
async function openFileStream() {
// 44988 entries
return await FileAttachment("liste-des-immeubles-proteges-au-titre-des-monuments-historiques.csv.gz").stream()
}
Insert cell
Insert cell
visualBatchSize = 1000
Insert cell
loadedObjects = {
await start;

const map = newMap(mapContainer);
invalidation.then(() => map.remove());

// This transformation chain converts compressed binaries to GeoJSON
const f = compressedBinaryToGeoJson(';');

let counter = 0;
const bufferSize = visualBatchSize;
const N = 10; // Math.round(Math.min(20, visualBatchSize / 4));
const buffer = [];

/* Create a heatmap layer and add an utility method to add coordinates in batch */
// For colors see the "ColorHunt" section here: https://observablehq.com/@makio135/give-me-colors
// ['#6a2c70', '#b83b5e', '#f08a5d', '#f9ed69']
const heat = L.heatLayer([], {
gradient: {0.3: '#6a2c70', 0.5 : '#b83b5e', 0.75: '#f08a5d', 1: '#f9ed69'}
}).addTo(map);
heat.setLatLngs = function (list) {
this._latlngs = [...list];
return this.redraw();
}
invalidation.then(() => heat.remove());

// Group of markers
const group = L.featureGroup().addTo(map);
invalidation.then(() => group.remove());

for await (let feature of f(await openFileStream())) {
const latlng = feature.geometry.coordinates;
const marker = L.circleMarker(latlng, {
stroke : true,
color: 'white',
opacity : 0.9,
weight : 1,
fillColor: 'red',
fillOpacity: 0.8,
radius: 3
})
group.addLayer(marker);
buffer.push(marker);
while (buffer.length > bufferSize) {
const toRemove = buffer.shift();
group.removeLayer(toRemove);
}
counter++;
if ((counter % N) !== 0) continue;
yield {
total : counter,
visible : [counter - buffer.length, counter]
}

let center = { lat : 0, lng : 0};
const coords = [];
let count = 0;
group.eachLayer(layer => {
const { lat, lng } = layer.getLatLng();
coords.push([lat, lng]);
center.lat += lat;
center.lng += lng;
count++;
});
if (count) {
center.lat /= count;
center.lng /= count;
map.panTo(center);
}

heat.setLatLngs(coords);
await Promises.delay(100);
}
yield {
total : counter,
visible : [counter - buffer.length, counter]
}
}
Insert cell
Insert cell
function toJson() {
return async function*(it) {
let header;
for await (let array of it) {
if (!header) {
header = array;
} else {
const obj = {};
for (let i =0; i < header.length; i++) {
obj[header[i]] = array[i];
}
yield obj;
}
}
}
}

Insert cell
// This method returns a function transforming readable stream to AsyncGenerator
function fromStream() {
return async function*(stream) {
const reader = stream.getReader();
try {
let chunk;
while ((chunk = await reader.read()) && !chunk.done) {
yield chunk.value;
}
} finally {
reader.cancel && reader.cancel();
stream.cancel && stream.cancel();
}
}
}
Insert cell
md`## Map Utilities`
Insert cell
mapConfig = ({
center : [48.854, 2.348],
zoom : 6,
maxZoom: 10
});
Insert cell
tilesUrl = {
return 'https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png';
// return 'https://maps.wikimedia.org/osm-intl/{z}/{x}/{y}@2x.png';
}
Insert cell
function newMap(mapContainer) {
const map = L.map(mapContainer, mapConfig);
L.tileLayer(tilesUrl, {
attribution: '&copy; <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors'
}).addTo(map);
return map;
}
Insert cell
function* listenMap(map, event, filter =(_)=>_) {
yield* Generators.observe((next) => {
let handler = (ev) => next(filter(ev));
map.on(event, handler);
return () => map.off(event, handler);
})
}
Insert cell
Insert cell
agen = require(
'@agen/utils@0.8', // Common utility methods - composition, mapping, transformation etc
'@agen/gzip', // GZip inflating utilities for streams
'@agen/encoding', // Text decoding, splitting to lines
'@agen/dsv', // Transformation lines to delimited separated values
)
Insert cell
L = {
const L = await require("leaflet");
L.cssUrl = await require.resolve("leaflet/dist/leaflet.css");
document.head.appendChild(html`<link href='${L.cssUrl}' rel='stylesheet' />`);
// See https://github.com/Leaflet/Leaflet.heat
await require("https://leaflet.github.io/Leaflet.heat/dist/leaflet-heat.js").catch(
(err) => {}
);
return L;
}
Insert cell

One platform to build and deploy the best data apps

Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.
Learn more