I map 🗺️ & graph 📈 data 🈸
arrow = require('apache-arrow@0.3.1')
crimes = loadData(dataUrl).then(buffer => arrow.Table.from(new Uint8Array(buffer)))
async function loadData(dataUrl){
const response = await fetch(dataUrl);
return await response.arrayBuffer();
}
rowCount = crimes.count()
fields = crimes.schema.fields.map(f => f.name)
fieldTypes = crimes.schema.fields.map(f => f.type)
fieldTypeNames = crimes.schema.fields.map(f => f.type.toString())
firstRow = crimes.get(0).toString() // 1st row data
lastRow = crimes.get(rowCount-1).toString() // last row data
randomRow = crimes.get(Math.random() * rowCount)
toJSON = crimes.get(0).toJSON()
toArray = crimes.get(rowCount-1).toArray()
every10KRow = range(crimes, 0, rowCount, 10000)
function range(data, start, end, step) {
const slice = [];
const rowCount = data.count();
for (let i=start; i<end && i <rowCount; i+= step) {
slice.push(data.get(i).toArray());
}
return slice;
}
md`${getMarkdown(every10KRow, fields)}`
function getMarkdown (dataFrame, fields, dateFields = []) {
let markdown = `${fields.join(' | ')}\n --- | --- | ---`; // header row
let i=0;
for (let row of dataFrame) {
markdown += '\n ';
let td = '';
let k = 0;
for (let cell of row) {
if ( Array.isArray(cell) ) {
td = '[' + cell.map((value) => value == null ? 'null' : value).join(', ') + ']';
} else if (fields[k] === 'Date' || dateFields.indexOf(fields[k]) >= 0) {
td = toDate(cell).toLocaleString(); // convert Apache arrow Timestamp to Date and format
} else {
td = cell.toString();
}
markdown += ` ${td} |`;
k++;
}
}
return markdown;
}
function toDate(timestamp) {
// Appache Arrow Timestamp is a 64-bit int of milliseconds since the epoch,
// represented as two 32-bit ints in JS to preserve precision.
// The fist number is the "low" int and the second number is the "high" int.
return new Date((timestamp[1] * Math.pow(2, 32) + timestamp[0])/1000);
}
latitude = columnStats('Latitude')
longitude = columnStats('Longitude')
function columnStats(columnName) {
const column = crimes.getColumn(columnName);
let max = column.get(0);
let min = max;
for (let value of column) {
if (value > max) {
max = value;
}
else if (value < min) {
min = value;
}
}
return {min, max, range: max-min};
}
function dateStats(columnName) {
const column = crimes.getColumn(columnName);
let max = toDate(column.get(0));
let min = max;
for (let value of column) {
const date = toDate(value);
if (date > max) {
max = date;
}
else if (date < min) {
min = date;
}
}
return {min, max, range: max-min};
}
timestampVector = crimes.getColumn('Date')
timestamps = {
const dates = []
let i = 0;
for (const timestamp of timestampVector.asEpochMilliseconds()) {
dates.push(new Date(timestamp));
if (++i >= 10) {
break;
}
}
return dates;
}
millisPerHour = 1*60*60*1000
startDate = new Date(dates.max - 2*millisPerHour)
endDate = new Date(dates.max - 1*millisPerHour)
crimesIn1Hour = filterByDate(startDate, endDate)
function filterByDate(startDate, endDate) {
let lat, long, date, results = [];
const dateFilter = arrow.predicate.custom(i => {
const date = toDate(crimes.getColumn('Date').get(i));
return date >= startDate && date <= endDate;
}, b => 1);
crimes.filter(dateFilter)
.scan((index) => {
results.push({
'lat': lat(index),
'long': long(index),
'date': toDate(date(index))
});
}, (batch) => {
lat = arrow.predicate.col('Latitude').bind(batch);
long = arrow.predicate.col('Longitude').bind(batch);
date = arrow.predicate.col('Date').bind(batch);
}
);
return results;
}
millisPerDay = 24*millisPerHour
days = Math.floor(dates.range / millisPerDay)
crimesInDays = filterByDate(
daySliderToDate(startDaySlider),
daySliderToDate(endDaySlider)
)
function daySliderToDate(daySlider) {
return new Date(dates.min.getTime() + daySlider*millisPerDay);
}
mapWidth = width/2 // half of window width
mapHeight = width * .6
function scaleX(long) {
return (long - longitude.min) * (mapWidth/longitude.range);
}
function scaleY(lat) {
return mapHeight - (lat - latitude.min) * (mapHeight/latitude.range);
}
mappedCrimes = {
const canvas = DOM.canvas(mapWidth, mapHeight);
const context = canvas.getContext('2d');
context.fillStyle = 'steelblue'; //'#333333';
let lat, long, x, y, count = 0;
const startDate = daySliderToDate(startMapDaySlider);
const endDate = daySliderToDate(endMapDaySlider);
const dateFilter = arrow.predicate.custom(i => {
const date = toDate(crimes.getColumn('Date').get(i));
return date >= startDate && date <= endDate;
}, b => 1);
crimes.filter(dateFilter)
.scan((index) => {
x = scaleX(long(index));
y = scaleY(lat(index));
context.fillRect(x, y, 1, 1); // rectWidth, rectHeight);
count++;
}, (batch) => {
lat = arrow.predicate.col('Latitude').bind(batch);
long = arrow.predicate.col('Longitude').bind(batch);
}
);
return {canvas, count};
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
