DSP Engineer turned Data Scientist @ccri
arrow = require('apache-arrow')
crimes = {
let response = await fetch(dataUrl);
let dataTable = await response.arrayBuffer().then(buffer => {
return arrow.Table.from(new Uint8Array(buffer));
});
return dataTable;
}
rowCount = crimes.count()
fields = crimes.schema.fields.map(f => f.name)
fieldTypes = crimes.schema.fields.map(f => f.type)
fieldTypeNames = crimes.schema.fields.map(f => f.type.toString())
firstRow = crimes.get(0).toString() // 1st row data
lastRow = crimes.get(rowCount-1).toString() // last row data
randomRow = crimes.get(Math.random() * rowCount)
toJSON = crimes.get(0).toJSON()
every10KRow = range(0, rowCount, 10000)
function range(start, end, step) {
let slice = [];
for (let i=start; i<end && i <rowCount; i+= step) {
slice.push(crimes.get(i).toArray());
}
return slice;
}
md`${getMarkdown(every10KRow)}`
function getMarkdown (dataFrame) {
let markdown = `${fields.join(' | ')}\n --- | --- | ---`; // header row
let i=0;
for (let row of dataFrame) {
markdown += '\n ';
let td = '';
let k = 0;
for (let cell of row) {
if ( Array.isArray(cell) ) {
td = '[' + cell.map((value) => value == null ? 'null' : value).join(', ') + ']';
} else if (fields[k] === 'Date') {
td = toDate(cell); // convert Apache arrow Timestamp to Date
} else {
td = cell.toString();
}
markdown += ` ${td} |`;
k++;
}
}
return markdown;
}
function toDate(timestamp) {
// Appache Arrow Timestamp is a 64-bit int of milliseconds since the epoch,
// represented as two 32-bit ints in JS to preserve precision.
// The fist number is the "low" int and the second number is the "high" int.
return new Date((timestamp[1] * Math.pow(2, 32) + timestamp[0])/1000);
}
latitude = columnStats('Latitude')
longitude = columnStats('Longitude')
function columnStats(columnName) {
const column = crimes.getColumn(columnName);
let max = column.get(0);
let min = max;
for (let value of column) {
if (value > max) {
max = value;
}
else if (value < min) {
min = value;
}
}
return {min, max, range: max-min};
}
function dateStats(columnName) {
const column = crimes.getColumn(columnName);
let max = toDate(column.get(0));
let min = max;
for (let value of column) {
let date = toDate(value);
if (date > max) {
max = date;
}
else if (date < min) {
min = date;
}
}
return {min, max, range: max-min};
}
timestampVector = crimes.getColumn('Date')
timestamps = {
let dates = []
let i = 0;
for (const timestamp of timestampVector.asEpochMilliseconds()) {
dates.push(new Date(timestamp));
if (++i >= 10) {
break;
}
}
return dates;
}
startDate = new Date(dates.max - 2*hour)
endDate = new Date(dates.max - 1*hour)
crimesIn1Hour = filterByDate(startDate, endDate)
function filterByDate(startDate, endDate) {
let lat, long, date, results = [];
const dateFilter = arrow.predicate.custom(i => {
let date = toDate(crimes.getColumn('Date').get(i));
return date >= startDate && date <= endDate;
}, b => 1);
crimes.filter(dateFilter)
.scan((index) => {
results.push({
'lat': lat(index),
'long': long(index),
'date': toDate(date(index))
});
}, (batch) => {
lat = arrow.predicate.col('Latitude').bind(batch);
long = arrow.predicate.col('Longitude').bind(batch);
date = arrow.predicate.col('Date').bind(batch);
}
);
return results;
}
days = Math.floor(dates.range / millisPerDay)
crimesInDays = filterByDate(startDateFilter, endDateFilter)
function scaleX(long) {
return (long - longitude.min) * (mapWidth/longitude.range);
}
function scaleY(lat) {
return mapHeight - (lat - latitude.min) * (mapHeight/latitude.range);
}
mappedCrimes = {
const canvas = DOM.canvas(mapWidth, mapHeight);
const context = canvas.getContext("2d");
context.fillStyle = '#333333';
let lat, long, x, y, count = 0;
const startDate = new Date(dates.min.getTime() + startMapDaySlider*millisPerDay);
const endDate = new Date(dates.min.getTime() + endMapDaySlider*millisPerDay);
const dateFilter = arrow.predicate.custom(i => {
let date = toDate(crimes.getColumn('Date').get(i));
return date >= startDate && date <= endDate;
}, b => 1);
crimes.filter(dateFilter)
.scan((index) => {
x = scaleX(long(index));
y = scaleY(lat(index));
context.fillRect(x, y, 1, 1); // rectWidth, rectHeight);
count++;
}, (batch) => {
lat = arrow.predicate.col('Latitude').bind(batch);
long = arrow.predicate.col('Longitude').bind(batch);
}
);
return {canvas, count};
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
