Digital Humanist. Manhattan-based.
Published
Edited
Dec 6, 2021
Importers
12 stars
local_data = {
// Not used; faster, but less flexible than the version below.
return []
const filtered = table.filter(arrow.predicate.col("INSTNM").eq(my_school))
const start = Date.now()
const output = []
for (let row of filtered) {
output.push(row)
}
console.log(`counting took ${Date.now() - start} ms`)
return output
}
selected = aq.table({"INSTNM": my_schools})
local_data4 = {
const start = Date.now()
const output = table
.semijoin(aq.table({"INSTNM": my_schools}))
.groupby("year", "Discipline", "CIPTitle", "Bachelors")
.rollup({degrees: d => op.sum(d.degrees)})
console.log(Date.now() - start, "milliseconds for an arquero filter and summary")
return output
}
md`# Data Loading`
buff = fetch("https://benschmidt.org/degrees.feather").then(d => d.arrayBuffer())
function bless_getter_with_cache(col) {
if (!col.dictionary) {return}
if (!col._materialized_dictionary) {
col._materialized_dictionary = []
}
col.get = function(i) {
const ix = col.indices.get(i);
if (ix === null) {return ix}
return col._materialized_dictionary[ix] ?
col._materialized_dictionary[ix] :
col._materialized_dictionary[ix] = col.dictionary.get(ix)
}
}
unpack_arrow_to_arquero = function(table) {
const cols = table.schema.fields.map(d => d.name)
const data = {}
for (const col of cols) {
const d = table.getColumn(col)
if (d.dictionary) {
bless_getter_with_cache(d);
data[col] = d;
} else {
data[col] = d
}
}
return aq.table(data)
}
table.column("INSTNM").indices.get(100)
table = unpack_arrow_to_arquero(arrow_table)
function unpack_dictionary_column(table, col_name) {
// A helper function
const g = table.getColumn(col_name)
const codes = new Array(g.length)
const ks = g.chunks[g.chunks.length - 1].dictionary.toArray()
let i = 0;
for (const chunk of g.chunks) {
for (const ix of chunk.data.values) {
codes[i] = ks[ix]
i++
}
}
return codes
}
arrow_table = arrow.Table.from(buff)
school_info = d3.csv("https://benschmidt.org/hd2017.csv")
school_table = aq.from(school_info)
import {autoSelect, select, checkbox, text} from "@jashkenas/inputs"
import { aq, op } from '@uwdata/arquero'
d3 = require('d3@v5', 'd3-array')
arrow = require("apache-arrow@0.17")
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
