Digital Humanist. Manhattan-based.
Published
Edited
Dec 6, 2021
Importers
12 stars
local_data = {
// Not used; faster, but less flexible than the version below.
return []
const filtered = table.filter(arrow.predicate.col("INSTNM").eq(my_school))
const start = Date.now()
const output = []
for (let row of filtered) {
output.push(row)
}
console.log(`counting took ${Date.now() - start} ms`)
return output
}
selected = aq.table({"INSTNM": my_schools})
local_data4 = {
const start = Date.now()
const output = table
.semijoin(aq.table({"INSTNM": my_schools}))
.groupby("year", "Discipline", "CIPTitle", "Bachelors")
.rollup({degrees: d => op.sum(d.degrees)})
console.log(Date.now() - start, "milliseconds for an arquero filter and summary")
return output
}
md`# Data Loading`
buff = fetch("https://benschmidt.org/degrees.feather").then(d => d.arrayBuffer())
function bless_getter_with_cache(col) {
if (!col.dictionary) {return}
if (!col._materialized_dictionary) {
col._materialized_dictionary = []
}
col.get = function(i) {
const ix = col.indices.get(i);
if (ix === null) {return ix}
return col._materialized_dictionary[ix] ?
col._materialized_dictionary[ix] :
col._materialized_dictionary[ix] = col.dictionary.get(ix)
}
}
unpack_arrow_to_arquero = function(table) {
const cols = table.schema.fields.map(d => d.name)
const data = {}
for (const col of cols) {
const d = table.getColumn(col)
if (d.dictionary) {
bless_getter_with_cache(d);
data[col] = d;
} else {
data[col] = d
}
}
return aq.table(data)
}
table.column("INSTNM").indices.get(100)
table = unpack_arrow_to_arquero(arrow_table)
function unpack_dictionary_column(table, col_name) {
// A helper function
const g = table.getColumn(col_name)
const codes = new Array(g.length)
const ks = g.chunks[g.chunks.length - 1].dictionary.toArray()
let i = 0;
for (const chunk of g.chunks) {
for (const ix of chunk.data.values) {
codes[i] = ks[ix]
i++
}
}
return codes
}
arrow_table = arrow.Table.from(buff)
school_info = d3.csv("https://benschmidt.org/hd2017.csv")
school_table = aq.from(school_info)
import {autoSelect, select, checkbox, text} from "@jashkenas/inputs"
import { aq, op } from '@uwdata/arquero'
d3 = require('d3@v5', 'd3-array')
arrow = require("apache-arrow@0.17")
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.