Public
Edited
Sep 8, 2023
Importers
localForage = require("localforage")
zip = (rows) => rows[0].map((_, c) => rows.map((row) => row[c]))
function chunks(array, chunkSize) {
return Array.from(
{ length: Math.ceil(array.length / chunkSize) },
(_, index) => array.slice(index * chunkSize, (index + 1) * chunkSize)
);
}
class ACSClient {
BASE_URL = "https://census-api.bunkum.us";
constructor(api_key) {
this.api_key = api_key;
}
async definition(year, field) {
const url = this._definition(year, field);
const key = `census-${url}`;
const cached = await localForage.getItem(key);
if (cached !== null) {
return cached;
}
const resp = await fetch(url);
var data;
if (resp.status === 200) {
data = await resp.json();
localForage.setItem(key, data);
} else if (resp.status === 404) {
data = null;
}
return data;
}
async _definitions_fetch(year, headers, fields) {
/*
* if we are getting a group variable, we can significantly
* reduce the number of API calls we need to make to get the definitions
* for all the group items.
*
* this could be made a bit faster by making the variable definition
* requests in parallel
*/
const vars = new Map();
const group_pattern = /group\((.*?)\)/;
for (const field of fields) {
if (group_pattern.test(field)) {
const base_variable = field.match(group_pattern)[1];
const url = this._group_definitions(year, base_variable);
const key = `census-${url}`;
var data;
data = await localForage.getItem(key);
if (data === null) {
const resp = await fetch(url);
data = await resp.json();
localForage.setItem(key, data);
}
for (const [key, value] of Object.entries(data.variables)) {
vars.set(key, value);
}
}
}
for (const header of headers) {
if (!vars.has(header)) {
const definition = await this.definition(year, header);
vars.set(header, definition);
}
}
return vars;
}
_field_type(definition) {
const VAR_TYPES = new Map([
["fips-for", String],
["fips-in", String],
["int", parseInt],
["float", parseFloat],
["string", String]
]);
if (definition && definition.name === "ADJINC") {
return parseFloat;
} else if (definition && definition.predicateType) {
return VAR_TYPES.get(definition.predicateType);
} else {
return String;
}
}
async query(fields, geo, year, filter) {
if (fields.length > 49) {
const variable_pattern = /(B|C)\d{5}_\d{3}[A-Z]{1,2}/;
const vars = fields.filter((field) => variable_pattern.test(field));
const non_vars = fields.filter((field) => !variable_pattern.test(field));
let results = [];
for (const field_chunk of chunks(vars, 49 - non_vars.length)) {
const partial_result = await this.query(
[...non_vars, ...field_chunk],
geo,
year,
filter
);
results = [...results, ...partial_result];
}
return results;
}
const url = this._endpoint(year);
const params = {
get: fields.join(","),
for: geo["for"],
key: this.api_key,
...(filter ? filter : {})
};
if ("in" in geo) {
params.in = geo.in;
}
url.search = new URLSearchParams(params);
const resp = await fetch(url);
if (resp.status === 204) {
return [];
} else {
const data = await resp.json();
return await this._reshape(data, year, fields);
}
}
async _reshape(data, year, fields) {
/*
* we are going to pivot the data into a long format with
* a row for each variable. We are also going to add a
* detailed definition of the variable, cast the variable
* to the right type (usually numeric), add the data
* vintage (year), add dataset source, and add an attribute
* saying what type of variable it is
*/
const headers = data.shift();
const definitions_map = await this._definitions_fetch(
year,
headers,
fields
);
const definitions = headers.map((header) => definitions_map.get(header));
const types = definitions.map((definition) => this._field_type(definition));
const variable_pattern = /(B|C)\d{5}_\d{3}[A-Z]{1,2}/;
const non_vars = new Set(
headers.filter((header) => !variable_pattern.test(header))
);
return data
.map((row) => zip([headers, definitions, types, row]))
.map((row) => {
const common = Object.fromEntries(
row
.filter(([variable, definition, type, value]) =>
non_vars.has(variable)
)
.map(([variable, definition, type, value]) => [variable, value])
);
return row
.filter(
([variable, definition, type, value]) =>
!non_vars.has(variable) && value !== null
)
.map(([variable, definition, type, value]) => ({
...common,
variable,
value: value === null ? value : type(value),
definition,
vintage: year,
dataset: this.dataset,
type: definition.label.split("!!")[0].toLowerCase(),
labels: definition.label.split("!!").slice(1)
}));
})
.flat();
}
_endpoint(vintage) {
return new URL(`/data/${vintage}/acs/${this.dataset}`, this.BASE_URL);
}
_definition(vintage, field) {
return new URL(
`/data/${vintage}/acs/${this.dataset}/variables/${field}.json`,
this.BASE_URL
);
}
_group_definitions(vintage, field) {
return new URL(
`/data/${vintage}/acs/${this.dataset}/groups/${field}.json`,
this.BASE_URL
);
}
}
class ACS5Client extends ACS1Client {
dataset = "acs5";
zcta(fields, zcta, year, filter) {
return this.query(
fields,
{ for: `zip code tabulation area:${zcta}` },
year,
filter
);
}
}
class ACS1Client extends ACSClient {
dataset = "acs1";
state(fields, state, year, filter) {
return this.query(fields, { for: `state:${state}` }, year, filter);
}
state_place(fields, state, place, year, filter) {
return this.query(
fields,
{ in: `state:${state}`, for: `place:${place}` },
year,
filter
);
}
}
class PUMSACS1Client extends ACSClient {
dataset = "acs1/pums";
state(fields, state, year, filter) {
return this.query(fields, { for: `state:${state}` }, year, filter);
}
puma(fields, state, puma, year, filter) {
return this.query(
fields,
{ in: `state:${state}`, for: `public use microdata area:${puma}` },
year,
filter
);
}
async _reshape(data, year, fields) {
const headers = data.shift();
const definitions_map = await this._definitions_fetch(
year,
Array.from(new Set(headers)),
fields
);
const definitions = headers.map((header) => definitions_map.get(header));
const types = definitions.map((definition) => this._field_type(definition));
const filtered_definitions = Array.from(definitions_map.values()).filter(
(d) => d
);
return data.map((row) => {
const d = Object.fromEntries(
zip([headers, types, row]).map(([header, type, value]) => [
header,
type(value)
])
);
d.definitions = filtered_definitions;
return d;
});
}
}
class PUMSACS5Client extends PUMSACS1Client {
dataset = "acs5/pums";
puma(fields, state, puma, year, filter) {
if (year > 2011 && year < 2016) {
throw "puma not supported for this year";
} else {
return this.query(
fields,
{ in: `state:${state}`, for: `public use microdata area:${puma}` },
year,
filter
);
}
}
}
new ACS1Client().query(
["B19131_030E"],
{ in: "state:17", for: "place:14000" },
2021
)
new ACS5Client().state_place(["group(B19131)"], 17, 14000, 2020)
new ACS5Client().zcta(["B01001_030E", "B19131_033E"], 48103, 2021)
new ACS5Client()._definitions_fetch(2021, ["B01001_001E"], ["B01001_001E"])
Type JavaScript, then Shift-Enter. Ctrl-space for more options. Arrow ↑/↓ to switch modes.
missing_kids = new PUMSACS5Client().puma(
["AGEP", "SCH", "SCHL", "PWGTP", "ADJINC"],
"17",
chicago_pumas.join(","),
2016,
{ SCH: 1, AGEP: "5:17", SCHL: "0:15" }
)
d3.sum(missing_kids, (d) => d.PWGTP)
chicago_pumas = [
"03501",
"03502",
"03503",
"03504",
"03520",
"03521",
"03522",
"03523",
"03524",
"03525",
"03526",
"03527",
"03528",
"03529",
"03530",
"03531",
"03532"
]
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
