Published
Edited
Mar 10, 2021
SAVED_SENATE_ID_MAP = await F.waitFor('/idMap')
F = {
const F = new AFileSystem({
idMap: [
FileAttachment("SENATE_ID_MAP.json"),
FileAttachment("SENATE_ID_MAP@1.json"),
FileAttachment("SENATE_ID_MAP@2.json")
],
congress: {
rawMembers: [
new AFile(
'RAW_MEMBERS.json',
() => F.computeRawMembers(),
'application/json'
),
FileAttachment("RAW_MEMBERS.json"),
FileAttachment("RAW_MEMBERS@1.json"),
FileAttachment("RAW_MEMBERS@2.json"),
FileAttachment("RAW_MEMBERS@3.json")
]
},
async computeRawMembers() {}
});
return F;
}
(F.computeRawMembers = async () => {
const all = {};
const dedup = m => (all[m.id] ? m.id : ((all[m.id] = m), m.id));
const m2021 = (await members(117)).map(dedup);
const m2019 = (await members(116)).map(dedup);
const m2017 = (await members(115)).map(dedup);
const m2015 = (await members(114)).map(dedup);
const m2013 = (await members(113)).map(dedup);
const m2011 = (await members(112)).map(dedup);
const m2009 = (await members(111)).map(dedup);
return {
id: all,
2009: m2009,
2011: m2011,
2013: m2013,
2015: m2015,
2017: m2017,
2019: m2019,
2021: m2021
};
})
V = {
const vote = (body, year, number) =>
new AFile(
`roll_${body}_${year}_${number}`,
() => F.readVotes(body, year, number),
'application/json'
);
F.readVotes = async (body, year, roll) => {
const votes = await rollCall(body, year, roll);
return parse_votes(body, year, roll, votes);
};
const votes = body => attached => {
const getYearVotes = year => (fs, path, name, version, rest, tree) => [
vote(body, year, name)
];
const getVotes = (fs, path, year, version, rest, tree) => ({
[FILE]: getYearVotes(year)
});
return {
[DIRECTORY]: getVotes,
...Object.entries(attached).reduce((acc, [year, value]) => {
acc[year] = {
[FILE]: getYearVotes(year),
...Object.entries(value).reduce((yacc, [number, versions]) => {
yacc[number] = [vote(body, year, number), ...versions];
return yacc;
}, {})
};
return acc;
}, {})
};
};
const tree = {
congress: {
house: {
votes: votes('house')({
2021: {
10: [FileAttachment("ROLL_HOUSE_2021_10@2.json")],
11: [FileAttachment("ROLL_HOUSE_2021_11.json")],
14: [FileAttachment("ROLL_HOUSE_2021_14.json")],
16: [FileAttachment("ROLL_HOUSE_2021_16.json")],
17: [FileAttachment("ROLL_HOUSE_2021_17.json")]
},
2020: {
253: [FileAttachment("ROLL_HOUSE_2020_253.json")]
},
2019: {
1: [FileAttachment("ROLL_HOUSE_2019_1.json")]
},
2018: {
500: [FileAttachment("ROLL_HOUSE_2018_500.json")]
},
2017: {
1: [FileAttachment("ROLL_HOUSE_2017_1.json")]
}
})
},
senate: {
votes: votes('senate')({
2021: {
1: [
FileAttachment("ROLL_SENATE_2021_1.json"),
FileAttachment("ROLL_SENATE_2021_1@1.json")
],
2: [FileAttachment("ROLL_SENATE_2021_2@1.json")]
},
2020: {
292: [
FileAttachment("ROLL_SENATE_2020_292.json"),
FileAttachment("ROLL_SENATE_2020_292@1.json")
]
},
2019: {
1: [
FileAttachment("ROLL_SENATE_2019_1.json"),
FileAttachment("ROLL_SENATE_2019_1@1.json")
]
},
2018: {
274: [
FileAttachment("ROLL_SENATE_2018_274.json"),
FileAttachment("ROLL_SENATE_2018_274@1.json")
]
},
2017: {
// With Jeff Sessions
1: [
FileAttachment("ROLL_SENATE_2017_1.json"),
FileAttachment("ROLL_SENATE_2017_1@1.json")
],
// Use this later roll call to pick up Luther Strange
200: [
FileAttachment("ROLL_SENATE_2017_200.json"),
FileAttachment("ROLL_SENATE_2017_200@1.json")
]
},
2016: {
163: [FileAttachment("ROLL_SENATE_2016_163.json")]
},
2015: {
1: [FileAttachment("ROLL_SENATE_2015_1.json")]
},
2014: {
366: [
FileAttachment("ROLL_SENATE_2014_366.json"),
FileAttachment("ROLL_SENATE_2014_366@1.json")
]
},
2013: {
1: [
FileAttachment("ROLL_SENATE_2013_1.json"),
FileAttachment("ROLL_SENATE_2013_1@1.json")
]
}
})
}
}
};
F.tree.congress.house = tree.congress.house;
F.tree.congress.senate = tree.congress.senate;
return F;
}
R292 = V.find('/congress/senate/votes/2012/6')
R292.json()
NICKNAMES = ({
Bob: 'Robert',
Mike: 'Michael',
Jim: 'James',
Jack: 'John',
Bernie: 'Bernard',
Chuck: 'Charles',
Maggie: 'Margaret',
Joe: 'Joseph',
Jacky: 'Jacklyn',
Jay: 'John'
})
getSenateVote = n => {
const voteId = n.select('lis_member_id').text();
const name = n.select('member_full').text();
const last = n.select('last_name').text();
const first = n.select('first_name').text();
const PARTY = n.select('party').text();
const STATE = n.select('state').text();
const state = postalState(STATE);
const vote = n.select('vote_cast').text();
const party = PARTIES[PARTY] || PARTY;
const m =
findMember({ voteId, last, first, state }) ||
Throw(
`Could not find Senator ${last}${
first ? `, ${first}` : ''
} ${state} ${voteId} ${SAVED_SENATE_ID_MAP[voteId]?.id}`
);
return {
id: m.id,
voteId,
last,
first,
party,
...opt({ state }),
district: 'At-large',
body: 'Senate',
vote,
name
};
}
parseMember = c => {
const selectField = field => n => {
const item = n.filter((d, i, c) =>
field.test(
d3
.select(c[i])
.select('strong')
.text()
)
);
return item.empty()
? undefined
: d3opt(item.select('span'))
.text()
.trim();
};
const selectState = selectField(/State:/);
const selectParty = selectField(/Party:/);
const selectDistrictBare = selectField(/District:/);
const selectDistrict = n => selectDistrictBare(n) || 'At Large';
const l = c.select('.result-heading a');
const resultItem = c.selectAll('span.result-item');
const fromTo = formatted => {
const [from, to] = formatted.split('-');
return { from, to, formatted };
};
const service = s =>
s.reduce((acc, v) => {
const [body, years] = v.split(/:\s*/);
const lBody = body.toLowerCase();
acc[lBody] = [
...(acc[lBody] || []),
...years.split(/\s*,\s*/).map(fromTo)
];
return acc;
}, {});
const profile = l.attr('href');
// Parse out the ID from the profile link.
const id = /.*\/(?<ID>[^\/]+)$/.exec(profile).groups.ID;
const state = postalState(selectState(resultItem));
const { first, last: vlast, suffix, body, role, head } = parseHeading(
l.text()
);
const last = fix_name(vlast, state);
const image = d3opt(c.select('.member-image img')).attr('src');
return {
id,
last: fix_name(last, state),
first,
...opt({ suffix }),
body,
state,
role,
district: selectDistrict(resultItem),
party: selectParty(resultItem),
profile: `https://www.congress.gov${profile}`,
...opt3(image, 'image', `https://www.congress.gov${image}`),
served: service(
c
.select('ul.member-served')
.selectAll('li')
//.each((d, i, nodes) => d3.select(nodes[i]).text() || '')
.call(eachDatum((p, c) => c.text()))
.data()
),
head
};
}
members(117)
F.computeRawMembers
findMember({ last: 'Nelson', state: 'NE' })
findMember = {
const findMember = ({ voteId, id = SAVED_SENATE_ID_MAP[voteId]?.id, state, last, first, body}, members = MEMBERS) =>
id
? members.id[id]
: members.name[last]?.find(
m => m &&
(!body || m.body === body) &&
(!state || m.state === state) &&
(m.last === last) &&
(!first || nickMatch(first, m.first) || (first.length == 2 && first.endsWith('.'))))
|| (body && findMember({voteId, id, state, last, first}))
|| Throw(`Member not found: ${JSON.stringify([last, first, state, body, '≠', members.name[last] || []].filter(a => a))}`);
return findMember;
}
t
nickMatch = (name1, name2) => {
const first = name1.split(/\s+/)[0];
const m = name2.split(/\s+/)[0];
return (
!first ||
m.startsWith(first) ||
first.startsWith(m) ||
m.startsWith(NICKNAMES[first]) ||
first.startsWith(NICKNAMES[m])
);
}
byIds = list => {
const ids = {};
list.forEach(m => (ids[m.id] = { ...MEMBERS.id[m.id], ...m }));
return ids;
}
format_table = (group, options = { label: 'Member' }) =>
d3
.create('table')
.classed('members', true)
.classed(options.class || 'members', true)
.call(format_rows(group))
.node()
format_rows = (group, options={}) => n =>
n
.call(n =>
n
.selectAll('tr')
.data(['', options.label, 'Representing', 'Body', 'Votes', 'In Office'])
.join('th')
.text(d => d)
)
.selectAll('tr')
.data(group)
.join('tr')
.call(n => n.classed(n.datum().party.toLowerCase(), true))
.selectAll('td')
.data(d => {
const m = MEMBERS.id[d.id] || Throw(`No member with id ${d.id} found.`);
return [
d,
d,
d.body === 'House' && m.state !== 'MT' && m.state !== 'WY'
? `${m.state}-${m.district}`
: d.state,
d.body,
d.votes?.join(', '),
Object.keys(m.served || {})
.map(
k =>
`${k === d.body.toLowerCase() ? '' : `${k}: `}${m.served[k]
.map(s => s.formatted)
.join(', ')}`
)
.join(', ')
]})
.join('td')
.each((d, i, n) => {
const nn = d3.select(n[i]);
switch (i) {
case 0:
nn.append('img').attr('src', d.image);
break;
case 1:
const m = MEMBERS.id[d.id];
nn.classed('name', true)
.append('a')
.attr('href', m.profile)
.text(`${m.last}, ${m.first}`);
break;
case 2:
nn.classed('state', true).text(d => d);
break;
case 3:
nn.classed('body', true).text(d => d);
case 4:
nn.classed('votes', true).text(d => d);
break;
case 5:
nn.classed('in-office', true).text(d => d);
break;
default:
}
})
regenerator = {
return async function* regenerator(obj) {
let p = Promise.resolve(obj);
obj.updateCount = 0;
while (true) {
p = new Promise((acc, rej) => {
obj.updated = acc;
obj.rejected = rej;
});
yield obj;
await p;
obj.updateCount++;
}
};
}
r = regenerator({ cat: 'dog' })
//r.updated()
style = html`<style>${(await cors_fetch(
`congress/${styleurl}`
)).text()}</style>`
styleurl = d3
.select(hr1html)
.select('link[type="text/css"]')
.attr('href')
.substr(1)
One platform to build and deploy the best data apps
Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.