${'title'}

2c:null 2d:["$","title",null,{"children":"Visualising EU Parliament Votes | Stefan Wehrmeyer | Observable"}] 32:Td95,chart = { const el = d3.select(DOM.element('div')) el.styles({ 'padding': '1rem' }) el .selectAll('h2') .data([{}]).enter() .append('h2') .styles({ 'font-family': 'sans-serif', 'max-width': '100%', }) .text(title) el .selectAll('.container') .data([{}]) .enter() .append('div') .attr('class', 'container') .styles({ 'display': 'flex', 'flex-wrap': 'wrap', 'flex-direction': 'column', 'justify-content': 'flex-start', 'overflow-y': 'scroll', 'max-height': containerHeight + "px", }) let items = el.select('.container') .selectAll('.item') .data(data); let item = items.enter() .append('div') .styles({ 'margin-right': '20px', 'width': '18rem', 'padding': '0 0.2rem 0', 'display': 'flex', }) .styles(d => ({ 'background-color': voteColors[d.vote], // 'margin-top': d.firstInGroup ? '1rem' : '0px' })) // .styles((d) => {'background-color .attr('class', 'item') // item // .append('span') // .text(d => voteLabelMapping[d.vote]) // .styles({ // 'flex': '0 0 10%', // 'max-width': '10%', // }) item.append('span') .text(d => voteLabelMapping[d.vote]) .styles({ 'flex': '0 0 5%', 'width': '20px', 'height': '23px', 'padding': '0.2rem 0.5rem 0.2rem 0.2rem' }) const name = item .append('span') .attr('title', d => `${d.mep.firstName} ${d.mep.lastName}`) .styles({ 'flex': '0 0 50%', 'max-width': '50%', 'font-size': '14px', 'font-family': 'sans-serif', 'text-overflow': 'ellipsis', 'overflow': 'hidden', 'white-space': 'nowrap', 'padding-top': '4px', }) name.append('span') .text(d => d.mep.firstName[0] + '. ') name.append('strong') .text(d => d.mep.lastName) item .append('span') .html(d => d.correction ? '!' : ' ') .attr('title', d => d.correction ? 'Corrected vote' : ' ') .styles({ 'font-size': '12px', 'padding-top': '5px', 'font-family': 'sans-serif', 'flex': '0 0 2%', }) .styles(d => d.correction ? { 'color': voteTextColors[d.correction], 'cursor': 'help', } : {}) item // .filter(d => { // return partyPics[d.mep.countryCode] && partyPics[d.mep.countryCode][d.mep.nationalPoliticalGroupCode] // }) .append('span') .styles({ 'flex': '1 1 30%', 'max-height': '20px', 'object-fit': 'contain', 'padding-top': '4px', 'padding-right': '0.8rem', 'overflow': 'hidden', 'text-overflow': 'ellipsis', 'font-family': 'sans-serif', 'font-size': '14px', 'font-weight': 'bold', 'white-space': 'nowrap', 'text-align': 'right', }) .attr('title', d => `${d.mep.nationalPoliticalGroup} (${d.mep.nationalPoliticalGroupCode}, ${d.mep.country})`) // .attr('alt', d => `${d.mep.nationalPoliticalGroup} (${d.mep.nationalPoliticalGroupCode}, ${d.mep.country})`) // .attr('src', d => partyPics[d.mep.countryCode] && partyPics[d.mep.countryCode][d.mep.nationalPoliticalGroupCode]); .text(d => d.mep.nationalPoliticalGroupCode !== 'Independent' ? d.mep.nationalPoliticalGroupCode : ''); items.exit().remove(); return el.node() }33:T7f0,partyPics = ({ 'DE': { 'CDU': 'https://upload.wikimedia.org/wikipedia/commons/thumb/6/6e/Cdu-logo.svg/120px-Cdu-logo.svg.png', 'CSU': 'https://upload.wikimedia.org/wikipedia/commons/thumb/5/55/Csu_Logo_2016_neu.svg/120px-Csu_Logo_2016_neu.svg.png', 'SPD': 'https://upload.wikimedia.org/wikipedia/commons/thumb/a/a4/SPD_logo.svg/70px-SPD_logo.svg.png', 'GRÜNE.': 'https://upload.wikimedia.org/wikipedia/commons/thumb/4/4b/B%C3%BCndnis_90_-_Die_Gr%C3%BCnen_Logo.svg/100px-B%C3%BCndnis_90_-_Die_Gr%C3%BCnen_Logo.svg.png', 'DIE LINKE.': 'https://upload.wikimedia.org/wikipedia/commons/thumb/4/45/Die_Linke_logo.svg/120px-Die_Linke_logo.svg.png', 'AfD': 'https://upload.wikimedia.org/wikipedia/commons/thumb/b/b8/Alternative-fuer-Deutschland-Logo-2013.svg/100px-Alternative-fuer-Deutschland-Logo-2013.svg.png', 'FDP': 'https://upload.wikimedia.org/wikipedia/commons/thumb/e/e2/Logo_der_Freien_Demokraten.svg/120px-Logo_der_Freien_Demokraten.svg.png' // 'CDU': 'http://lisacharlotterost.de/icons/cdu.png', // 'CSU': 'http://lisacharlotterost.de/icons/csu.png', // 'SPD': 'http://lisacharlotterost.de/icons/spd.png', // 'GRÜNE.': 'http://lisacharlotterost.de/icons/gruene.png', // 'DIE LINKE.': 'http://lisacharlotterost.de/icons/linke.png', // 'AfD': 'http://lisacharlotterost.de/icons/afd.png', // 'FDP': 'http://lisacharlotterost.de/icons/fdp.png', // 'Piratenpartei DE': 'http://lisacharlotterost.de/icons/piraten.png', // 'ÖDP': 'http://lisacharlotterost.de/icons/odp.png', // 'Freie Wähler': 'http://lisacharlotterost.de/icons/freie.png', // 'NPD': 'http://lisacharlotterost.de/icons/npd.png', // 'Die PARTEI': 'http://lisacharlotterost.de/icons/partei.png', // 'Die blaue Partei': 'http://lisacharlotterost.de/icons/blaue.png', // 'LKR': 'http://lisacharlotterost.de/icons/lkr.png', // 'BDNC': 'http://lisacharlotterost.de/icons/buendnisc.png', // 'Independent': 'http://lisacharlotterost.de/icons/none.png', // 'V': 'Volt' } })34:T795,voteToc = { if (voteXML !== null) { const els = Array.from(voteXML.getElementsByTagName('RollCallVote.Description.Text')) let toc = {} els.forEach((e, i) => { toc[i + 1] = {label: e.textContent} }) return toc } const tocStart = 3 let toc = {} let newOnPage let currentVote = 1 let currentVoteFound = false let currentRegex = new RegExp(`^${currentVote}\\.`) let pageNumberRegex = new RegExp(/.*\.{2,}\s*(\d+).*/) let voteLabelRegex = new RegExp(/(.*[^\. ])\s*\.{3,}.*/) for (let pageNumber = tocStart; pageNumber <= doc.numPages; pageNumber++) { console.log('TOC on page', pageNumber) newOnPage = false const page = await doc.getPage(pageNumber); const textContent = await page.getTextContent(); for (let item of textContent.items) { let str = item.str if (str.length === 0) { continue } console.log(pageNumber, str) while (currentRegex.exec(str) !== null || currentVoteFound) { if (!currentVoteFound) { let match = currentRegex.exec(str) console.log(str, match) if (match === null) { break } currentVoteFound = true str = str.substring(match.index + 1) } let voteLabelMatch = voteLabelRegex.exec(str) let pageNumberMatch = pageNumberRegex.exec(str) if (pageNumberMatch === null) { break } console.log(str, pageNumberMatch) toc[currentVote] = { page: parseInt(pageNumberMatch[1], 10), label: voteLabelMatch !== null ? voteLabelMatch[1] : null, tocPage: pageNumber } console.log(`Matched ${currentVote} to page ${toc[currentVote]}`) currentVote += 1 currentRegex = new RegExp(`^${currentVote}\\.`) currentVoteFound = false newOnPage = true } } if (!newOnPage) { break } } return toc }35:T497,voteTexts = { if (voteXML !== null) { return null } let votes = []; let currentVote = 1 mutable downloadProgressMax = doc.numPages mutable downloadProgress = 0 mutable downloadProgressTitle = 'Parsing PDF pages' let text = [] let currentRegex = new RegExp(`^${currentVote}\\.$`) for (let pageNumber = startPage; pageNumber <= doc.numPages; pageNumber++) { console.log('VoteTexts', pageNumber) const page = await doc.getPage(pageNumber); const textContent = await page.getTextContent(); mutable downloadProgress += 1 for (let item of textContent.items) { let str = item.str if (str.length === 0) { continue } if (currentRegex.exec(str) !== null) { console.log('Found', currentVote,' on page', pageNumber, str, 'with', currentRegex) if (currentVote > 1) { votes.push(text.join('')) } text = [] currentVote += 1 currentRegex = new RegExp(`^${currentVote}\\.$`) continue } text.push(`${item.str}\n`);// [${item.transform}]\n\n`; } } votes.push(text.join('')) mutable downloadProgress = doc.numPages return votes }36:T7a9,votes = { function buildIntentionMap(nodes, intention) { const intentions = {} if (nodes.length !== 1) { return intentions } const node = nodes[0] Array.from(node.getElementsByTagName('Member.Name')).forEach(m => { intentions[m.getAttribute('PersId')] = intention }) return intentions } function buildVotes(nodes, vote, intentions) { const node = nodes[0] const votes = [] Array.from(node.getElementsByTagName('Result.PoliticalGroup.List')).forEach(function(group) { const groupId = group.getAttribute('Identifier') Array.from(group.getElementsByTagName('PoliticalGroup.Member.Name')).forEach(function(member) { const persId = member.getAttribute('PersId') if (persId === '197817') { console.log('MICKs', intentions) } votes.push({ name: member.textContent, persId: persId, mepId: member.getAttribute('MepId'), vote: vote, group: groupId, correction: intentions[persId] || null }) }) }) return votes } if (voteXML !== null) { // XML is available return Array.from(voteXML.getElementsByTagName('RollCallVote.Result')).map(vote => { // collect intentions let intentions = { ...buildIntentionMap(vote.getElementsByTagName('Intentions.Result.For'), 'plus'), ...buildIntentionMap(vote.getElementsByTagName('Intentions.Result.Against'), 'minus'), ...buildIntentionMap(vote.getElementsByTagName('Intentions.Result.Abstention'), 'zero'), } console.log('intentions', intentions) return [ ...buildVotes(vote.getElementsByTagName('Result.For'), 'plus', intentions), ...buildVotes(vote.getElementsByTagName('Result.Against'), 'minus', intentions), ...buildVotes(vote.getElementsByTagName('Result.Abstention'), 'zero', intentions) ] }) } return voteTexts.map(makeVote) }37:T68a,function parseVoteText(text) { const plusMatch = plusVotes.exec(text) if (plusMatch === null) { console.error('plus not found', text) return } let plusCount = parseInt(plusMatch[1], 10) text = text.substring(plusMatch.index) const minusMatch = minusVotes.exec(text) if (minusMatch === null) { console.error('minus not found') return } let minusCount = parseInt(minusMatch[1], 10) let plusVoteText = text.substring(plusMatch[0].length, minusMatch.index) text = text.substring(minusMatch.index) const zeroMatch = zeroVotes.exec(text) if (zeroMatch === null) { console.error('zero not found') return } let zeroCount = parseInt(zeroMatch[1], 10) let minusVoteText = text.substring(minusMatch[0].length, zeroMatch.index) text = text.substring(zeroMatch.index) const correctionLabelStartMatch = correctionLabelStart.exec(text) let zeroVoteText let corrections = {plus: '', minus: '', zero: ''} if (correctionLabelStartMatch === null) { zeroVoteText = text.substring(zeroMatch[0].length) } else { zeroVoteText = text.substring(zeroMatch[0].length, correctionLabelStartMatch.index) const correctionLabelEndMatch = correctionLabelEnd.exec(text) if (correctionLabelEndMatch === null) { console.log(text) console.error('correctionLabelEndMatch not found') return } let restText = text.substring(correctionLabelEndMatch.index + correctionLabelEndMatch[0].length) corrections = parseCorrections(restText) } return { plusCount, plus: plusVoteText, minusCount, minus: minusVoteText, zeroCount, zero: zeroVoteText, corrections } }38:[] 30:{"id":"95a6ef59ce75b09b","slug":"parsing-eu-parliament-vote-pdfs","title":"Visualising EU Parliament Votes","thumbnail":"d2d47034303bbee9c257d2bfa143652366f2356b69361c4407c9d318546aace7","access_level":"public","create_time":"$D2019-04-08T10:37:31.694Z","update_time":"$D2022-05-21T09:21:19.539Z","view_time":null,"first_public_time":"$D2019-04-09T09:35:03.574Z","has_unpublished_edits":null,"fork":false,"stars":1,"starred":false,"owner":{"type":"team","id":"9cfe39bcc2ad1e7b","login":"stefanw","name":"Stefan Wehrmeyer","avatar_url":"https://avatars-next.observableusercontent.com/eadb924f6d029e1f508f02cdc376b1caffa73e8e5cbf0b0a0daa7775ed297f6b","tier":"starter_2024"},"version":774,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"51dd5d08c8447531","login":"stefanw","name":"Stefan Wehrmeyer","avatar_url":"https://avatars-next.observableusercontent.com/eadb924f6d029e1f508f02cdc376b1caffa73e8e5cbf0b0a0daa7775ed297f6b","tier":null}],"members":[{"type":"individual","id":"51dd5d08c8447531","login":"stefanw","name":"Stefan Wehrmeyer","avatar_url":"https://avatars-next.observableusercontent.com/eadb924f6d029e1f508f02cdc376b1caffa73e8e5cbf0b0a0daa7775ed297f6b","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Visualising EU Parliament Votes","cells":[{"id":0,"value":"md`# Visualising EU Parliament Votes\n\n**Step 1**: Select the date of [the plenary sitting in the European Parliament](http://www.europarl.europa.eu/plenary/en/votes.html?tab=votes).\n\n**Step 2**: Select correct vote number and adjust other parameters\n\n**Step 3**: Open chart in new window, adjust window size and take screenshot.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":201,"value":"html`

${progressTitle}

${downloadProgressTitle}

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":81,"value":"viewof voteNum = number({min: 1, max: maxVotes, value: 1, title: '2. Vote number'})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":545,"value":"md`Vote name: ${voteToc[voteNum].label}`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":126,"value":"viewof title = text({title: 'Chart title'})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":180,"value":"viewof filterCountry = select({\n title: \"Filter by country\",\n options: [\n { label: \"-\", value: '' },\n { label: \"Germany\", value: \"DE\" },\n ],\n value: \"DE\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":190,"value":"viewof containerHeight = slider({min: 100, max: 4000, value: 725, step: 25, title: 'Chart height'})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":118,"value":"md`### Result`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":128,"value":"viewof openButton = {\n const form = html`