2c:null 2d:["$","title",null,{"children":"Matching OpenStreetmap District Electoral Division boundaries with 1911 Census data in Cantabular | Mike Thompson | Observable"}] 32:T8be,// From https://observablehq.com/@mbostock/mapbox-fly-to viewof map = { let container = html`

`; let resolve; container.value = new Promise(_ => resolve = _); // Give the container dimensions. yield container; let map = new mapboxgl.Map({ container, style: {version: 8, sources: {},layers: [ { id: 'background', type: 'background', paint: { 'background-color': '#a3d4ff' } } ]}, bounds: [[-10.65073, 51.4453], [-5.86564, 55.3846]], minZoom: 6, maxZoom: 18, }).addControl(new mapboxgl.NavigationControl(), "top-right"); resolve(map); invalidation.then(() => map.remove()); map.on('load', () => { map.addSource('osm-ded', { type: 'geojson', data: dedsGeojson, }); map.addLayer({ id: 'osm-ded-fill', type: 'fill', source: 'osm-ded', layout: {}, paint: { 'fill-color': '#ddd', 'fill-outline-color': 'black', }, }); map.addSource('ctb-ded', { type: 'geojson', data: featuresWithBestMatch, }); map.addLayer({ id: 'ctb-ded-fill', type: 'fill', source: 'ctb-ded', layout: {}, paint: { 'fill-color': [ 'match', ['get', 'matchType'], 'levenshtein', 'orange', 'manual', 'purple', '#94d1a2', ], 'fill-outline-color': 'black', }, }); map.on('click', (e) => { const ff = map.queryRenderedFeatures(e.point, { layers: ['ctb-ded-fill', 'osm-ded-fill'] }); if (ff.length === 0) return; // Set the popup title based on the type of match let title = 'No match'; if (ff[0].layer.id === 'ctb-ded-fill') { if (ff[0].properties.matchType.startsWith('exact')) { title = 'Exact match'; } else if (ff[0].properties.matchType === 'levenshtein') { title = 'Fuzzy match'; } else { title = 'Manual match'; } } const desc = makePopupDescription(ff[0], title); new mapboxgl.Popup() .setLngLat(e.lngLat) .setHTML(desc) .addTo(map); }); }); }33:T4a6,dedsWithMatches = { const manualMatches = await FileAttachment("manual-ded-matches-2024-02-29.csv").csv(); const codeToMatch = d3.group(manualMatches, (d) => d.code); return dedsWithFuzzyMatches.map((d) => { // Skip if we already have matches if (d.matches.length > 0) return d; // Skip if no manual match if (!codeToMatch.has(d.code)) return d; const match = codeToMatch.get(d.code)[0]; // Skip if no URL if (!match.url) return d; const osmId = match.url.match(/[^/]+$/)[0]; // Skip if no OSM ID found if (osmId === null) return d; dedsGeojson.features.forEach((f) => { if (f.properties.id === parseInt(osmId)) { if (f.properties.cantabularCode?.length) { console.log('Warning: already matched'); console.log('- ', f.properties.id, f.properties.name, f.properties.normalisedLabel); console.log('- ', f.properties.cantabularCode, f.properties.cantabularLabel, f.properties.matchType); } f.properties.matchType = 'manual'; f.properties.cantabularCode = d.code; f.properties.cantabularLabel = d.label; d.matches.push(f); } }); return d; }); }34:[{"id":"efacec62bb1791ef1420223c782904ce4c326f826e05112a2f271befa85315a2a7bb5d0ed951924e1a1672a49d2a8de55da536851b0779be52e16d1464e23ebc","href":"https://static.observableusercontent.com/files/efacec62bb1791ef1420223c782904ce4c326f826e05112a2f271befa85315a2a7bb5d0ed951924e1a1672a49d2a8de55da536851b0779be52e16d1464e23ebc","name":"manual-ded-matches-2024-02-26.csv","type":"text/csv","lastModified":1708949624187.733,"size":9169},{"id":"873b1d2d07f339d889901c25328321fa79731e7bf5d2f408c9afb2f4f6b2f39c385bf102c3d419229b8970798b50e742a1195f8d6154a34720770b529d816904","href":"https://static.observableusercontent.com/files/873b1d2d07f339d889901c25328321fa79731e7bf5d2f408c9afb2f4f6b2f39c385bf102c3d419229b8970798b50e742a1195f8d6154a34720770b529d816904","name":"manual-ded-matches-2024-02-26@1.csv","type":"text/csv","lastModified":1708949802618.949,"size":9201},{"id":"f32703b3309df54c45d23ba9d5e18d3b89a053507e814c108ea2e811e8d4ef0684501e1b105e4eff74c2a8f5c4c76015d60c59754038a6c7ae733dea0bff2588","href":"https://static.observableusercontent.com/files/f32703b3309df54c45d23ba9d5e18d3b89a053507e814c108ea2e811e8d4ef0684501e1b105e4eff74c2a8f5c4c76015d60c59754038a6c7ae733dea0bff2588","name":"manual-ded-matches-2024-02-26@2.csv","type":"text/csv","lastModified":1708952630731.573,"size":9547},{"id":"963a12201a68cdd784722bdd792b802ce6fcc5f00d6e77b51dc3ffcb0bb19dbc531d12ebeb03f7e4266d17b12c05e19c08b4205b0e4d9a16f06849b2e45eb0b6","href":"https://static.observableusercontent.com/files/963a12201a68cdd784722bdd792b802ce6fcc5f00d6e77b51dc3ffcb0bb19dbc531d12ebeb03f7e4266d17b12c05e19c08b4205b0e4d9a16f06849b2e45eb0b6","name":"manual-ded-matches-2024-02-27.csv","type":"text/csv","lastModified":1709028727251.862,"size":12449},{"id":"db8f240c070edefa649bbf4c5e4e992b7c977048fc6280719586fb77cff77e7955cdce5d8b6b582448f535b86cb7385c03d1034b07157a901a86481942e3a731","href":"https://static.observableusercontent.com/files/db8f240c070edefa649bbf4c5e4e992b7c977048fc6280719586fb77cff77e7955cdce5d8b6b582448f535b86cb7385c03d1034b07157a901a86481942e3a731","name":"deds-simplified.geojson","type":"application/geo+json","lastModified":1708359756685.298,"size":10701898},{"id":"a6bf0bf2d8283fc9be7b86ecb544627262faa5e7625fc357df8053b58d02d492e9df34b770bab82414322d1cd04c948c91a71e1f7e0b6b2a2f30eff60c8fe2c1","href":"https://static.observableusercontent.com/files/a6bf0bf2d8283fc9be7b86ecb544627262faa5e7625fc357df8053b58d02d492e9df34b770bab82414322d1cd04c948c91a71e1f7e0b6b2a2f30eff60c8fe2c1","name":"deds-simplified-proj.geojson","type":"application/geo+json","lastModified":1708430476190.084,"size":16616594},{"id":"dc2c9a099dea5efbff9788a5112feff9b20b074c1bdfcf910ef7a605081d832533eb3b50a07df2ad35624aa0cc3ba98184e8d851f4b8810a8897b8b4e63806bb","href":"https://static.observableusercontent.com/files/dc2c9a099dea5efbff9788a5112feff9b20b074c1bdfcf910ef7a605081d832533eb3b50a07df2ad35624aa0cc3ba98184e8d851f4b8810a8897b8b4e63806bb","name":"manual-ded-matches-2024-02-29.csv","type":"text/csv","lastModified":1709227644421.658,"size":11753}] 30:{"id":"f6bb506b202f0e0b","slug":"matching-openstreetmap-ded-boundaries","title":"Matching OpenStreetmap District Electoral Division boundaries with 1911 Census data in Cantabular","thumbnail":"4df578909ad56945b7e6a5848fe289ea5c42d29a090ced8ff3caa0aca86f2825","access_level":"public","create_time":"$D2024-02-19T16:16:57.823Z","update_time":"$D2024-03-04T10:00:43.346Z","view_time":null,"first_public_time":"$D2024-02-22T12:14:41.613Z","has_unpublished_edits":null,"fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"34b3e592976f9cd2","login":"mikejamesthompson","name":"Mike Thompson","avatar_url":"https://avatars-next.observableusercontent.com/57e8ff24c2d3843f76fd089c7a740c6b9315ea6136523fb1f5b6f5567e4470c9","tier":"starter_2024"},"version":562,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"775820c39ba45082","login":"mikejamesthompson","name":"Mike Thompson","avatar_url":"https://avatars-next.observableusercontent.com/57e8ff24c2d3843f76fd089c7a740c6b9315ea6136523fb1f5b6f5567e4470c9","tier":null}],"members":[{"type":"team","id":"34b3e592976f9cd2","login":"mikejamesthompson","name":"Mike Thompson","avatar_url":"https://avatars-next.observableusercontent.com/57e8ff24c2d3843f76fd089c7a740c6b9315ea6136523fb1f5b6f5567e4470c9","tier":"starter_2024","role":"editor"},{"type":"individual","id":"775820c39ba45082","login":"mikejamesthompson","name":"Mike Thompson","avatar_url":"https://avatars-next.observableusercontent.com/57e8ff24c2d3843f76fd089c7a740c6b9315ea6136523fb1f5b6f5567e4470c9","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Matching OpenStreetmap District Electoral Division boundaries with 1911 Census data in Cantabular","cells":[{"id":0,"value":"# Matching OpenStreetmap District Electoral Division boundaries with 1911 Census data in Cantabular","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":305,"value":"A notebook to match up historic District Electoral Divisions (DEDs) from the [1911 Ireland census data](https://ireland-census-preview.cantabular.com) with DED boundaries [extracted from OpenStreetmap](https://github.com/mikejamesthompson/ireland-osm-boundaries/).","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":203,"value":"## Sources\n\n- Boundary data comes from OpenStreetmap extracted with code from this [GitHub repository](https://github.com/mikejamesthompson/ireland-osm-boundaries)\n- 1911 census data comes from the 1911 Ireland Census [republished with Cantabular](https://ireland-census-preview.cantabular.com/)","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":132,"value":"## Analysis of matches\n\nWe first look for an exact match of normalised names between the two data sources, within a specific county. For Cantabular DEDs that do not have a match after this first step, we then use a fuzzy matching approach to identify potential matches, again within the same county. Finally, we use a manually-compiled CSV file to match specific Cantabular DEDs to specific OSM features. These approaches are described in more detail below.\n\nAfter potential best matches have been identified, the `selectBestMatch` function, described below, is used to select between them.\n\nFor example:\n\n- the OSM DED \"Aghagallon DED\" in Antrim will have a normalised label created of \"aghagallon\". The 1911 census DED of \"Antrim, Aghagallon\" will be normalised to \"aghagallon\" and the two will match.\n- the OSM DED \"Jerpointchurch\" in Kilkenny will be fuzzy-matched to the 1911 census DED of \"Kilkenny, Jerpoint Church\".","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":157,"value":"html`\n

Breakdown of matches

${Math.round(100 * featuresWithBestMatch.features.length / deds.length)}% have at least one match.

${c.label}: ${c.value}

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":141,"value":"analysisOfMatches = {\n const counts = {};\n dedsWithMatches.forEach((d) => {\n const c = d.matches.length;\n if (counts[c]) {\n counts[c] += 1;\n } else {\n counts[c] = 1;\n }\n });\n return Object.values(counts).map((c, i) => {\n return {\n label: i === 1 ? '1 match' : `${i} matches`,\n value: c,\n };\n });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":534,"value":"## Download final GeoJSON file\n\nGet a GeoJSON file containing all best-match OSM features, each with only a single properties field—`code`—that corresponds to the feature's matched Cantabular DED category code.\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":539,"value":"button = {\n featuresWithBestMatch.features.forEach((f) => filterFeatureProperties(f));\n const downloadData = new Blob([JSON.stringify(featuresWithBestMatch)], {\n type: \"application/json\"\n });\n const size = (downloadData.size / 1024).toFixed(0);\n return DOM.download(\n downloadData,\n 'county_district_geoid.geojson',\n `Download GeoJSON (~${size} KB)`\n );\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":523,"value":"function filterFeatureProperties(feature) {\n feature.properties.code = feature.properties.cantabularCode;\n const keys = Object.keys(feature.properties);\n keys.forEach((k) => {\n if (k === 'code') return;\n // delete feature.properties[k];\n });\n return feature;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":134,"value":"## Download DEDs with no matches\n\nThis CSV lists DEDs in Cantabular with no corresponding match in OSM.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":71,"value":"DOM.download(serializeToCSV(dedsWithNoMatches), \"1911-deds-with-no-osm-match\", \"Download CSV\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":90,"value":"function serializeToCSV(data) {\n const s = data.map((d) => d.code + \",\" + d.county + \",\\\"\" + d.label + \"\\\"\\n\");\n s.unshift(\"Cantabular code,county,DED name\\n\")\n return new Blob(s, {type: \"text/csv\"}) \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":64,"value":"dedsWithNoMatches = dedsWithMatches.filter((d) => d.matches.length == 0);","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":184,"value":"## Map of DEDs\n\nThe map below shows successful exact matches (in green), fuzzy matches (in orange) and manual matches (in purple) overlayed over all OSM DED data (in grey).\n\nClick a feature to see its metadata in OSM supplemented with the matched Cantabular code and label where relevant.\n\n- **Green**: OSM DED with an exact normalised name match to a Cantabular DED\n- **Orange**: OSM DED with a fuzzy match (Levenshtein distance of 1) to a Cantabular DED\n- **Purple**: OSM DED with a manual match to a Cantabular DED\n- **Grey**: OSM DED with no best match in Cantabular","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":181,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":269,"value":"function makePopupDescription(feature, title) {\n const props = Object.entries(feature.properties);\n const description = html`\n

${title}

${key}: ${value}

`;\n return description.outerHTML;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":259,"value":"html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":416,"value":"### Create GeoJSON data structure to load into map\n\nUsing the matches identified below, we build a GeoJSON data structure to load on the map, selecting a single best match for each DED using the `selectBestMatch` function. This function prefers OSM features that mention \"1911\" or \"1901\" in their date property over any other feature.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":177,"value":"featuresWithBestMatch = {\n const fc = {\n type: \"FeatureCollection\",\n features: [],\n };\n dedsWithMatches.forEach((d) => {\n const match = selectBestMatch(d);\n if (match) fc.features.push(match);\n });\n return fc;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":187,"value":"function selectBestMatch(ded) {\n if (ded.matches.length === 1) {\n return ded.matches[0];\n }\n for (let m of ded.matches) {\n if (m.properties.date === '1911' || m.properties.source === 'Census of Ireland 1911 online') {\n return m;\n } else if(m.properties.date === '1901') {\n return m;\n }\n }\n return ded.matches[0];\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":431,"value":"## Name matching","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":403,"value":"### Attempt exact name matching of Cantabular DEDs to OSM DED features\n\nFor each Cantabular DED, we look through all OSM DEDs in the same county and compare the values of their precomputed normalised labels to try and identify an exact match.\n\nIf the default OSM `name` field doesn't match, then we also attempt a match using the `name:en` field and the `alt_name` field. This is because some OSM contributors will but the Gaelic name for the DED in the `name` field and put the English name in one of the two other fields. The 1911 census data loaded into Cantabular always uses the English name.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":36,"value":"dedsWithExactMatches = {\n return deds.map((d) => {\n d.matches = [];\n dedsGeojson.features.forEach((f) => {\n if (d.county !== f.properties.county) return;\n let match = false;\n if (d.normalisedLabel === f.properties.normalisedLabel) {\n f.properties.matchType = \"exact,name\";\n match = true;\n } else if (d.normalisedLabel === normaliseLabel(f.properties['name:en'])) {\n f.properties.matchType = \"exact,name:en\";\n match = true;\n } else if (d.normalisedLabel === normaliseLabel(f.properties['alt_name'])) {\n f.properties.matchType = \"exact,alt_name\";\n match = true;\n }\n if (match) {\n f.properties.cantabularCode = d.code;\n f.properties.cantabularLabel = d.label;\n d.matches.push(f);\n }\n });\n return d;\n });\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":405,"value":"### Attempt fuzzy name matching of unmatched Cantabular DEDs to OSM DED features\n\nAfter looking for exact name matches, we now look for fuzzy matches. For each remaining Cantabular DED with no matches to an OSM feature, we again look through all OSM DEDs that have no matches and use fuzzy matching to try and identify a good match.\n\nSpecifically, we calculate the [Levenshtein distance](https://en.wikipedia.org/wiki/Levenshtein_distance) between the two normalised labels and accept as a match any Cantabular DED-OSM feature pair where the distance is 1.\n\nThis successfully matches an additional 5% of DEDs as many of the DED names have one-character differences between the two sources that are down to mistakes made in transcription (e.g. \"Cork, Meena\" instead of \"Cork, Meens\"), differences in spacing (e.g. \"Jerpoint Church\" vs \"Jerpointchurch\") and alternate spellings of the same name being in common use.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":343,"value":"dedsWithFuzzyMatches = {\n return dedsWithExactMatches.map((d) => {\n // Don't add fuzzy matches if we already have exact matches\n if (d.matches.length > 0) return d;\n \n // Don't attempt to fuzzy match DEDs with numbers in or we end up with false positives\n // notably things like \"Cork No. 4 Urban\" and \"Cork No. 5 Urban\"\n if (d.label.search(/[0-9]/) > -1) {\n return d;\n }\n dedsGeojson.features.forEach((f) => {\n if (d.county !== f.properties.county) return; // Only match within the same county\n if (f.properties.cantabularCode?.length) return; // Don't match a feature that has already been matched\n const distance = levenshtein(d.normalisedLabel, f.properties.normalisedLabel);\n if (distance === 1) {\n f.properties.matchType = 'levenshtein';\n f.properties.cantabularCode = d.code;\n f.properties.cantabularLabel = d.label;\n d.matches.push(f);\n }\n });\n return d;\n });\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":442,"value":"### Use CSV of manual matches to improve accuracy\n\nFinally we use a manually compiled CSV that maps remaining unmatched Cantabular DEDs to the correct OSM features.\n\nThis was put together by our team by using the CSV download of unmatched DEDs above together with the map above and OSM search.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":444,"value":"$33","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":427,"value":"## Data loading","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":389,"value":"### Load OSM GeoJSON source\n\nWe open the attached GeoJSON file previously extracted ([see source code here](https://github.com/mikejamesthompson/ireland-osm-boundaries)) from OpenStreetmap. We filter out more modern DEDs that are not relevant to us and precompute county and normalised label fields.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":17,"value":"dedsGeojson = {\n const json = await FileAttachment(\"deds-simplified.geojson\").json();\n // Loop through the features backwards so we can remove ones we're not interested in\n for (let i = json.features.length - 1; i >= 0; i--) {\n const f = json.features[i];\n if (parseInt(f.properties.date) > 1970) json.features.splice(i, 1);\n f.properties.county = translateCountyName(f.properties.county.replace(/^County/, '').trim());\n f.properties.normalisedLabel = normaliseLabel(f.properties.name);\n }\n return json;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":386,"value":"### Fetch DED data from Cantabular\n\nFetch all of the DED data from the 1911 dataset in Cantabular. We supplement the code and label for each area with a precomputed normalised label (with the county name prefix removed) and a county name extracted from the label prefix into a separate field.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":11,"value":"deds = {\n const response = await queryCantabularGraphQL(ENDPOINT, CATEGORIES_QUERY, {\n \"dataset\": DATASET,\n \"variable\": \"county_district_geoid\",\n });\n let categoryEdges = response.data.dataset.variables.edges[0].node.categories.edges;\n return categoryEdges.map((e) => {\n // Remove the county name from each DED label for the normalised label\n const nl = e.node.label.replace(/^[^,]+, /, '');\n return {\n label: e.node.label,\n normalisedLabel: normaliseLabel(nl),\n code: e.node.code,\n county: e.node.label.match(/^[^,]+/)[0],\n };\n });\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":429,"value":"## Utility functions, imports and constants","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":377,"value":"### Function to translate Laois and Offaly names to 1911 equivalents\n\nThe 1911 data in Cantabular uses the anachronistic \"Queen's Co.\" and \"King's Co.\" names for Laois and Offaly respectively. This function translates the names in OSM to the 1911 equivalents to facilitate matching.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":15,"value":"function translateCountyName(name) {\n if (name === \"Laois\") return \"Queen's Co.\";\n if (name === \"Offaly\") return \"King's Co.\";\n return name;\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":381,"value":"### Function to normalise DED names in both sources before matching\n\nDED names in OSM often have a suffix denoting the boundary type or source year that we need to remove before matching. We also remove punctuation and repeated whitespace characters.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":13,"value":"function normaliseLabel(n) {\n if (!n) return '';\n const replacements = [\n { search: / [0-9]{4}$/, replace: '' }, // Some OSM DEDs have a year suffix, e.g. \"1911\" or \"1986\"\n { search: / District Electoral Division$/i, replace: '' },\n { search: / Electoral Division$/i, replace: '' },\n { search: / DED$/, replace: ''},\n { search: / ED$/, replace: ''},\n { search: /[.,'\\/#!$%\\^&\\*;:{}=\\-_`~()]/g, replace: ' ' },\n { search: /\\s{2,}/g, replace: ' ' },\n ];\n replacements.forEach((r) => n = n.replace(r.search, r.replace).trim());\n return n.toLowerCase();\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":379,"value":"### Imports and constants\n\nThese functions, libraries and constants are used by the rest of the notebook.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":9,"value":"DATASET = \"Ireland-1911-Preview\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":7,"value":"ENDPOINT = \"https://api.ireland-census-preview.cantabular.com/graphql\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":214,"value":"mapboxgl = {\n const gl = await require(\"mapbox-gl@3.1.2\");\n if (!gl.accessToken) {\n gl.accessToken = ctb();\n const href = await require.resolve(\"mapbox-gl@3.1.0/dist/mapbox-gl.css\");\n document.head.appendChild(html``);\n }\n return gl;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":206,"value":"function ctb() {\n return \"pk.eyJ1Ijoic2Vuc2libGVjb2RlIiwiYSI6ImNsc3VodG93MzB1c3kycHBnbjlyaGtiY3EifQ.KU_Le8Ez-YHCZyfp2MiYXA\"\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":319,"value":"import {levenshtein} from \"800f395ee21cd06a\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"import { CATEGORIES_QUERY, queryCantabularGraphQL } from \"@mikejamesthompson/cantabular-helpers-for-1911-irish-census-data\";","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W34","stdlib":"1"}} 31:"$30:body"