Public
Edited
Mar 4, 2024
featuresWithBestMatch = {
const fc = {
type: "FeatureCollection",
features: [],
};
dedsWithMatches.forEach((d) => {
const match = selectBestMatch(d);
if (match) fc.features.push(match);
});
return fc;
}
dedsWithExactMatches = {
return deds.map((d) => {
d.matches = [];
dedsGeojson.features.forEach((f) => {
if (d.county !== f.properties.county) return;
let match = false;
if (d.normalisedLabel === f.properties.normalisedLabel) {
f.properties.matchType = "exact,name";
match = true;
} else if (d.normalisedLabel === normaliseLabel(f.properties['name:en'])) {
f.properties.matchType = "exact,name:en";
match = true;
} else if (d.normalisedLabel === normaliseLabel(f.properties['alt_name'])) {
f.properties.matchType = "exact,alt_name";
match = true;
}
if (match) {
f.properties.cantabularCode = d.code;
f.properties.cantabularLabel = d.label;
d.matches.push(f);
}
});
return d;
});
}
dedsWithFuzzyMatches = {
return dedsWithExactMatches.map((d) => {
// Don't add fuzzy matches if we already have exact matches
if (d.matches.length > 0) return d;
// Don't attempt to fuzzy match DEDs with numbers in or we end up with false positives
// notably things like "Cork No. 4 Urban" and "Cork No. 5 Urban"
if (d.label.search(/[0-9]/) > -1) {
return d;
}
dedsGeojson.features.forEach((f) => {
if (d.county !== f.properties.county) return; // Only match within the same county
if (f.properties.cantabularCode?.length) return; // Don't match a feature that has already been matched
const distance = levenshtein(d.normalisedLabel, f.properties.normalisedLabel);
if (distance === 1) {
f.properties.matchType = 'levenshtein';
f.properties.cantabularCode = d.code;
f.properties.cantabularLabel = d.label;
d.matches.push(f);
}
});
return d;
});
}
dedsWithMatches = {
const manualMatches = await FileAttachment("manual-ded-matches-2024-02-29.csv").csv();
const codeToMatch = d3.group(manualMatches, (d) => d.code);
return dedsWithFuzzyMatches.map((d) => {
// Skip if we already have matches
if (d.matches.length > 0) return d;
// Skip if no manual match
if (!codeToMatch.has(d.code)) return d;
const match = codeToMatch.get(d.code)[0];
// Skip if no URL
if (!match.url) return d;
const osmId = match.url.match(/[^/]+$/)[0];
// Skip if no OSM ID found
if (osmId === null) return d;
dedsGeojson.features.forEach((f) => {
if (f.properties.id === parseInt(osmId)) {
if (f.properties.cantabularCode?.length) {
console.log('Warning: already matched');
console.log('- ', f.properties.id, f.properties.name, f.properties.normalisedLabel);
console.log('- ', f.properties.cantabularCode, f.properties.cantabularLabel, f.properties.matchType);
}
f.properties.matchType = 'manual';
f.properties.cantabularCode = d.code;
f.properties.cantabularLabel = d.label;
d.matches.push(f);
}
});
return d;
});
}
dedsGeojson = {
const json = await FileAttachment("deds-simplified.geojson").json();
// Loop through the features backwards so we can remove ones we're not interested in
for (let i = json.features.length - 1; i >= 0; i--) {
const f = json.features[i];
if (parseInt(f.properties.date) > 1970) json.features.splice(i, 1);
f.properties.county = translateCountyName(f.properties.county.replace(/^County/, '').trim());
f.properties.normalisedLabel = normaliseLabel(f.properties.name);
}
return json;
}
deds = {
const response = await queryCantabularGraphQL(ENDPOINT, CATEGORIES_QUERY, {
"dataset": DATASET,
"variable": "county_district_geoid",
});
let categoryEdges = response.data.dataset.variables.edges[0].node.categories.edges;
return categoryEdges.map((e) => {
// Remove the county name from each DED label for the normalised label
const nl = e.node.label.replace(/^[^,]+, /, '');
return {
label: e.node.label,
normalisedLabel: normaliseLabel(nl),
code: e.node.code,
county: e.node.label.match(/^[^,]+/)[0],
};
});
}
function translateCountyName(name) {
if (name === "Laois") return "Queen's Co.";
if (name === "Offaly") return "King's Co.";
return name;
}
function normaliseLabel(n) {
if (!n) return '';
const replacements = [
{ search: / [0-9]{4}$/, replace: '' }, // Some OSM DEDs have a year suffix, e.g. "1911" or "1986"
{ search: / District Electoral Division$/i, replace: '' },
{ search: / Electoral Division$/i, replace: '' },
{ search: / DED$/, replace: ''},
{ search: / ED$/, replace: ''},
{ search: /[.,'\/#!$%\^&\*;:{}=\-_`~()]/g, replace: ' ' },
{ search: /\s{2,}/g, replace: ' ' },
];
replacements.forEach((r) => n = n.replace(r.search, r.replace).trim());
return n.toLowerCase();
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
