25:["$","title",null,{"children":"Living with Machines: Toponym recognition and resolution — input and output exploration | Daniel C.S. Wilson | Observable"}] 49:T485,# Living with Machines: Toponym recognition and resolution — input and output exploration This notebook follows a recent call from the IEEE’s visualisation interest group: with the growing complexity of AI models, visualisation can be a powerful technique in response to the “critical need for understanding their inner-workings.”^{[1](http://visxai.io/)} The Living with Machines project uses 19th-century British newspapers and other large, digitised primary source collections to write new histories of industrialisation. Finding named places in full text and geolocating (or "resolving") them is an important part of many text analysis experiments, as this task allows researchers to understand the geospatial footprint of a corpus. Here, we make it possible to **explore the input and the output** for the toponym recognition and resolution pipeline designed by Mariona Coll-Ardanuy and Federico Nanni for Living with Machines. This pipeline has been designed specifically to perform well on 19th-century newspapers in English. You can learn more about the pipeline [here](https://github.com/Living-with-machines/toponym-resolution).4a:T6ff,sample = { const titles = Object.fromEntries( Object.entries(data.models).map(([model, arr]) => [ model, _.uniq(arr.map((o) => o.title)) ]) ); const articles = Object.fromEntries( Object.entries(data.models).map(([model, arr]) => [ model, _.uniq(arr.map((o) => `${o.NLP}-${o.issue}-${o.art_num}`)) ]) ); const locations = Object.fromEntries( Object.entries(data.models).map(([model, arr]) => [ model, _.uniq(arr.map((o) => o.location)) ]) ); const meanOCR = Object.fromEntries( Object.entries(data.models).map(([model, arr]) => [ model, _.mean(_.uniq(arr.map((o) => o.ocrquality))) ]) ); const medianOCR = Object.fromEntries( Object.entries(data.models).map(([model, arr]) => [ model, median(_.uniq(arr.map((o) => o.ocrquality))) ]) ); const wikiDataHierarchy = Object.fromEntries( Object.entries(data.models).map(([model, arr]) => { const parents = _.uniq(arr.map((d) => d.wkdt_class).filter((d) => d)); const hierarchy = { name: model, children: parents.map((parent) => ({ name: wikidataLabels.find((o) => o.id === parent) ? wikidataLabels.find((o) => o.id === parent).label : parent, children: arr .filter((d) => d.wkdt_class === parent) .map((d) => d.prediction) .map((d) => ({ name: wikidataLabels.find((o) => o.id === d) ? wikidataLabels.find((o) => o.id === d).label : d })) })) }; return [model, hierarchy]; }) ); return { titles, articles, locations, ocr: { mean: meanOCR, median: medianOCR }, wikiDataHierarchy }; }4b:T4b2,filteredData = includeMethods.map((model) => { const arr = data.models[model]; const toponyms = arr .filter((o) => includeTags.includes(o.tag)) .filter( (o) => o.ed_score >= edScoreRange[0] && o.ed_score <= edScoreRange[1] ) .filter( (o) => o.ocrquality >= OCRQualityRange[0] && o.ocrquality <= OCRQualityRange[1] ); const parents = _.uniq(toponyms.map((d) => d.wkdt_class).filter((d) => d)); const hierarchy = { name: model, children: parents.map((parent) => ({ name: wikidataLabels.find((o) => o.id === parent) ? wikidataLabels.find((o) => o.id === parent).label : parent, children: toponyms .filter((d) => d.wkdt_class === parent) .map((d) => d.prediction) .map((d) => ({ name: wikidataLabels.find((o) => o.id === d) ? wikidataLabels.find((o) => o.id === d).label : d })) })) }; const labelCounts = d3 .groups(toponyms, (d) => d.tag) .map((d) => ({ label: d[0], count: d[1].length, model: model })); return { model, toponyms, hierarchy, numToponyms: toponyms.length, labelCounts }; })4c:T570,data = { const deezyMostPopular = await FileAttachment("deezy_mostpopular.csv").csv({ typed: true }), deezyRel = await FileAttachment("deezy_rel_without_publ.csv").csv({ typed: true }), perfectMostPopular = await FileAttachment("perfect_mostpopular.csv").csv({ typed: true }); const methods = ["deezyMostPopular", "deezyRel", "perfectMostPopular"]; let val; val = "ed_score"; const edScoresRange = [ ...deezyMostPopular.map((d) => d[val]), ...deezyRel.map((d) => d[val]), ...perfectMostPopular.map((d) => d[val]) ]; val = "ocrquality"; const ocrRange = [ ...deezyMostPopular.map((d) => d[val]), ...deezyRel.map((d) => d[val]), ...perfectMostPopular.map((d) => d[val]) ]; val = "tag"; const tagRange = [ ...deezyMostPopular.map((d) => d[val]), ...deezyRel.map((d) => d[val]), ...perfectMostPopular.map((d) => d[val]) ].filter((d) => d); // continuous const edScores = d3.extent(edScoresRange); const OCRQualities = d3.extent(ocrRange); // categorical const tags = _.uniq(tagRange).sort(); return { uniques: { edScores, OCRQualities, tags, methods }, ranges: { edScores: edScoresRange, OCRQualities: ocrRange, tags: tagRange }, models: { deezyMostPopular, deezyRel, perfectMostPopular } }; }4d:T13b9,// instead of import {Pack} from "@d3/pack" function Pack( data, { // data is either tabular (array of objects) or hierarchy (nested objects) path, // as an alternative to id and parentId, returns an array identifier, imputing internal nodes id = Array.isArray(data) ? (d) => d.id : null, // if tabular data, given a d in data, returns a unique identifier (string) parentId = Array.isArray(data) ? (d) => d.parentId : null, // if tabular data, given a node d, returns its parent’s identifier children, // if hierarchical data, given a d in data, returns its children value, // given a node d, returns a quantitative value (for area encoding; null for count) sort = (a, b) => d3.descending(a.value, b.value), // how to sort nodes prior to layout label, // given a leaf node d, returns the display name title, // given a node d, returns its hover text link, // given a node d, its link (if any) linkTarget = "_blank", // the target attribute for links, if any width = 640, // outer width, in pixels height = 400, // outer height, in pixels margin = 1, // shorthand for margins marginTop = margin, // top margin, in pixels marginRight = margin, // right margin, in pixels marginBottom = margin, // bottom margin, in pixels marginLeft = margin, // left margin, in pixels padding = 3, // separation between circles fill = "#ddd", // fill for leaf circles fillOpacity, // fill opacity for leaf circles stroke = "#bbb", // stroke for internal circles strokeWidth, // stroke width for internal circles strokeOpacity // stroke opacity for internal circles } = {} ) { // If id and parentId options are specified, or the path option, use d3.stratify // to convert tabular data to a hierarchy; otherwise we assume that the data is // specified as an object {children} with nested objects (a.k.a. the “flare.json” // format), and use d3.hierarchy. const root = path != null ? d3.stratify().path(path)(data) : id != null || parentId != null ? d3.stratify().id(id).parentId(parentId)(data) : d3.hierarchy(data, children); // Compute the values of internal nodes by aggregating from the leaves. value == null ? root.count() : root.sum((d) => Math.max(0, value(d))); // Compute labels and titles. const descendants = root.descendants(); const leaves = descendants.filter((d) => !d.children); leaves.forEach((d, i) => (d.index = i)); const L = label == null ? null : leaves.map((d) => label(d.data, d)); const T = title == null ? null : descendants.map((d) => title(d.data, d)); // Sort the leaves (typically by descending value for a pleasing layout). if (sort != null) root.sort(sort); // Compute the layout. d3 .pack() .size([width - marginLeft - marginRight, height - marginTop - marginBottom]) .padding(padding)(root); const svg = d3 .create("svg") .attr("viewBox", [-marginLeft, -marginTop, width, height]) .attr("width", width) .attr("height", height) .attr("style", "max-width: 100%; height: auto; height: intrinsic;") .attr("font-family", "sans-serif") .attr("font-size", 10) .attr("text-anchor", "middle"); const node = svg .selectAll("a") .data(descendants) .join("a") .attr("xlink:href", link == null ? null : (d, i) => link(d.data, d)) .attr("target", link == null ? null : linkTarget) .attr("transform", (d) => `translate(${d.x},${d.y})`); const circle = node .append("circle") .attr("fill", (d) => (d.children ? "#fff" : fill)) .attr("fill-opacity", (d) => (d.children ? null : fillOpacity)) .attr("stroke", (d) => (d.children ? stroke : null)) .attr("stroke-width", (d) => (d.children ? strokeWidth : null)) .attr("stroke-opacity", (d) => (d.children ? strokeOpacity : null)) .attr("r", (d) => d.r); if (T) node.append("title").text((d, i) => T[i]); if (L) { // A unique identifier for clip paths (to avoid conflicts). const uid = `O-${Math.random().toString(16).slice(2)}`; const leaf = node.filter( (d) => !d.children && d.r > 10 && L[d.index] != null ); leaf .append("clipPath") .attr("id", (d) => `${uid}-clip-${d.index}`) .append("circle") .attr("r", (d) => d.r); leaf .append("text") .attr( "clip-path", (d) => `url(${new URL(`#${uid}-clip-${d.index}`, location)})` ) .selectAll("tspan") .data((d) => `${L[d.index]}`.split(/\n/g)) .join("tspan") .attr("x", 0) .attr("y", (d, i, D) => `${i - D.length / 2 + 0.85}em`) .attr("fill-opacity", (d, i, D) => (i === D.length - 1 ? 0.7 : null)) .text((d) => d); } function pointermoved(event) { // tooltip.style("display", null); svg.property("value", "hello").dispatch("input", { bubbles: true }); } function pointerleft() { // tooltip.style("display", "none"); svg.node().value = null; svg.dispatch("input", { bubbles: true }); } return svg.node(); }4e:[{"id":"4edfdced60469d4dcb9fa269852116c32441b28b1c073c74a7e72765efc9f29db425fac599346cfba12ae5371a6e8c81906df9fce1cd32aaa08f5e36c1f564f5","href":"https://static.observableusercontent.com/files/4edfdced60469d4dcb9fa269852116c32441b28b1c073c74a7e72765efc9f29db425fac599346cfba12ae5371a6e8c81906df9fce1cd32aaa08f5e36c1f564f5","name":"deezy_mostpopular.csv","type":"text/csv","lastModified":1682342552463.376,"size":3553063},{"id":"9c42b216cf830798707d4374b86d755e6e6b9e612f4cfbbd4c439d3079ef892e977d40d85a4463261c84275554ec943f5f03da7138d5cf9b8f5efab42e81077f","href":"https://static.observableusercontent.com/files/9c42b216cf830798707d4374b86d755e6e6b9e612f4cfbbd4c439d3079ef892e977d40d85a4463261c84275554ec943f5f03da7138d5cf9b8f5efab42e81077f","name":"deezy_rel_without_publ.csv","type":"text/csv","lastModified":1682342552463.376,"size":3542389},{"id":"4262b87281fd72c76bb54a397dd96ca84e46f868b28cb05b01cb443f459813ca70de8f0a1d8feb94e867467d8aed4aee643fd5cd9888fda9a5eee513173dc267","href":"https://static.observableusercontent.com/files/4262b87281fd72c76bb54a397dd96ca84e46f868b28cb05b01cb443f459813ca70de8f0a1d8feb94e867467d8aed4aee643fd5cd9888fda9a5eee513173dc267","name":"perfect_mostpopular.csv","type":"text/csv","lastModified":1682342552463.376,"size":3542264},{"id":"f302224f56b9d023bc343f0a7c55aeabb6bf7127bbdbccdf4f7ad3463d5efb9b602af82cf31a1b9449f400b9d14a1634b7a046ae795a70f67958f4e9946c3854","href":"https://static.observableusercontent.com/files/f302224f56b9d023bc343f0a7c55aeabb6bf7127bbdbccdf4f7ad3463d5efb9b602af82cf31a1b9449f400b9d14a1634b7a046ae795a70f67958f4e9946c3854","name":"wikidata_labels.json","type":"application/json","lastModified":1682342552463.376,"size":34498}] 29:{"id":"e972c5d37e683a4c","slug":null,"title":"Living with Machines: Toponym recognition and resolution — input and output exploration","thumbnail":"48d9c62404d67903fdf0d6780963926605a8394a5a5f181cd4ecb784013ab9c4","access_level":"public","create_time":"$D2023-04-24T13:22:32.463Z","update_time":"$D2023-04-24T13:22:32.463Z","view_time":null,"first_public_time":"$D2023-04-24T13:22:32.463Z","has_unpublished_edits":null,"fork":true,"stars":1,"starred":false,"owner":{"type":"team","id":"cf3f43b31edd9217","login":"dcsw2","name":"Daniel C.S. Wilson","avatar_url":"https://avatars-next.observableusercontent.com/3147ee0007ffaf52c9c52ec37a26e30d1b19171977037e0f873c31b50bd04b39","tier":"starter_2024","bio":"","followed_by_you":false,"follows_you":false},"version":815,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"f6b17ba6c89f39f5","login":"dcsw2","name":"Daniel C.S. Wilson","avatar_url":"https://avatars-next.observableusercontent.com/3147ee0007ffaf52c9c52ec37a26e30d1b19171977037e0f873c31b50bd04b39","tier":null,"bio":"","followed_by_you":false,"follows_you":false}],"members":[{"type":"individual","id":"f6b17ba6c89f39f5","login":"dcsw2","name":"Daniel C.S. Wilson","avatar_url":"https://avatars-next.observableusercontent.com/3147ee0007ffaf52c9c52ec37a26e30d1b19171977037e0f873c31b50bd04b39","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Living with Machines: Toponym recognition and resolution — input and output exploration","cells":[{"id":0,"value":"$49","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":77,"value":"

","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":59,"value":"## Input Review\n\nUsually, the input to the toponym pipeline is a sample of newspaper articles from very large collections. Sample selection can be based on a range of factors relevant to a research questions. At the point of reviewing the results of the pipeline, it is useful to sketch the outlines of the sample - its size, what newspaper titles it includes, its temporal coverage, etc. Having these basic features of the input sample visible here informs our interpretation of the results.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":185,"value":"> info about the sample itself: # of titles, publication date span, # of articles, average OCR quality [replace with something actually useful]","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":208,"value":"$4a","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":722,"value":"#### Publication information\n\n${_.uniq(_.flatten(Object.values(sample.titles))).length} title(s) represented: \n\n- ${_.uniq(_.flatten(Object.values(sample.titles))).join(\"\\n- \")}\n\n${_.uniq(_.flatten(Object.values(sample.locations))).length} location(s) represented: \n\n- ${_.uniq(_.flatten(Object.values(sample.locations))).join(\"\\n- \")}\n\n${_.uniq(_.flatten(Object.values(sample.articles))).length} article ID(s) in sample dataset represented: \n\n- ${_.uniq(_.flatten(Object.values(sample.articles))).join(\"\\n- \")}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":764,"value":"## Output Exploration\n\nThe toponym pipeline performs 3 tasks:\n1. **Recognises** named places in full text of newspaper articles\n2. **Ranks** candidates from Wikidata that are matches to the recognised names\n3. **Links** the best match to an entity in Wikidata\n\n_The Ranker and the Linker elements each have multiple methods that can impact the final results. One of the most important roles of this notebook is to quickly see the impact that choosing one combination of these methods over another has on the quality of the results._\n\nThe visualisations below help us explore the results of these three tasks for any given input sample.\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":779,"value":"### Recogniser Method\n\nThis pipeline takes a multi-scale approach to toponym recognition. Named places can be tagged by the recogniser as \"LOC\", \"STREET\", \"BUILDING\", or \"ALIEN\".^{[2](Details about these classes can be found in the data paper sharing the gold standard annotations for this research: https://bl.iro.bl.uk/concern/articles/a43a09e9-59bf-475b-8a3a-f01a5df5eba1)}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":775,"value":"### Ranker and Linker Methods\n\n#### For ranking\n```perfectmatch``` - exact match between words\n```partialmatch``` - overlap matching\n```deezymatch``` - uses [deezymatch fuzzy string matching method](https://github.com/Living-with-machines/DeezyMatch)\n\n#### For linking\n```mostpopular``` - most relevant Wikipedia candidate \n```reldisamb``` - top candidate using method based on [REL](https://github.com/informagi/REL)\n\n#### Below, you can see the results for these combinations of methods:\n- deezyMostPopular\n- deezyRel\n- perfectMostPopular","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":719,"value":"#### WikiData across all sample","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":407,"value":"This is the WikiData ID distribution across the different methods:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":430,"value":"getPacks(sample.wikiDataHierarchy)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":717,"value":"_Hover over the enclosing circle to see the WikiData group information, and over the gray dots to see more information about the WikiData ID that is contained in that group._","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":79,"value":"

Top 10 parent circles

${filteredData.map((d) => {\n const data = d.hierarchy.children\n .sort((a, b) => b.children.length - a.children.length)\n .map((d) => ({ name: d.name, children: d.children.length }))\n .slice(0, 10);\n\n const elem = d3.create(\"table\").attr(\"style\", \"margin-right: 20px;\");\n\n const div = elem.selectAll(\"tr\").data(data).join(\"tr\");\n\n div\n .append(\"td\")\n .text((d) => d.name)\n .on(\"mouseover\", function (evt, d) {\n console.log(d);\n });\n div.append(\"td\").text((d) => d.children);\n\n return elem.node();\n })}

`;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":567,"value":"#### Confidence score\n\n...statistics about the confidence score for each method + label...","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":194,"value":"

${model}

${Pack(\n hierarchy,\n settings\n )}

`\n );\n\n return html`\n

${svgs}

\n `;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":575,"value":"$4d","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":31,"value":"### Imports","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":14,"value":"import {interval} from \"@mootari/range-slider\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":799,"value":"deezy_mostpopular = await FileAttachment(\"deezy_mostpopular.csv\").csv({typed: \"auto\"})","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"deezy_mostpopular","database":"$undefined"},{"id":802,"value":"## spatial center of resolved toponyms\n\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"}],"files":"$W4e","stdlib":"1"}} 24:null