2a:null 2b:["$","title",null,{"children":"An Interactive Introduction to Probabilistic Record Linkage | Robin Linacre | Observable"}] 30:T9aa,splink_settings = { return { proportion_of_matches: 1 / 60e6, comparison_columns: [ { num_levels: 4, col_name: "fname", u_probabilities: [ 0.9883897757902356, 0.004639565877697178, 0.002783859108435161, 0.004186799223632191 ], m_probabilities: [ 0.013910835049471115, 0.03580070792581899, 0.0026304325373041827, 0.9431284756772805 ], case_expression: fname_case_stmt }, { num_levels: 4, col_name: "mname", u_probabilities: [ 0.9883897757902356, 0.004639565877697178, 0.002783859108435161, 0.004186799223632191 ], m_probabilities: [ 0.013910835049471115, 0.03580070792581899, 0.0026304325373041827, 0.9431284756772805 ], case_expression: mname_case_stmt }, { num_levels: 4, col_name: "sname", u_probabilities: [ 0.9971958658858057, 0.0015765242689686637, 0.0003754879678726784, 0.0008521218773530913 ], m_probabilities: [ 0.01809857679375992, 0.02769695651313621, 0.0014468181425986496, 0.9514336326024172 ], case_expression: sname_case_stmt }, { col_name: "dob", u_probabilities: [ 0.2807916467157474, 0.20073737664045227, 0.5164828418823143, 0.0019211842885877536, 9.766061108210657e-7, 6.597386678746452e-5 ], m_probabilities: [ 0.00024675771766021674, 0.0017179728285089994, 0.009868620663422667, 0.023118927215599792, 0.0036827860949234354, 0.9571689718277999 ], case_expression: dob_case_stmt }, { col_name: "pc", u_probabilities: [ 0.6751190264364737, 0.22792766449447727, 0.09517523421356786, 1.7091033037790942e-6, 0.0017755009047047013, 8.648474727307328e-7 ], m_probabilities: [ 0.18962646163517627, 0.02145498887358967, 0.4023991769621992, 0.11064711462002584, 0.07144100324386156, 0.20090472079462598 ], case_expression: pc_case_stmt } ] }; }31:[] 2e:{"id":"d51533bbe054b3d8","slug":"d51533bbe054b3d8","title":"An Interactive Introduction to Probabilistic Record Linkage","thumbnail":"121c41ab6a6369d7f96c7fbb8513c2b11db5a8d8c0896ae521519a1dcd130ffd","access_level":"unlisted","create_time":"$D2021-05-25T18:31:59.155Z","update_time":"$D2021-08-11T08:04:42.178Z","view_time":null,"first_public_time":"$D2021-05-25T18:50:30.077Z","has_unpublished_edits":null,"fork":true,"stars":1,"starred":false,"owner":{"type":"team","id":"4baeb83528a5ca3e","login":"robinl","name":"Robin Linacre","avatar_url":"https://avatars-next.observableusercontent.com/68a9fb29f18d689995452badc30bc11082c1ae33c6fa13bef8a0a866b2e51264","tier":"starter_2024"},"version":510,"publish_version":510,"role":null,"authored":false,"authors":[{"type":"individual","id":"d30d5a0e95d0c3da","login":"robinl","name":"Robin Linacre","avatar_url":"https://avatars-next.observableusercontent.com/68a9fb29f18d689995452badc30bc11082c1ae33c6fa13bef8a0a866b2e51264","tier":null}],"members":[{"type":"individual","id":"d30d5a0e95d0c3da","login":"robinl","name":"Robin Linacre","avatar_url":"https://avatars-next.observableusercontent.com/68a9fb29f18d689995452badc30bc11082c1ae33c6fa13bef8a0a866b2e51264","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"An Interactive Introduction to Probabilistic Record Linkage","cells":[{"id":0,"value":"title = md`# An Interactive Introduction to Probabilistic Record Linkage`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":505,"value":"md`*A final version of this notebook can be viewed on my blog [here](https://www.robinlinacre.com/intro_to_probabilistic_linkage/).*`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":447,"value":"para_1 = md`Probablistic record linkage is a technique used to link together records that lack unique identifiers. \n\nIn the absence of a unique identifier such as a National Insurance number, we can use a combination of individually non-unique variables such as name, gender and date of birth to identify individuals.\n\nRecord linkage can be done within datasets (deduplication) or between datasets (linkage), or both.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":454,"value":"para_2 = md`Linkage is probabilistic in the sense that it relies on the balance on evidence. In a large dataset, observing that two records match on the full name 'John Smith' provides some evidence that these two records may refer to the same person, but this evidence is inconclusive. \n\nMore broadly, it is often impossible to classify pairs of records as matches or non-matches beyond any doubt. Instead, the aim of probabilisitic record linkage to to quantify the probability that a pair of records refer to the same entity by considering evidence in favour and against a match and weighting it appropriately.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":461,"value":"para_3 = md`In probabilistic record linkage, we start with a prior, which represents the probability that two records drawn at random are a match. We then compare the two records, increasing the match probability where columns agree, and decreasing it when columns disagree. \n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":464,"value":"para_4 = md`The most common type of probabilistic record linkage model is called the Fellegi-Sunter model. \n\nLet's take a look at at an example of a trained Fellegi-Sunter model to calculate match probability interactively. This model will compare the two records in the table, and assess whether they refer to the same person, or different people.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":223,"value":"para_5 = md`\n✨You can edit the records in the table, and the match probability will be estimated.✨`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":21,"value":"table = html`\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n

First name	Middle name	Surname	DOB	Postcode
Mary	Emma	Smith	1990-05-09	SW1A 1AA
Mary	Emma	Smith	1990-05-09	SW1A 1AA

\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":210,"value":"estimated_probability = html`

Estimated match probability = ${match_prob}%

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":386,"value":"para_6 = md`We can represent the model graphically using a waterfall chart, which is read left to right. We start with the prior, and take each column into account in turn, with different match weights being applied to each column. You can hover over the bars to see how the probability changes as each subsequent field is taken into account.\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":390,"value":"viewof show_complex = inputs.toggle({\n label: \"Show details in waterfall chart\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":87,"value":"waterfall_chart = get_waterfall_chart(\n df_gammas.objects()[0],\n splink_settings,\n overrides,\n !show_complex,\n term_freqs\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":473,"value":"para_7 = md`Columns with higher number of distinct values, such as date of birth, tend to represent stronger evidence in favour of a match since it's less likely that two records chosen at random would match by coincidence. \n\nWhere a column does not match, this is usually evidence against a match. However, the evidence may not be strong - for example, people move house, which could cause a mismatch on the 'postcode' column.\n\nThis, in a nutshell, is how probabilistic record linkage works. By comparing records and weighting evidence appropriately, we estimate the probability of a match.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":481,"value":"structure = md`## Structure of this training material`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":478,"value":"para_8 = md` The rest of this training material explores the theory and practice of building probabilistic linkage models, focussing on the Fellegi-Sunter model. The sections are as follows:\n\n1. [The mathematics of the Fellegi-Sunter model](https://www.robinlinacre.com/maths_of_fellegi_sunter/)\n2. [Visualising the Fellegi Sunter model](https://www.robinlinacre.com/visualising_fellegi_sunter/)\n3. [The intuition behind match weights](https://www.robinlinacre.com/understanding_match_weights/)\n\nThese materials align closely to the probabilisic model used by [Splink](https://github.com/moj-analytical-services/splink), a piece of open source software for record linkage at scale.\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":206,"value":"match_prob = {\n let prob = get_match_probability(\n df_gammas.objects()[0],\n splink_settings,\n term_freqs\n );\n return (prob * 100).toPrecision(4);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":19,"value":"tableData = Generators.observe((notify) => {\n const keyinput = (event) => notify(parseTableData(table));\n table.addEventListener(\"input\", keyinput, false);\n notify(parseTableData(table));\n return () => window.removeEventListener(\"input\", keyinput);\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":17,"value":"function parseTableData(table) {\n const data = [];\n\n table.querySelectorAll(\"tr\").forEach((row, i) => {\n let cells = row.cells;\n let row_data = [];\n\n for (let cell of cells) {\n let value = cell.innerText;\n value = value.trim();\n value = value.toLowerCase();\n if (value == \"\") {\n value = null;\n }\n row_data.push(value);\n }\n data.push(row_data);\n });\n\n const formattedData = data.reduce((accumulator, row, i) => {\n if (i == 0) {\n accumulator.properties = row;\n } else {\n const rowObject = {};\n rowObject[\"uid\"] = i;\n accumulator.properties.forEach((key, i) => {\n rowObject[key] = row[i];\n });\n accumulator.push(rowObject);\n }\n return accumulator;\n }, []);\n return formattedData;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":52,"value":"import {\n get_df_gammas,\n get_df_e,\n get_df_comparison,\n get_match_probability\n} from \"d25176b0fe784ad0\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":84,"value":"df_e = get_df_e(df_gammas, splink_settings)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":62,"value":"df_gammas = get_df_gammas(df_comparison, splink_settings)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":60,"value":"df_comparison = get_df_comparison(records_l, records_r)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":54,"value":"records_l = {\n return [rename_dict_keys(tableData[0])];\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":56,"value":"records_r = {\n return [rename_dict_keys(tableData[1])];\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":70,"value":"function rename_dict_keys(d) {\n return {\n uid: d[\"uid\"],\n fname: d[\"first name\"],\n mname: d[\"middle name\"],\n sname: d[\"surname\"],\n dob: d[\"dob\"],\n pc: d[\"postcode\"]\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":289,"value":"fname_case_stmt = function (row) {\n if (row.fname_l === null || row.fname_r === null) {\n return -1;\n }\n if (row.fname_l == row.fname_r) {\n return 3;\n }\n if (dm_intersect(row.fname_l, row.fname_r)) {\n return 2;\n }\n if (distance(row.fname_l, row.fname_r) > 0.85) {\n return 1;\n } else {\n return 0;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":300,"value":"mname_case_stmt = function (row) {\n if (row.mname_l === null || row.mname_r === null) {\n return -1;\n }\n if (row.mname_l == row.mname_r) {\n return 3;\n }\n if (dm_intersect(row.mname_l, row.mname_r)) {\n return 2;\n }\n if (distance(row.mname_l, row.mname_r) > 0.85) {\n return 1;\n } else {\n return 0;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":312,"value":"sname_case_stmt = function (row) {\n if (row.sname_l === null || row.sname_r === null) {\n return -1;\n }\n if (row.sname_l == row.sname_r) {\n return 3;\n }\n if (dm_intersect(row.sname_l, row.sname_r)) {\n return 2;\n }\n if (distance(row.sname_l, row.sname_r) > 0.85) {\n return 1;\n } else {\n return 0;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":341,"value":"abs_date_diff_in_years = function (date_l, date_r) {\n let d_l = new Date(date_l);\n let d_r = new Date(date_r);\n let divisor = 24 * 60 * 60 * 1000 * 365;\n return Math.abs(d_l - d_r) / divisor;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":317,"value":"dob_case_stmt = function (row) {\n if (row.dob_l === null || row.dob_r === null) {\n return -1;\n }\n if (row.dob_l == row.dob_r && row.dob_l.slice(-5) == \"01-01\") {\n return 4;\n }\n if (row.dob_l == row.dob_r) {\n return 5;\n }\n if (levenshtein(row.dob_l, row.dob_r) <= 1) {\n return 3;\n }\n if (abs_date_diff_in_years(row.dob_l, row.dob_r) < 1) {\n return 2;\n }\n if (abs_date_diff_in_years(row.dob_l, row.dob_r) < 10) {\n return 1;\n } else {\n return 0;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":321,"value":"pc_case_stmt = function (row) {\n if (row.pc_l === null || row.pc_r === null) {\n return -1;\n }\n if (row.pc_l == row.pc_r) {\n return 5;\n }\n if (row.pc_l.slice(0, 3) == row.pc_r.slice(0, 3)) {\n return 4;\n } else {\n return 0;\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":64,"value":"$30","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":91,"value":"overrides = {\n return {\n title: {\n text:\n \"Match weights and their cumulative contribution to match probability\",\n subtitle: \"\"\n },\n height: 200\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":93,"value":"import { get_waterfall_chart } from \"843cb6e82d7c3d9b\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":113,"value":"dm = require(\"https://bundle.run/double-metaphone@2.0.0\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":121,"value":"distance = require(\"jaro-winkler@0.2.8/index.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":139,"value":"function dm_intersect(val_l, val_r) {\n let arr_l = dm.doubleMetaphone(val_l);\n let arr_r = dm.doubleMetaphone(val_r);\n return arr_l.filter((value) => arr_r.includes(value)).length > 0;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":262,"value":"first_name_lookup = {\n return d3\n .csv(\n \"https://gist.githubusercontent.com/RobinL/66df3f5d7dbea3d1add1a8e6067c7624/raw/d4d642e631563418e7b7889496c7a9149f259447/firstname_noyear_count2.csv\",\n d3.autoType\n )\n .then(csv_file => {\n let lookup = {};\n csv_file.forEach(row => {\n lookup[row.firstname] = row.freq;\n });\n return lookup;\n });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":264,"value":"surname_lookup = {\n return d3\n .csv(\n \"https://gist.githubusercontent.com/RobinL/3fe7f712ff93393fe0a28cbb1b8c3e46/raw/a87f572e194122dde4743dd905c1ce6c22fd33db/surname_noyear_count2.csv\",\n d3.autoType\n )\n .then(csv_file => {\n let lookup = {};\n csv_file.forEach(row => {\n lookup[row.surname] = row.freq;\n });\n return lookup;\n });\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":266,"value":"term_freqs = {\n return {\n fname: first_name_lookup,\n mname: first_name_lookup,\n sname: surname_lookup\n };\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":335,"value":"levenshtein = function (a, b) {\n if (a.length == 0) return b.length;\n if (b.length == 0) return a.length;\n\n var matrix = [];\n\n // increment along the first column of each row\n var i;\n for (i = 0; i <= b.length; i++) {\n matrix[i] = [i];\n }\n\n // increment each column in the first row\n var j;\n for (j = 0; j <= a.length; j++) {\n matrix[0][j] = j;\n }\n\n // Fill in the rest of the matrix\n for (i = 1; i <= b.length; i++) {\n for (j = 1; j <= a.length; j++) {\n if (b.charAt(i - 1) == a.charAt(j - 1)) {\n matrix[i][j] = matrix[i - 1][j - 1];\n } else {\n matrix[i][j] = Math.min(\n matrix[i - 1][j - 1] + 1, // substitution\n Math.min(\n matrix[i][j - 1] + 1, // insertion\n matrix[i - 1][j] + 1\n )\n ); // deletion\n }\n }\n }\n\n return matrix[b.length][a.length];\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":410,"value":"d3 = require(\"d3@6\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":412,"value":"inputs = require(\"@observablehq/inputs\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W31","stdlib":"1"}} 2f:"$2e:body"