3e:null 3f:["$","title",null,{"children":"Threshold Optimisation | Matt Simmons | Observable"}] 45:T609,{ const width = 950 //width of svg const height = 350 //height of svg const svg = d3.select(DOM.svg(width, height)) const s = 0.6 // scale unit (pixels per £) const dx = Math.ceil(width/100) const bars = svg.selectAll("g") .data(data).enter() .append("g"); const paid = bars.append("rect") .style("fill", d => d.prediction > threshold ? palette.tp : palette.fn) // .attr("height", 0) .attr("width", dx) // .attr("y", d => height) .attr("x", d => d.prediction * width)// + width/100 * .5) .attr("height", d => d['paid'] * (revenue - cost) * s) .attr("y", d => height - d['paid'] * (revenue - cost) * s) // paid.transition().duration(100).delay((d, i) => i * 10) const unpaid = bars.append("rect") .style("fill", d => d.prediction > threshold ? palette.fp : palette.tn ) // .attr("height", 0) .attr("width", dx) // .attr("y", d => height) .attr("x", d => d.prediction * width) .attr("height", d => d['unpaid'] * cost * s) .attr("y", d => height - d['unpaid'] * cost * s) svg.append("text") .attr("x", threshold * width + 15) .attr("y", height - 10) .text("send") svg.append("text") .attr("x", threshold * width) .attr("y", height - 10) .attr("fill", "white") .attr("text-anchor", "end") .text("don't send") // unpaid.transition().duration(1000).delay((d, i) => i * 50) // .attr("height", d => d['unpaid'] * s) // .attr("y", d => height - d['unpaid'] * s) return svg.node() }46:T4ad,html`

true negatives: ${confusionMatrix.tn}
false negatives: ${confusionMatrix.fn}

`47:T13a0,data = [ { prediction: 0.00, paid: 0, unpaid: 15 }, { prediction: 0.01, paid: 1, unpaid: 17 }, { prediction: 0.02, paid: 0, unpaid: 19 }, { prediction: 0.03, paid: 1, unpaid: 21 }, { prediction: 0.04, paid: 1, unpaid: 26 }, { prediction: 0.05, paid: 1, unpaid: 34 }, { prediction: 0.06, paid: 1, unpaid: 45 }, { prediction: 0.07, paid: 2, unpaid: 34 }, { prediction: 0.08, paid: 1, unpaid: 43 }, { prediction: 0.09, paid: 1, unpaid: 45 }, { prediction: 0.10, paid: 2, unpaid: 50 }, { prediction: 0.11, paid: 2, unpaid: 54 }, { prediction: 0.12, paid: 2, unpaid: 51 }, { prediction: 0.13, paid: 2, unpaid: 52 }, { prediction: 0.14, paid: 2, unpaid: 58 }, { prediction: 0.15, paid: 2, unpaid: 53 }, { prediction: 0.16, paid: 2, unpaid: 50 }, { prediction: 0.17, paid: 2, unpaid: 51 }, { prediction: 0.18, paid: 2, unpaid: 66 }, { prediction: 0.19, paid: 2, unpaid: 67 }, { prediction: 0.20, paid: 5, unpaid: 75 }, { prediction: 0.21, paid: 4, unpaid: 70 }, { prediction: 0.22, paid: 6, unpaid: 71 }, { prediction: 0.23, paid: 6, unpaid: 75 }, { prediction: 0.24, paid: 7, unpaid: 72 }, { prediction: 0.25, paid: 7, unpaid: 74 }, { prediction: 0.26, paid: 8, unpaid: 73 }, { prediction: 0.27, paid: 10, unpaid: 76 }, { prediction: 0.28, paid: 11, unpaid: 77 }, { prediction: 0.29, paid: 12, unpaid: 81 }, { prediction: 0.30, paid: 13, unpaid: 77 }, { prediction: 0.31, paid: 13, unpaid: 82 }, { prediction: 0.32, paid: 14, unpaid: 90 }, { prediction: 0.33, paid: 15, unpaid: 94 }, { prediction: 0.34, paid: 14, unpaid: 98 }, { prediction: 0.35, paid: 15, unpaid: 100 }, { prediction: 0.36, paid: 17, unpaid: 99 }, { prediction: 0.37, paid: 20, unpaid: 95 }, { prediction: 0.38, paid: 24, unpaid: 90 }, { prediction: 0.39, paid: 21, unpaid: 87 }, { prediction: 0.40, paid: 24, unpaid: 77 }, { prediction: 0.41, paid: 26, unpaid: 60 }, { prediction: 0.42, paid: 28, unpaid: 56 }, { prediction: 0.43, paid: 30, unpaid: 51 }, { prediction: 0.44, paid: 28, unpaid: 48 }, { prediction: 0.45, paid: 29, unpaid: 46 }, { prediction: 0.46, paid: 30, unpaid: 44 }, { prediction: 0.47, paid: 32, unpaid: 43 }, { prediction: 0.48, paid: 35, unpaid: 41 }, { prediction: 0.49, paid: 36, unpaid: 40 }, { prediction: 0.50, paid: 42, unpaid: 38 }, { prediction: 0.51, paid: 41, unpaid: 35 }, { prediction: 0.52, paid: 44, unpaid: 32 }, { prediction: 0.53, paid: 47, unpaid: 30 }, { prediction: 0.54, paid: 49, unpaid: 29 }, { prediction: 0.55, paid: 46, unpaid: 24 }, { prediction: 0.56, paid: 42, unpaid: 26 }, { prediction: 0.57, paid: 46, unpaid: 27 }, { prediction: 0.58, paid: 43, unpaid: 26 }, { prediction: 0.59, paid: 48, unpaid: 25 }, { prediction: 0.60, paid: 50, unpaid: 26 }, { prediction: 0.61, paid: 50, unpaid: 27 }, { prediction: 0.62, paid: 51, unpaid: 23 }, { prediction: 0.63, paid: 53, unpaid: 26 }, { prediction: 0.64, paid: 50, unpaid: 24 }, { prediction: 0.65, paid: 54, unpaid: 29 }, { prediction: 0.66, paid: 53, unpaid: 25 }, { prediction: 0.67, paid: 54, unpaid: 21 }, { prediction: 0.68, paid: 51, unpaid: 22 }, { prediction: 0.69, paid: 50, unpaid: 25 }, { prediction: 0.70, paid: 48, unpaid: 19 }, { prediction: 0.71, paid: 43, unpaid: 21 }, { prediction: 0.72, paid: 46, unpaid: 22 }, { prediction: 0.73, paid: 47, unpaid: 23 }, { prediction: 0.74, paid: 48, unpaid: 21 }, { prediction: 0.75, paid: 43, unpaid: 20 }, { prediction: 0.76, paid: 44, unpaid: 19 }, { prediction: 0.77, paid: 45, unpaid: 16 }, { prediction: 0.78, paid: 43, unpaid: 18 }, { prediction: 0.79, paid: 40, unpaid: 17 }, { prediction: 0.80, paid: 39, unpaid: 11 }, { prediction: 0.81, paid: 37, unpaid: 10 }, { prediction: 0.82, paid: 34, unpaid: 14 }, { prediction: 0.83, paid: 33, unpaid: 13 }, { prediction: 0.84, paid: 38, unpaid: 12 }, { prediction: 0.85, paid: 39, unpaid: 10 }, { prediction: 0.86, paid: 36, unpaid: 6 }, { prediction: 0.87, paid: 34, unpaid: 8 }, { prediction: 0.88, paid: 33, unpaid: 5 }, { prediction: 0.89, paid: 30, unpaid: 3 }, { prediction: 0.90, paid: 29, unpaid: 5 }, { prediction: 0.91, paid: 27, unpaid: 2 }, { prediction: 0.92, paid: 25, unpaid: 1 }, { prediction: 0.93, paid: 29, unpaid: 1 }, { prediction: 0.94, paid: 24, unpaid: 2 }, { prediction: 0.95, paid: 21, unpaid: 1 }, { prediction: 0.96, paid: 19, unpaid: 1 }, { prediction: 0.97, paid: 15, unpaid: 2 }, { prediction: 0.98, paid: 13, unpaid: 1 }, { prediction: 0.99, paid: 12, unpaid: 1 }, { prediction: 1.0, paid: 10, unpaid: 0 }, ]48:[] 43:{"id":"aef6819db610d712","slug":null,"title":"Threshold Optimisation","thumbnail":"67e99820f7596ed8d218ef283cdecfb32bcfc911e81acb335404c516653b44b6","access_level":"public","create_time":"$D2021-12-07T14:47:18.165Z","update_time":"$D2025-05-07T16:40:21.555Z","view_time":null,"first_public_time":"$D2023-10-16T12:34:25.676Z","has_unpublished_edits":null,"fork":true,"stars":1,"starred":false,"owner":{"type":"team","id":"2f56a9e0c10f640a","login":"datasparq-matt","name":"Matt Simmons","avatar_url":"https://avatars-next.observableusercontent.com/6be6333d48f09b772cd2f2445d42d3c0a90a826d17167bb4dd09a83193748083","tier":"starter_2024","bio":"","following":false},"version":756,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"b3a32e38a20f5b21","login":"datasparq-matt","name":"Matt Simmons","avatar_url":"https://avatars-next.observableusercontent.com/6be6333d48f09b772cd2f2445d42d3c0a90a826d17167bb4dd09a83193748083","tier":null,"bio":"","following":false}],"members":[{"type":"individual","id":"b3a32e38a20f5b21","login":"datasparq-matt","name":"Matt Simmons","avatar_url":"https://avatars-next.observableusercontent.com/6be6333d48f09b772cd2f2445d42d3c0a90a826d17167bb4dd09a83193748083","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"titling":"auto","title":"Threshold Optimisation","cells":[{"id":0,"value":"md`# Threshold Optimisation`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":640,"value":"Let's imagine we're sending out an offer to all of our valued customers:\n\n \"Dear customer, please buy a rowing machine. Only £${revenue.toFixed(2)}!\"\n\nIt costs £${cost.toFixed(2)} to send this offer, so if they pay, we'll earn £${(revenue - cost).toFixed(2)}.\n\nWe'll use machine learning to train a model on historical data that predicts which customers are likely to buy. The model gives us a score for every customer that tells us their **likelihood to buy**. We now need to decide how high this score needs to be before we send the offer. There is an **optimal** threshold, which will maximise the profit by balancing potential revenue and cost to send.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":204,"value":"md`Customers with values above **${threshold}** are predicted to buy. The histogram shows the distribution of our model's predictions and is coloured by the true outcome.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":83,"value":"md`Set the **threshold** for classification with the slider. Try to find the optimal threshold to maximise profit.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":73,"value":"viewof threshold = slider()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":2,"value":"$45","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":160,"value":"$46","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":661,"value":"We use a **confusion matrix** and the costs involved to calculate the profit we expect from this threshold. If revenue, cost, or the model's accuracy change then the optimal threshold will change too.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":182,"value":"viewof revenue = slider({\n min: 5, \n max: 20, \n step: 0.1, \n value: 15, \n title: \"Revenue\", \n description: \"£ value earned when customer pays\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":178,"value":"viewof cost = slider({\n min: 0, \n max: 5, \n step: 0.1, \n value: 5, \n title: \"Cost\", \n description: \"£ cost of sending\"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":663,"value":"Now that we know the optimal threshold, we can use this when labelling predictions on new data, for which we don't know the outcome. We have no guarantees that this threshold will still be the best when applied to new data, but it's a pretty good educated guess with lots of data to back it up.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":589,"value":"appendix = md`\n---\n\n## Appendix`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":155,"value":"confusionMatrix = {\n let cm = {\n tp: d3.sum(data.filter(d => d.prediction > threshold), d => d.paid), // prediction = paid, result = paid >> make money\n fp: d3.sum(data.filter(d => d.prediction > threshold), d => d.unpaid), // prediction = paid, result = unpaid >> loose money\n tn: d3.sum(data.filter(d => d.prediction <= threshold), d => d.unpaid),\n fn: d3.sum(data.filter(d => d.prediction <= threshold), d => d.paid),\n }\n cm.revenue = cm.tp * (revenue - cost);\n cm.cost = cm.fp * cost;\n cm.profit = cm.revenue - cm.cost;\n return cm\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":141,"value":"$47","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":539,"value":"palette = {\n return {\n tp: \"#44bd7c88\",\n fp: \"#ed571a88\",\n tn: \"#666b73aa\",\n fn: \"#666b73aa\",\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":359,"value":"html`\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"d3 = require(\"d3@v5\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":75,"value":"import {slider} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":578,"value":"Inputs = require(\"@observablehq/inputs@0.7.17/dist/inputs.umd.min.js\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W48","stdlib":"1"}} 44:"$43:body"