27:null 28:["$","title",null,{"children":"Predictive Maintenance | Nico's Workspace | Observable"}] 2d:T664,plt3 = Plot.plot({ width: 1000, y: { grid: true, label: " (£)", domain: [-500000, 1100000], tickFormat: ".2s"}, x: { grid: true, label: "Threshold", domain: [0, 1]}, color: {domain: ["Cost (FP)", "Cost (FN)", "Total Savings"], range: ["darkred", "darkblue", "green"], legend: true}, marks: [ Plot.lineY(data, { x: "threshold", y: "fp_cost", tip: true, stroke: "darkred", strokeWidth: 3 }), Plot.lineY(data, { x: "threshold", y: "fn_cost", tip: true, stroke: "darkblue", strokeWidth: 3 }), Plot.lineY(data, { x: "threshold", y: "total_savings", tip: true, stroke: "green", strokeWidth: 3 }), Plot.circle(["fp_cost"], { x: () => data[i].threshold, y: (d) => data[i][d], fill: "lightpink", stroke: "black", r: 7 }), Plot.circle(["fn_cost"], { x: () => data[i].threshold, y: (d) => data[i][d], fill: "lightblue", stroke: "black", r: 7 }), Plot.circle(["total_savings"], { x: () => data[i].threshold, y: (d) => data[i][d], fill: "lightgreen", stroke: "black", strokeWidth: "4", r: 10 }), Plot.ruleY([0], { strokeWidth: 5 }) ] })2e:T4e7,From the graph we can see that, the lower the threshold, the more false positives our model will predict. That means that components will be replaced far more frequently than they need to be, cutting repair costs (blue line) to 0, but causing higher replacement costs (red line) because of unnecessary replacements. The green line shows the total amount of money saved at each threshold point. As the threshold is raised, the number of false positives will go down (saving money on unnecessary replacements), but at some point a couple of failures will go unpredicted, raising the repair costs. The higher the threshold, the more certain we can be that predicted failures will be true positives, but at the expense of missing some failures. The lower the threshold, the more false positives we will get, causing unnecessary replacements. There is a "sweet spot" where we are saving the maximum amount of money, i.e. the best balance between unnecessary replacements and missed failures - a threshold of 0.3 (where the green line is at its peak) for this example. _Here we should note that these costs are separate from scheduled maintenance costs._ For now we are only working towards minimising machine failures irrespective of scheduled maintenance.2f:T466,plt4 = Plot.plot({ width: 1000, y: { grid: true, label: "(£)", domain:[-500000, 1100000], tickFormat: ".2s"}, x: { grid: true, label: "Threshold", domain: [0, 1]}, color: {domain: ["Cost (FP)", "Cost (FN)", "Total Savings"], range: ["darkred", "darkblue", "green"], legend: true}, marks: [ Plot.lineY(data, { x: "threshold", y: "fp_cost", tip: true, stroke: "darkred", strokeWidth: 3 }), Plot.lineY(data, { x: "threshold", y: "fn_cost", tip: true, stroke: "darkblue", strokeWidth: 3 }), Plot.lineY(data, { x: "threshold", y: "total_savings", tip: true, stroke: "green", strokeWidth: 3 }), Plot.circle(["fp_cost", "fn_cost"], { x: () => data[30].threshold, y: (d) => data[30][d], fill: "white", stroke: "black", r: 7 }), Plot.circle(["total_savings"], { x: () => data[30].threshold, y: (d) => data[30][d], fill: "gold", stroke: "black", r: 10 }), Plot.ruleY([0], { strokeWidth: 5 }) ] })30:T50a,plt5 = Plot.plot({ width: 1000, y: { grid: true, label: "(£)", // domain:[-500000, 1100000], tickFormat: ".2s"}, x: { grid: true, label: "Threshold", domain: [0, 1]}, color: {domain: ["Cost (FP)", "Cost (FN)", "Total Savings"], range: ["darkred", "darkblue", "green"], legend: true}, marks: [ Plot.lineY(data1, { x: "threshold", y: "fpCost", tip: true, stroke: "darkred", strokeWidth: 3 }), Plot.lineY(data1, { x: "threshold", y: "fnCost", tip: true, stroke: "darkblue", strokeWidth: 3 }), Plot.lineY(data1, { x: "threshold", y: "totalSavings", tip: true, stroke: "green", strokeWidth: 3 }), Plot.dot(["fpCost"], { x: () => data1[ind].threshold, y: (d) => data1[ind][d], fill: "lightpink", stroke: "black", r: 7 }), Plot.dot(["fnCost"], { x: () => data1[ind].threshold, y: (d) => data1[ind][d], fill: "lightblue", stroke: "black", r: 7 }), Plot.dot(["totalSavings"], { x: () => data1[ind].threshold, y: (d) => data1[ind][d], fill: "lightgreen", stroke: "black", r: 10 }), Plot.ruleY([0], { strokeWidth: 5 }) ] })31:[] 2b:{"id":"cc81abf828c18569","slug":null,"title":"Predictive Maintenance","thumbnail":"d221e35f493020b6970cd7e7885000a26ef5f4ff5142b8738ca277d663ba06bd","access_level":"public","create_time":"$D2023-08-16T11:57:47.853Z","update_time":"$D2023-09-25T15:55:43.929Z","view_time":null,"first_public_time":"$D2023-09-18T07:49:56.496Z","has_unpublished_edits":null,"fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"282800d6068491b9","login":"nico2-ws","name":"Nico's Workspace","avatar_url":"https://avatars-next.observableusercontent.com/83e98629181134bc8f825e00391decff65898c500884f07b663aa7104d9f7718","tier":"starter_2024"},"version":1743,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"89b79a6d8fd25a28","login":"nico-dsq","name":"Nico","avatar_url":"https://avatars-next.observableusercontent.com/7130f307192d50db50b7b1b472a136e962722c8dea9c43d2c04441f74bf16bfa","tier":null}],"members":[{"type":"team","id":"282800d6068491b9","login":"nico2-ws","name":"Nico's Workspace","avatar_url":"https://avatars-next.observableusercontent.com/83e98629181134bc8f825e00391decff65898c500884f07b663aa7104d9f7718","tier":"starter_2024","role":"editor"},{"type":"individual","id":"89b79a6d8fd25a28","login":"nico-dsq","name":"Nico","avatar_url":"https://avatars-next.observableusercontent.com/7130f307192d50db50b7b1b472a136e962722c8dea9c43d2c04441f74bf16bfa","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"Predictive Maintenance","cells":[{"id":0,"value":"# Predictive Maintenance","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":8,"value":"Heraclitus said:\n\n_\"To do the same thing over and over again is not only boredom: it is to be controlled by rather than to control what you do.\"_\n\nMaintaining machines properly, keeping them running optimally and doing it as cost-efficiently as possible has been a key part of production and business thinking for a long, long time. \n\n**\"For large mechanical systems, maintenance can account for over 30% of the operating cost. A careful optimisation of these costs may be necessary in many cases.\"\n**\n\n*_A. Birolini, Reliability Engineering: Theory and Practice (New York, Springer-Verlag Berlin Heidelberg, 2004), p. 122_\n\nThinking differently about \"the way we have always done things\" and using the technologies and tools at our disposal to optimise maintenance is at the heart of what we tried to achieve with this piece of work.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":12,"value":"## Context","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":21,"value":"Imagine you are the Maintenance Director at the BW Manufacturing Company (BWMC), a company that produce parts used in aeroplane engines. You are in charge of making sure the 100 machines producing these parts keep running efficiently as well as managing the team of engineers that do the maintenance.\n\nYou are tasked to cut maintenance costs by minimising unscheduled repairs (i.e. when a machine breaks). Replacing a component is much cheaper than repairing a machine after it has broken. The aim of this work is to minimise machine downtime and repair costs by using machine learning to _predict when machines will fail_ in order to replace components _before_ machine failures occur.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":31,"value":"## Calculating costs","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":811,"value":"Cost to replace (${tex`C_{replace}`}) = £200","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":814,"value":"Cost to repair (${tex`C_{repair}`}) = £7000","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1362,"value":"Assuming these values we can work out the average cost per machine per year.\n\nIn the last year there has been a total of 1461 scheduled component replacements and 702 machine failures across 100 machines, which equates to an average cost of £52062 per machine per year (£2922 for scheduled maintenance and £49140 for unscheduled repair).\n\n

${tex`((C_{replace} * 1461) + (C_{repair} * 702))/100 = £52062`}

","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1355,"value":"We will discuss the machine learning principles and approach a bit later, but for now we only need to take note of is this: \nThe machine learning model will use telemetry readings and other features to give the probability that a machine will fail in the next 24 hours. We use a confusion matrix to evaluate each prediction:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1348,"value":"confusionMatrix = d3.image(\"https://play-ml.datasparq.ai/api/data/predictive-maintenance/confusion_matrix_image.png\", {width:500})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":864,"value":"False negative (${tex`FN`}) - When the model **wrongly** predicts a machine _will not fail_ (repair cost).\n\nFalse positive (${tex`FP`}) - When the model **wrongly** predicts a machine _will fail_ (replace cost).\n\nTrue negative (${tex`TN`}) - When the model **correctly** predicts a machine _will not fail_ (0 cost).\n\nTrue positive (${tex`TP`}) - When the model **correctly** predicts a machine _will fail_ (replace cost).\n\n**We want (${tex`FN`}) and (${tex`FP`}) to be as low as possible, with (${tex`TP`}) as high as possible.**\n\nThe model outputs probability of failure. The **threshold** is our way to decide the tipping point for the model's prediction either way (_failure-within-24-hours_ or _non-failure-within-24-hours_).\n\nAs the threshold value changes, the cost implications associated with false negatives, true positives and false positives are shown in the graphs below.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":595,"value":"viewof i = Inputs.range([0, data.length - 1], { step: 1, label: \"Threshold %\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1378,"value":"\n \n \n \n

\n ${plt}\n

\n ${plt2}\n

\n \n","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1422,"value":"\n \n \n \n

\n ${plt3}\n

\n \n","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":582,"value":"$2d","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1382,"value":"plt = Plot.plot({\n height: 350,\n y: { grid: true,\n label: \"False Positives\",\n domain: [0, 5500]},\n x: { grid: true, \n label: \"Threshold\", \n domain: [0, 1]},\n marks: [\n Plot.lineY(data, {\n x: \"threshold\",\n y: \"false_positive\",\n tip: true,\n stroke: \"darkred\",\n strokeWidth: 3\n }),\n Plot.circle([\"false_positive\"], {\n x: () => data[i].threshold,\n y: (d) => data[i][d],\n fill: \"lightpink\",\n stroke: \"black\",\n r: 7\n }),\n Plot.ruleY([0], { strokeWidth: 5 })\n ]\n })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1392,"value":"plt2 = Plot.plot({\n height: 350,\n y: { grid: true,\n label: \"False Negatives\",\n domain: [0, 100]},\n x: { grid: true, \n label: \"Threshold\", \n domain: [0, 1]},\n marks: [\n Plot.lineY(data, {\n x: \"threshold\",\n y: \"false_negative\",\n tip: true,\n stroke: \"darkblue\",\n strokeWidth: 3\n }),\n Plot.circle([\"false_negative\"], {\n x: () => data[i].threshold,\n y: (d) => data[i][d],\n fill: \"lightblue\",\n stroke: \"black\",\n r: 7\n }),\n Plot.ruleY([0], { strokeWidth: 5 })\n ]\n })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":662,"value":"$2e","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1447,"value":"\n \n \n \n

\n ${plt4}\n

\n \n","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":664,"value":"$2f","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":736,"value":"## Limitations","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":738,"value":"In the above example the costs are set to £200 for replacement and £7000 for repair. What would happen if these numbers change? How will the model perform if these prices were different?\n\nLet us imagine that component prices went up and new technologies are bringing repair costs down. The impact of these changes on how well the model performs can be shown in the graph below.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":750,"value":"viewof replace = Inputs.range([100, 1000], { step: 100, label: \"Replace cost (£)\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":766,"value":"viewof repair = Inputs.range([1000, 10000], { step: 1000, label: \"Repair cost (£)\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":932,"value":"viewof ind = Inputs.range([0, data.length - 1], { step: 1, label: \"Threshold %\" })","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1450,"value":"\n \n \n \n

\n ${plt5}\n

\n \n","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":902,"value":"$30","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1293,"value":"As you can see, the model performs better when there is a bigger difference between replacement and repair costs. \n\nWhat you will also notice is that, as the repair and replacement costs change, the optimal threshold also changes.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1217,"value":"## Maths","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1219,"value":"Below are the logic and mathematics to explain how we got to the numbers we used.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1221,"value":"The cost (${tex`C`}) of false negative predictions (${tex`FN`}) --i.e. breakdowns that were missed-- can be calculated by multiplying the repair cost by the number of these occurrences.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":820,"value":"

${tex`C_{FN} = FN * C_{repair}`}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":836,"value":"False positive predictions (${tex`FP`}) result in unnecessary replacements.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":834,"value":"

${tex`C_{FP} = FP * C_{replace}`}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":842,"value":"True positive predictions (${tex`TP`}) result in replacements (at a cost of £200) instead of repair (at a cost of £7000) with a saving (${tex`S`}) of £6800 per event.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":823,"value":"

${tex`C_{TP} = TP * C_{replace}`}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":845,"value":"

${tex`S_{TP} = (TP * C_{repair}) - (TP * C_{replace})`}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":854,"value":"It then follows that:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":858,"value":"

${tex`S_{total} = S_{TP} - C_{FP} - C_{FN}`}","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":470,"value":"## Logistic Regression Model","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":472,"value":"We trained a logistic regression model to predict the probability of whether a machine will fail within the next 24 hours by using telemetry data, error logs, machine age and machine model (the features). \n\nSimply put, the logistic regression model predicts the likelihood between 0 and 1 - 0 being 0% chance of a machine failing, 1 being 100% certainty of a machine failing - given the features.\n\nLogistic regression is based on the Sigmoid Function:\n\n

${tex`Sigmoid(t) = \\dfrac{1}{1+e^{-t}}`}

\n\nwhere ${tex`t`} represents a linear combination of input features and their associated weights:\n\n

${tex`t = \\beta_{0} + \\beta_{1}x_{1} + \\beta_{2}x_{2} + ... + \\beta_{p}x_{p}`}

","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1705,"value":"We calculate the value of ${tex`Sigmoid(t)`} -which is the probability of a machine failing- and compare it to the threshold of our choosing. If the predicted likelihood is above the threshold, the model will predict machine failure. If the predicted likelihood is below the threshold, the model will predict non-failure.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1712,"value":"You may be wondering why we chose a Logistic Regression model. The short answer is that it is the simplest classification model with the best _Area Under the Curve_ (AUC) score. Other models are more complicated, but did not perform much better.\n\nTo calculate the AUC score we look at the _Receiver Operating Characterist_ (ROC) Curve. This is a graphical representation of a model's performance across different thresholds (the same thresholds we used in the above graphs) plotting the False Positive Rate vs. the True Positive Rate. Below you can see our model's ROC Curve:","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1464,"value":"\n \n \n \n

\n ${aucplt}\n

\n \n","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":1715,"value":"The AUC score the total area under the ROC Curve (the nicely blue/purple-shaded area). The AUC score is a way to compare and evaluate different binary classification models. It gives us a number between 0 and 1 that tells us how well the model can discriminate between two classes (in our case, whether a machine will fail or not). The closer the number is to 1, the better the model performs.\n\nThe AUC score for our model is 0.9578 which is not much different from the other, more complicated models we compared with scores of 0.9638 for the XGBoost Classifier and 0.969 for Gradient Boost. I believe Occam had some thoughts on this while shaving (see Occam's Razor).\n\nIn the end, the simplest model with a decent AUC score was picked, which is the Logistic Regression model. As Isaac Newton said: \n_\"Nature is pleased with simplicity.\"_ And if simplicity is good enough for nature, it's good enough for me.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":478,"value":"aucplt = Plot.plot({\n y: {grid: true, label: \"True Positive Rate\"},\n x: {grid: true, label: \"False Positive Rate\"},\n marks: [\n () => htl.svg`\n \n \n \n \n `,\n Plot.area(\n aucData,\n { x1: \"1\", y1:\"2\", x2:\"1\", y2:0, fill: \"url(#gradient)\"}\n ),\n Plot.lineY(\n aucData,\n { x: \"1\", y: \"2\", stroke: \"darkblue\", strokeWidth: 3, tip: true }\n )\n ]\n})\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1601,"value":"aucData = d3.csv(\"https://play-ml.datasparq.ai/api/data/predictive-maintenance/test_metrics.csv\", d3.autoType)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":572,"value":"// data = FileAttachment(\"df_scores@2.csv\").csv({ typed: true })\ndata = d3.csv(\"https://play-ml.datasparq.ai/api/data/predictive-maintenance/df_scores.csv\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1540,"value":"d3 = require(\"d3@6\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1542,"value":"data1 = data.map(d => ({...d, fpCost: (d[\"false_positive\"] * replace), fnCost: (d[\"false_negative\"]*repair), totalSavings: ((d[\"true_positive\"]*repair)-(d[\"true_positive\"]*replace)-(d[\"false_positive\"]*replace)-(d[\"false_negative\"]*repair))}))\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W31","stdlib":"1","titling":"auto"}} 2c:"$2b:body"