2c:null 2d:["$","title",null,{"children":"Math for journalists | Naël Shiab | Observable"}] 32:T4a2,{ const width = 400; const domain = d3.extent(geojsonfsa.features, (d) => d.properties.values); const canadaMap = Plot.plot({ width, projection: { type: "conic-conformal", rotate: [100, -60], domain: geojsoncanada }, color: { type: colorScale, n: 5, scheme: "blues", legend: true, width: width - 75, label: "Average income per FSA (2021 Census)", tickFormat: (d) => j.formatNumber(d, { round: true, prefix: "$" }) }, marks: [ Plot.geo(rewind(geojsonfsa), { fill: (d) => d.properties.value, stroke: (d) => d.properties.value }) ] }); const montrealMap = Plot.plot({ width, projection: { type: "mercator", domain: rewind(montrealcma) }, color: { type: colorScale, n: 10, scheme: "blues" }, marks: [ Plot.geo(rewind(geojsonfsa), { fill: (d) => d.properties.value, stroke: (d) => d.properties.value }) ] }); return htl.html`

Canada

${canadaMap}

Montreal metropolitan area

${montrealMap}

`; }33:T646,{ const minRaw = d3.min(torontoTemp, (d) => d.t); const maxRaw = d3.max(torontoTemp, (d) => d.t); const scaleMin = d3.scaleLinear([1, maxDaysRollingAverage], [minRaw, 7.85]); const scaleMax = d3.scaleLinear([1, maxDaysRollingAverage], [maxRaw, 9.85]); const minValue = scaleMin(k); const maxValue = scaleMax(k); let maxDate = new Date("2021-12-31"); let minDate; if (anchorRollingAverage === "end") { minDate = new Date(new Date("1900-01-01").getTime() + (k - 1) * 86_400_000); maxDate = new Date("2021-12-31"); } else if (anchorRollingAverage === "middle") { minDate = new Date( new Date("1900-01-01").getTime() + ((k - 1) * 86_400_000) / 2 ); maxDate = new Date( new Date("2021-12-31").getTime() - ((k - 1) * 86_400_000) / 2 ); } else if (anchorRollingAverage === "start") { minDate = new Date("1900-01-01"); maxDate = new Date(new Date("2021-12-31").getTime() - (k - 1) * 86_400_000); } return Plot.plot({ y: { domain: [minValue, maxValue] }, x: { domain: [minDate, maxDate] }, color: { scheme: "BuRd" }, marks: [ Plot.dot(torontoTemp, { x: "time", y: "t", fill: "t", opacity: 1, r: 1, filter: (d) => d.t >= minValue && d.t <= maxValue && d.time >= minDate && d.time <= maxDate }), Plot.line( torontoTemp, Plot.windowY( { k, strict: true, anchor: anchorRollingAverage }, { x: "time", y: "t", stroke: "grey" } ) ) ] }); }34:T40d,{ if (yVariable === "Coffee sales") { return md`The correlation between the **coffee sales** and the **temperature** is **${salesCorrelation.toFixed( 2 )}**. This is close to 0, meaning the two variables **are not related**. People don't buy more or less coffee depending on the temperature. They buy coffee all the time!`; } else if (yVariable === "Hot soup sales") { return md`The correlation between the **hot soup sales** and the **temperature** is **${salesCorrelation.toFixed( 2 )}**. This is close to -1, meaning the two variables **are related**. When one is going down (the temperature), the other is likely to go up (the hot soup sales).`; } else if (yVariable === "Ice cream sales") { return md`The correlation between the **ice cream sales** and the **temperature** is **${salesCorrelation.toFixed( 2 )}**. This is close to 1, meaning the two variables **are related**. When one is going up (the temperature), the other is likely to go up as well (the ice cream sales).`; } }35:T527,{ const lineData = shopSalesLR .map((d) => { return { x: d["Temperature"], y: d["Temperature"] * salesLR.m + salesLR.b }; }) .filter((d) => d.y > 0) .sort((a, b) => (a.x > b.x ? 1 : -1)); const chart = Plot.plot({ insetLeft: 10, insetBottom: 10, x: { domain: d3.extent(shopSalesLR, (d) => d["Temperature"]), tickFormat: (d) => `${d} C` }, y: { domain: [0, d3.max(shopSalesLR, (d) => d[yVariableLR])] }, marks: [ Plot.dot(shopSalesLR, { x: "Temperature", y: yVariableLR, fill: "Temperature", opacity: 0.5 }), Plot.line(lineData, { x: "x", y: "y", stroke: "black", strokeDasharray: 4 }) ] }); return htl.html`

y = ${salesLR.m.toFixed( 3 )}*x + ${salesLR.b.toFixed(3)}

y = 0 when x = ${startLR.toFixed( 3 )}

r2 = ${rSquared.toFixed( 4 )}

${chart}

`; }36:T463,{ if (yVariableLR === "Coffee sales") { return md`The **r-squared value** of the **coffee sales** over the **temperature** is **${rSquared.toFixed( 4 )}**. This is close to 0, meaning there is **no linear trend**.`; } else if (yVariableLR === "Hot soup sales") { return md`The **r-squared value** of the **hot soup sales** over the **temperature** is **${rSquared.toFixed( 2 )}**. This is close to 1, meaning there is ** a linear trend**. On average, **${Math.round( -salesLR.m )} more soup** is sold each time it gets **1 C colder**. This trend is visible when the temperature is **below ${Math.round( startLR )} C**.`; } else if (yVariableLR === "Ice cream sales") { return md`The **r-squared value** of the **ice cream sales** over the **temperature** is **${rSquared.toFixed( 2 )}**. This is close to 1, meaning there is ** a linear trend**. On average, **${Math.round( salesLR.m )} more ice creams** are sold each time it gets **1 C warmer**. This trend is visible when the temperature is above ${Math.round( startLR )} C.`; } }37:T521,The z-score is a number associated with each value, telling us how far the value is from the mean. The z-score unit is the [standard deviation](https://en.wikipedia.org/wiki/Standard_deviation), also known as sigma (σ). The z-score is interesting because we know that for random values following a [normal distribution](https://en.wikipedia.org/wiki/Normal_distribution): - 68.27% of values should be within one σ - 95.45% of values should be within two σ - 99.73% of values should be within three σ - 99.9937% of values should be within four σ - 99.9999998% of values should be within five σ So we can use the z-score to determine if a value is exceptional because values with: - one σ or more have one chance in 3 to happen - two σ or more have one chance in 22 to happen - three σ or more have one chance in 370 to happen - four σ or more have one chance in 15,787 to happen - five σ or more have one chance in 1,744,278 to happen Of course, this is true only if we can assume that our values have a normal distribution, easily identifiable as a [bell shape](https://en.wikipedia.org/wiki/Bell-shaped_function). In our case, we can assume that the temperatures for a given month in Vancouver are normally distributed. Update the inputs below and see which data points are considered exceptional.38:T544,{ const maxTemp = d3.max(zScores, (d) => d[variableZ]); const minTemp = d3.min(zScores, (d) => d[variableZ]); const rules = [ { x: meanZ, text: `0 σ` }, { x: meanZ + stdDevZ, text: "1 σ" }, { x: meanZ + stdDevZ * 2, text: "2 σ" }, { x: meanZ + stdDevZ * 3, text: "3 σ" }, { x: meanZ + stdDevZ * 4, text: "4 σ" }, { x: meanZ + stdDevZ * 5, text: "5 σ" }, { x: meanZ - stdDevZ, text: "-1 σ" }, { x: meanZ - stdDevZ * 2, text: "-2 σ" }, { x: meanZ - stdDevZ * 3, text: "-3 σ" }, { x: meanZ - stdDevZ * 4, text: "-4 σ" }, { x: meanZ - stdDevZ * 5, text: "-5 σ" } ].filter((d) => d.x >= minTemp && d.x <= maxTemp); return Plot.plot({ height: 425, marginTop: 20, x: { tickFormat: (d) => `${d} C` }, marks: [ Plot.dotX( zScores, Plot.dodgeY({ x: variableZ, r: 1.2, fill: (d) => { if (d.zScore >= sigmaZ) { return "red"; } else if (d.zScore <= -sigmaZ) { return "blue"; } else { return "lightGrey"; } } }) ), rules.map((d) => Plot.ruleX([d.x], { strokeDasharray: 4, stroke: "lightgrey" }) ), Plot.text(rules, { x: "x", text: "text", fill: "grey", dy: -200 }) ] }); }39:T42f,With a population of **${j.formatNumber(populationParty.length)}**, you would need to survey **${j.formatNumber(sampleSizeProportions)}** randomly selected people to achieve a **${marginOfErrorSampleSizeProportion}% margin of error** at a **${confidenceSampleSizeProportions}% confidence level**. The randomly selected individuals are highlighted below and here are the estimates based on their answers: * ${highlight(`${Math.round(libEstimate)}% ± ${marginOfErrorSampleSizeProportion}%`, "red")} would vote for the ${highlight("Liberal party", "red")} * ${highlight(`${Math.round(conEstimate)}% ± ${marginOfErrorSampleSizeProportion}%`, "blue")} would vote for the ${highlight("Conservative party", "blue")} * ${highlight(`${Math.round(ndpEstimate)}% ± ${marginOfErrorSampleSizeProportion}%`, "orange")} would vote for the ${highlight("New Democratic Party", "orange")} To see the results from a new random sample, click the button below. You'll notice that the estimates remain close to the true proportions, demonstrating the reliability of this sample size.3a:T592,Plot.plot({ title: "Vote estimates (bars) compared to vote intentions (dots)", grid: true, marginLeft: 60, marginBottom: 40, color: { legend: true, domain: ["CON", "LIB", "NDP"], range: ["blue", "red", "orange"] }, x: { domain: [0, 75], tickFormat: (d) => d + "%" }, marks: [ Plot.barX( [ { party: "LIB", percentage1: libEstimate - marginOfErrorSampleSizeProportion, percentage2: libEstimate + marginOfErrorSampleSizeProportion }, { party: "CON", percentage1: conEstimate - marginOfErrorSampleSizeProportion, percentage2: conEstimate + marginOfErrorSampleSizeProportion }, { party: "NDP", percentage1: Math.max( ndpEstimate - marginOfErrorSampleSizeProportion, 0 ), percentage2: ndpEstimate + marginOfErrorSampleSizeProportion } ], { y: "party", x1: "percentage1", x2: "percentage2", fill: "party", opacity: 0.15 } ), Plot.dotX( [ { party: "LIB", percentage: libProportion }, { party: "CON", percentage: conProportion }, { party: "NDP", percentage: npdProportion } ], { y: "party", x: "percentage", fill: "party", r: 8, sort: { y: "-x" } } ) ] })3b:T426,Olympians competing in **${sportZTest}** are: - **${zTestWeight.sampleMean < zTestWeight.populationMean ? "lighter" : "heavier"}** (${j.round(zTestWeight.sampleMean, {decimals: 1})} kg) than average (${j.round(zTestWeight.populationMean, {decimals: 1})} kg) ${zTestWeight.pValue < 0.05 ? "and" : "but"} this is **${zTestWeight.pValue < 0.05 ? "statistically significant" : "not statistically significant"}** (p-value: ${formatPValue(zTestWeight.pValue)}) - **${zTestHeight.sampleMean < zTestHeight.populationMean ? "shorter" : "taller"}** (${j.round(zTestHeight.sampleMean, {decimals: 2})} m) than average (${j.round(zTestHeight.populationMean, {decimals: 2})} m) ${zTestHeight.pValue < 0.05 ? "and" : "but"} this is **${zTestHeight.pValue < 0.05 ? "statistically significant" : "not statistically significant"}** (p-value: ${formatPValue(zTestHeight.pValue)}) The chart below highlights the ${highlight(j.formatNumber(zTestHeight.sampleSize) + " " + sportZTest + " athletes", "purple")} among the total ${j.formatNumber(zTestHeight.populationSize)} athletes.3c:T5f2,So, are Olympians significantly taller or shorter than the average Canadian? It's hard to decide just by looking at the chart. We could calculate the average height for our sample of Olympians and see if it's different from the Canadian average, but how would we know if that difference is significant? And how would we account for the spread in the athletes' heights? Let's use a statistical test, more specifically a one-sample t-test. The test assumes that there should be **no difference** between the averages, but it calculates the probability of seeing a difference like the one you found or larger by **random chance**. This probability is called the **p-value** and when it is less than 5% (or 0.05), we can say it's **statistically significant**. Here are the results for **${sportTTest}** athletes: - Olympian men are ${j.round(tTestMaleHeight.sampleMean, {decimals: 2})} m tall on average while Canadian men are ${tTestMaleHeight.hypothesizedMean} m. This gives us a p-value of ${formatPValue(tTestMaleHeight.pValue)}, meaning the difference is **${tTestMaleHeight.pValue < 0.05 ? "statistically significant" : "not statistically significant"}**. - Olympian women are ${j.round(tTestFemaleHeight.sampleMean, {decimals: 2})} m tall on average while Canadian women are ${tTestFemaleHeight.hypothesizedMean} m. This gives us a p-value of ${formatPValue(tTestFemaleHeight.pValue)}, meaning the difference is **${tTestFemaleHeight.pValue < 0.05 ? "statistically significant" : "not statistically significant"}**.3d:T511,It's not easy to determine if there is a significant difference in the weights of the athletes just by looking at the chart. We could calculate the average for each sport, but then how would we decide if the difference between the two is significant? And how to account for the spread of the weights? To help us, we can use a statistical test, more specifically a two-sample t-test. The test assumes that there should be **no difference** between the averages, but it calculates the probability of seeing a difference like the one you found or larger by **random chance**. This probability is called the **p-value** and when it is less than 5% (or 0.05), we can say it's **statistically significant**. And here's the result when comparing ${sportTwoSampleTTestGroup1} and ${sportTwoSampleTTestGroup2} athletes: - The average weight of the first group (${sportTwoSampleTTestGroup1}) is **${j.round(twoSampleTTestDataResult.group1Mean, {decimals: 1})} kg** compared to **${j.round(twoSampleTTestDataResult.group2Mean, {decimals: 1})} kg** for the second group (${sportTwoSampleTTestGroup2}). The p-value is ${formatPValue(twoSampleTTestDataResult.pValue)}, which means the difference is **${twoSampleTTestDataResult.pValue < 0.05 ? "statistically significant" : "not statistically significant"}**.3e:T63e,Depending on the numbers displayed on the table, it's not always easy to say if there is a big difference between the **observed** and **expected** values. To help us, we can use a statistical test (more specifically a chi-squared goodness of fit test). The test assumes that there should be **a perfect fit** between our observed and expected values, but it calculates the probability of seeing a difference as big as or bigger than the one we have by **random chance**. This probability is called the **p-value** and when it is less than 5% (or 0.05), we can say it's **statistically significant**. And here's the result of the test: - The p-value is **${formatPValue(goodnessOfFitResult.pValue)}**, which means the differences are **${goodnessOfFitResult.pValue < 0.05 ? "statistically significant" : "not statistically significant"}**. ${goodnessOfFitResult.pValue < 0.05 ? "Based on this data, we can say that the elected representatives are not representative of the overall population." : "Based on this data, the differences between the elected representatives and the overall population could be random."} Note that this test works with counts (the actual number of people), not percentages, because the sample size is a crucial part of the calculation. The math behind it is a bit complex to explain, but for those who code in Javascript, you can perform this calculation using the [performChiSquaredGoodnessOfFitTest](https://jsr.io/@nshiab/journalism/doc/~/performChiSquaredGoodnessOfFitTest) function from the open-source [journalism library](https://github.com/nshiab/journalism).3f:T914,function randomizeGoodnessOfFitData(data, variationFactor = 0.6) { const totalSum = 200; // 1. Clamp the variation factor to be between 0 and 1 for safety. const clampedVF = Math.max(0, Math.min(1, variationFactor)); // 2. Generate new "weights" by randomly perturbing the original values. // This ensures the new distribution is based on the original. const perturbedWeights = data.map((item) => { // Determine a random change multiplier. For a variationFactor of 0.2, // this will be a random number between 0.8 and 1.2. const randomPerturbation = (Math.random() - 0.5) * 2 * clampedVF; const newWeight = item.observed * (1 + randomPerturbation); // Ensure the weight is not negative (can happen with high variation). return Math.max(0, newWeight); }); // 3. From here, the process is the same as before: normalize these new // weights and convert them to integers that sum precisely to 100. const weightsSum = perturbedWeights.reduce((sum, val) => sum + val, 0); // Edge case: if all weights become 0, the sum would be 0, causing a division by zero. // In this unlikely event, we fall back to a purely random distribution to avoid an error. if (weightsSum === 0) { // Generate purely random values to use as weights instead const fallbackWeights = data.map(() => Math.random()); const fallbackSum = fallbackWeights.reduce((s, v) => s + v, 0); const normalizedFallback = fallbackWeights.map( (w) => (w / fallbackSum) * totalSum ); perturbedWeights.splice(0, data.length, ...normalizedFallback); } const normalizedFloats = perturbedWeights.map( (val) => (val / weightsSum) * totalSum ); const finalIntegers = normalizedFloats.map((val) => Math.floor(val)); let currentSum = finalIntegers.reduce((sum, val) => sum + val, 0); const remainders = normalizedFloats.map((val, index) => ({ index: index, remainder: val - Math.floor(val) })); remainders.sort((a, b) => b.remainder - a.remainder); let difference = totalSum - currentSum; for (let i = 0; i < difference; i++) { const indexToIncrement = remainders[i].index; finalIntegers[indexToIncrement]++; } // 4. Create and return the new data array. return data.map((item, index) => ({ ...item, observed: finalIntegers[index] })); }40:Td86,/** * Generates a dataset where the sum of counts is 100, using a simulation * method to produce more varied and realistic results for statistical testing. * * @param {object} [options={}] - Configuration options. * @param {string} [options.biasType='random'] - Can be 'random', 'none', 'subtle', or 'strong'. * @returns {Array