Public
Edited
Apr 29
// Import libraries
import {vl} from "@vega/vega-lite-api"
// Load the dataset
data = FileAttachment("smart_grid_dataset_4.csv").csv({typed: true})
// Display a sample of the data
Inputs.table(data.slice(0, 10))
// Convert data to appropriate types
processedData = data.map(d => ({
"Timestamp": new Date(d["Timestamp"]),
"Voltage (V)": +d["Voltage (V)"],
"Current (A)": +d["Current (A)"],
"Power Consumption (kW)": +d["Power Consumption (kW)"],
"Reactive Power (kVAR)": +d["Reactive Power (kVAR)"],
"Solar Power (kW)": +d["Solar Power (kW)"],
"Wind Power (kW)": +d["Wind Power (kW)"],
"Grid Supply (kW)": +d["Grid Supply (kW)"],
"Temperature (C)": +d["Temperature (C)"],
"Humidity (%)": +d["Humidity (%)"],
"Electricity Price (USD/kWh)": +d["Electricity Price (USD/kWh)"]
}))
powerGenerationTrend = vl.layer(
vl.markLine()
.data(processedData)
.encode(
vl.x().fieldT("Timestamp").title("Time"),
vl.y().fieldQ("Solar Power (kW)").title("Power (kW)"),
vl.color().datum("Solar Power").title("Power Source")
),
vl.markLine()
.encode(
vl.x().fieldT("Timestamp"),
vl.y().fieldQ("Wind Power (kW)"),
vl.color().datum("Wind Power")
),
vl.markLine()
.encode(
vl.x().fieldT("Timestamp"),
vl.y().fieldQ("Grid Supply (kW)"),
vl.color().datum("Grid Supply")
)
)
.width(800)
.height(400)
.title("Power Generation Trend Over Time")
.render();
zeroValues = processedData.filter(d => d["Power Consumption (kW)"] === 0).length
zeroPercentage = (zeroValues / processedData.length * 100).toFixed(2)
inconsistentZeros = processedData.filter(d =>
d["Power Consumption (kW)"] === 0 &&
d["Voltage (V)"] > 0 &&
d["Current (A)"] > 0
).length
inconsistentPercentage = (inconsistentZeros / processedData.length * 100).toFixed(2)
// Visualize distribution of Power Consumption values
powerDistribution = vl.markBar()
.data(processedData)
.encode(
vl.x().fieldQ("Power Consumption (kW)").bin({maxbins: 50}).title("Power Consumption (kW)"),
vl.y().count().title("Count of Records"),
vl.tooltip([
{field: "Power Consumption (kW)", bin: true, type: "quantitative"},
{aggregate: "count", type: "quantitative"}
])
)
.width(600)
.height(300)
.title(`Distribution of Power Consumption Values (${zeroPercentage}% zeros, ${inconsistentPercentage}% inconsistent)`)
.render()
// Function to correctly calculate power consumption
function calculateActualPower(d) {
// If power is already recorded and non-zero, keep it
if (d["Power Consumption (kW)"] > 0) {
return d["Power Consumption (kW)"];
}
// If voltage and current are valid, calculate power (V × I / 1000 for kW)
if (d["Voltage (V)"] > 0 && d["Current (A)"] > 0) {
return (d["Voltage (V)"] * d["Current (A)"]) / 1000;
}
// Otherwise keep as zero
return 0;
}
// Apply correction to data
correctedData = processedData.map(d => {
return {
...d,
"Power Consumption (kW)": calculateActualPower(d),
"Original Power Consumption (kW)": d["Power Consumption (kW)"]
};
});
// Visualize corrected power consumption
correctedDistribution = vl.markBar()
.data(correctedData)
.encode(
vl.x().fieldQ("Power Consumption (kW)").bin({maxbins: 50}).title("Power Consumption (kW)"),
vl.y().count().title("Count of Records"),
vl.tooltip([
{field: "Power Consumption (kW)", bin: true, type: "quantitative"},
{aggregate: "count", type: "quantitative"}
])
)
.width(600)
.height(300)
.title("Distribution of Corrected Power Consumption Values")
.render()
// Visualize power consumption anomalies
anomalyPlot = vl.markPoint()
.data(correctedData)
.transform([
{
calculate: "datum['Power Consumption (kW)'] <= 2 || datum['Power Consumption (kW)'] >= 11 ? 'Anomaly' : 'Normal'",
as: "anomalyStatus"
}
])
.encode(
vl.x().fieldT("Timestamp").title("Time"),
vl.y().fieldQ("Power Consumption (kW)").scale({domain: [0, 13]}),
vl.color().fieldN("anomalyStatus").scale({
domain: ["Normal", "Anomaly"],
range: ["#86c5da", "#ff5a5a"]
}),
vl.opacity().value(0.5),
vl.tooltip([
{field: "Timestamp", type: "temporal"},
{field: "Power Consumption (kW)", type: "quantitative"},
{field: "anomalyStatus", type: "nominal"}
])
)
.width(800)
.height(400)
.title("Power Consumption Over Time (Anomalies Highlighted)")
.render()
// Visualize voltage anomalies
voltageAnomalyPlot = vl.layer(
vl.markBoxplot().encode(
vl.x().fieldN("").title(""),
vl.y().fieldQ("Voltage (V)")
),
vl.markPoint({size: 30, color: "red"})
.transform([{filter: "datum['Voltage (V)'] < 200 || datum['Voltage (V)'] > 250"}])
.encode(
vl.x().fieldN("").title(""),
vl.y().fieldQ("Voltage (V)")
)
)
.data(correctedData)
.width(250)
.height(400)
.title("Voltage Distribution with Anomalies")
.render()
// Function to handle remaining anomalies
function handleRemainingAnomalies(data) {
return data.map(d => {
let newD = {...d};
// Handle extreme voltage values
if (d["Voltage (V)"] < 200 || d["Voltage (V)"] > 250) {
newD["Voltage (V)"] = 230; // Standard residential voltage
}
// Handle extreme power consumption values using a conditional approach
if (d["Power Consumption (kW)"] > 11) {
// Cap at 11 kW for extremely high values
newD["Power Consumption (kW)"] = 11;
}
return newD;
});
}
// Apply anomaly handling
cleanedData = handleRemainingAnomalies(correctedData);
// Visualize cleaned data
cleanedPlot = vl.markPoint({opacity: 0.3})
.data(cleanedData)
.encode(
vl.x().fieldT("Timestamp").title("Time"),
vl.y().fieldQ("Power Consumption (kW)").title("Power Consumption (kW)"),
vl.tooltip([
{field: "Timestamp", type: "temporal"},
{field: "Power Consumption (kW)", type: "quantitative"}
])
)
.width(800)
.height(400)
.title("Cleaned Power Consumption Data")
.render()
// Add month information to data
dataWithMonth = cleanedData.map(d => {
const date = d.Timestamp;
return {
...d,
Month: date.toLocaleString('default', { month: 'long' }),
MonthNum: date.getMonth() // For sorting
};
})
// Aggregate power consumption by month
monthlyConsumption = d3.rollup(
dataWithMonth,
v => d3.sum(v, d => d["Power Consumption (kW)"]),
d => d.Month
)
// Convert to array for visualization
monthlyConsumptionArray = Array.from(monthlyConsumption, ([month, value]) => ({
month: month,
value: value,
monthNum: ["January", "February", "March", "April", "May", "June", "July",
"August", "September", "October", "November", "December"].indexOf(month)
}))
// Sort by month for chronological view
chronologicalMonthlyConsumption = monthlyConsumptionArray.sort((a, b) => a.monthNum - b.monthNum)
// Create visualization
monthlyConsumptionChart = vl.markBar()
.data(chronologicalMonthlyConsumption)
.encode(
vl.x().fieldN("month").title("Month").sort({
field: "monthNum",
op: "min",
order: "ascending"
}),
vl.y().fieldQ("value").title("Total Power Consumption (kW)"),
vl.color().fieldQ("value").title("Consumption").scale({scheme: "blues"}),
vl.tooltip([
{field: "month", type: "nominal"},
{field: "value", type: "quantitative", format: ",.1f", title: "Consumption (kW)"}
])
)
.width(600)
.height(400)
.title("Monthly Power Consumption")
.render()
// Find month with highest consumption
highestMonth = chronologicalMonthlyConsumption.reduce(
(max, current) => (current.value > max.value) ? current : max,
chronologicalMonthlyConsumption[0]
)
// Display the month with highest consumption
monthSummary = md`
### Month with Highest Power Consumption
**${highestMonth.month}** shows the highest accumulated power consumption at **${highestMonth.value.toLocaleString(undefined, {maximumFractionDigits: 0})} kW**.
This peak consumption aligns with seasonal energy usage patterns, likely corresponding to increased heating or cooling demands during periods of temperature extremes.
`
// Calculate excess power and potential income
dataWithExcess = cleanedData.map(d => {
const renewablePower = d["Solar Power (kW)"] + d["Wind Power (kW)"];
const powerConsumption = d["Power Consumption (kW)"];
const excessPower = Math.max(0, renewablePower - powerConsumption);
const date = d.Timestamp;
return {
...d,
ExcessPower: excessPower,
PotentialIncome: excessPower * d["Electricity Price (USD/kWh)"],
Month: date.toLocaleString('default', { month: 'long' }),
MonthNum: date.getMonth()
};
})
// Aggregate income by month
monthlyIncome = d3.rollup(
dataWithExcess,
v => d3.sum(v, d => d.PotentialIncome),
d => d.Month
)
// Convert to array for visualization
monthlyIncomeArray = Array.from(monthlyIncome, ([month, value]) => ({
month: month,
value: value,
monthNum: ["January", "February", "March", "April", "May", "June", "July",
"August", "September", "October", "November", "December"].indexOf(month)
}))
// Sort by month chronologically
chronologicalMonthlyIncome = monthlyIncomeArray.sort((a, b) => a.monthNum - b.monthNum)
// Visualize monthly potential income
monthlyIncomeChart = vl.markBar()
.data(chronologicalMonthlyIncome)
.encode(
vl.x().fieldN("month").title("Month").sort({
field: "monthNum",
op: "min",
order: "ascending"
}),
vl.y().fieldQ("value").title("Potential Income (USD)"),
vl.color().fieldQ("value").title("Income").scale({scheme: "greens"}),
vl.tooltip([
{field: "month", type: "nominal"},
{field: "value", type: "quantitative", format: "$,.2f", title: "Income (USD)"}
])
)
.width(600)
.height(400)
.title("Potential Monthly Income from Excess Power")
.render()
// Calculate average monthly income
totalAnnualIncome = chronologicalMonthlyIncome.reduce((sum, month) => sum + month.value, 0)
averageMonthlyIncome = totalAnnualIncome / chronologicalMonthlyIncome.length
// Display the average monthly income
incomeSummary = md`
### Potential Income from Excess Power
Average Monthly Income: **$${averageMonthlyIncome.toFixed(2)}**
Potential revenue is if all excess renewable energy (when solar + wind generation exceeds consumption) and is sold back to the grid at the prevailing electricity price.
`
// Add week information
dataWithWeek = cleanedData.map(d => {
const date = d.Timestamp;
const startOfYear = new Date(date.getFullYear(), 0, 1);
const dayOfYear = Math.floor((date - startOfYear) / (24 * 60 * 60 * 1000));
const weekNumber = Math.ceil((dayOfYear + startOfYear.getDay()) / 7);
return {
...d,
Week: weekNumber,
Year: date.getFullYear(),
YearWeek: `${date.getFullYear()}-W${weekNumber.toString().padStart(2, '0')}`
};
})
// Calculate weekly energy waste percentage
weeklyWaste = d3.rollup(
dataWithWeek,
v => {
const reactivePower = d3.sum(v, d => d["Reactive Power (kVAR)"]);
const totalConsumption = d3.sum(v, d => d["Power Consumption (kW)"]);
const totalEnergy = totalConsumption + reactivePower;
return {
reactivePower,
totalConsumption,
totalEnergy,
wastePercentage: (reactivePower / totalEnergy) * 100
};
},
d => d.YearWeek
)
// Convert to array for visualization
weeklyWasteArray = Array.from(weeklyWaste, ([yearWeek, data]) => ({
yearWeek,
reactivePower: data.reactivePower,
totalConsumption: data.totalConsumption,
wastePercentage: data.wastePercentage
})).sort((a, b) => a.yearWeek.localeCompare(b.yearWeek))
// Visualize weekly waste percentage
weeklyWasteChart = vl.layer(
// Primary line chart
vl.markLine()
.data(weeklyWasteArray)
.encode(
vl.x().fieldN("yearWeek").title("Week").sort(null),
vl.y().fieldQ("wastePercentage").title("Waste Percentage (%)").scale({domain: [0, 25]}),
vl.tooltip([/* unchanged */])
),
// Threshold line (rule)
vl.markRule()
.data([{threshold: 15}])
.encode(
vl.y().fieldQ("threshold"),
vl.color().value("red"),
vl.size().value(2)
),
// Threshold label (text)
vl.markText({align: "left", dx: 5, dy: -5})
.data([{threshold: 15, label: "15% Efficiency Threshold"}])
.encode(
vl.y().fieldQ("threshold"),
vl.text().fieldN("label"),
vl.color().value("red")
)
)
.width(800)
.height(400)
.title("Weekly Energy Waste Percentage")
.render();
// Calculate average waste percentage
averageWastePercentage = d3.mean(weeklyWasteArray, d => d.wastePercentage)
// Calculate percentage of weeks above threshold
weeksAboveThreshold = weeklyWasteArray.filter(d => d.wastePercentage > 15).length
percentAboveThreshold = (weeksAboveThreshold / weeklyWasteArray.length * 100).toFixed(1)
// Display the waste percentage summary
wasteSummary = md`
### Energy Waste Analysis
- Average waste percentage: **${averageWastePercentage.toFixed(1)}%**
- Weeks exceeding 15% threshold: **${percentAboveThreshold}%**
${averageWastePercentage > 15 ?
"The smart grid system is **not considered efficient** according to the 15% industry standard." :
"The smart grid system is **considered efficient** according to the 15% industry standard."}
${averageWastePercentage > 15 ?
"Installing power factor correction equipment could improve efficiency by reducing reactive power." :
"The system demonstrates good power factor management, with reactive power kept within acceptable limits."}
`
// Create correlation data structure
correlationData = [
{factor: "Temperature", field: "Temperature (C)"},
{factor: "Humidity", field: "Humidity (%)"},
{factor: "Power Consumption", field: "Power Consumption (kW)"},
{factor: "Solar Power", field: "Solar Power (kW)"},
{factor: "Wind Power", field: "Wind Power (kW)"},
{factor: "Grid Supply", field: "Grid Supply (kW)"},
{factor: "Reactive Power", field: "Reactive Power (kVAR)"}
]
// Add hour of day for time correlation
dataWithHour = cleanedData.map(d => ({
...d,
HourOfDay: d.Timestamp.getHours(),
IsWeekend: (d.Timestamp.getDay() === 0 || d.Timestamp.getDay() === 6)
}))
// Calculate correlation coefficients
function calculateCorrelation(x, y) {
const n = x.length;
// Calculate means
const xMean = x.reduce((a, b) => a + b, 0) / n;
const yMean = y.reduce((a, b) => a + b, 0) / n;
// Calculate variances and covariance
let xVariance = 0, yVariance = 0, covariance = 0;
for (let i = 0; i < n; i++) {
const xDiff = x[i] - xMean;
const yDiff = y[i] - yMean;
xVariance += xDiff * xDiff;
yVariance += yDiff * yDiff;
covariance += xDiff * yDiff;
}
// Calculate correlation coefficient
return covariance / (Math.sqrt(xVariance) * Math.sqrt(yVariance));
}
// Calculate all correlations
correlationResults = correlationData.map(item => {
const validData = dataWithHour.filter(d =>
!isNaN(d[item.field]) && !isNaN(d["Electricity Price (USD/kWh)"])
);
const x = validData.map(d => d[item.field]);
const y = validData.map(d => d["Electricity Price (USD/kWh)"]);
return {
factor: item.factor,
correlation: calculateCorrelation(x, y)
};
})
// Add time of day correlation
hourlyPrices = d3.rollup(
dataWithHour,
v => d3.mean(v, d => d["Electricity Price (USD/kWh)"]),
d => d.HourOfDay
)
hourlyPriceArray = Array.from(hourlyPrices, ([hour, price]) => ({hour, price}))
.sort((a, b) => a.hour - b.hour)
// Sort by absolute correlation value
sortedCorrelations = correlationResults.sort((a, b) =>
Math.abs(b.correlation) - Math.abs(a.correlation)
)
// Visualize correlations
correlationChart = vl.markBar()
.data(sortedCorrelations)
.encode(
vl.x().fieldQ("correlation").title("Correlation Coefficient"),
vl.y().fieldN("factor").title("Factor").sort("-x"),
vl.color().fieldQ("correlation").scale({
domain: [-1, 0, 1],
range: ["#f7f7f7", "#1d7eb3", "#ca562c"]
}),
vl.tooltip([
{field: "factor", type: "nominal"},
{field: "correlation", type: "quantitative", format: ".3f"}
])
)
.width(600)
.height(400)
.title("Correlation of Factors with Electricity Price")
.render()
// Visualize hourly electricity price patterns
hourlyPriceChart = vl.markLine({point: true})
.data(hourlyPriceArray)
.encode(
vl.x().fieldQ("hour").title("Hour of Day"),
vl.y().fieldQ("price").title("Average Electricity Price (USD/kWh)"),
vl.tooltip([
{field: "hour", type: "quantitative", title: "Hour"},
{field: "price", type: "quantitative", format: "$.4f"}
])
)
.width(600)
.height(400)
.title("Electricity Price by Hour of Day")
.render()
// Weekend vs Weekday price comparison
weekdayWeekendPrices = vl.markBar()
.data(dataWithHour)
.transform([
{
calculate: "datum.IsWeekend ? 'Weekend' : 'Weekday'",
as: "dayType"
},
{
aggregate: [{op: "mean", field: "Electricity Price (USD/kWh)", as: "avgPrice"}],
groupby: ["dayType"]
}
])
.encode(
vl.x().fieldN("dayType").title("Day Type"),
vl.y().fieldQ("avgPrice").title("Average Electricity Price (USD/kWh)"),
vl.color().fieldN("dayType"),
vl.tooltip({field: "avgPrice", type: "quantitative", format: "$.4f"})
)
.width(400)
.height(300)
.title("Average Electricity Price: Weekday vs Weekend")
.render()
// Create scatterplot of strongest correlation
topFactor = sortedCorrelations[0].factor
topField = correlationData.find(d => d.factor === topFactor).field
scatterplot = vl.layer(
// Scatterplot layer
vl.markPoint({opacity: 0.2})
.data(dataWithHour)
.encode(
vl.x().fieldQ(topField).title(topFactor),
vl.y().fieldQ("Electricity Price (USD/kWh)").title("Electricity Price (USD/kWh)"),
vl.tooltip([/* unchanged */])
),
// Regression line layer
vl.markLine({color: "red"})
.transform([
{
regression: topField,
on: "Electricity Price (USD/kWh)",
method: "linear",
extent: [0, "max"]
}
])
.encode(
vl.x().fieldQ(topField),
vl.y().fieldQ("Electricity Price (USD/kWh)")
)
)
.width(600)
.height(400)
.title(`Relationship Between ${topFactor} and Electricity Price`)
.render();
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
