Designing and Engineering a Better Tomorrow
Public
Edited
Apr 14, 2023
3 forks
Importers
8 stars
data = {
const data = await FileAttachment("DJIA2020.csv").csv({ typed: true });
data._fromNotebook = true;
return data;
}
// https://github.com/vega/datalib/wiki/Statistics#dl_summary
dataProfile = dl.summary(data)
metrics = dataProfile
.filter(({ type, field }) => type == "number")
.map(({ field }) => field)
correlationPairs = {
function combination2(set) {
const pair = [];
for (let i = 0; i < set.length; i++) {
for (let j = i + 1; j < set.length; j++) {
pair.push([set[i], set[j]]);
}
}
return pair;
} // function
return combination2(metrics).map(([metric_1, metric_2]) => ({
metric_1,
metric_2
}));
}
rollupObj = {
const rollupObj = {};
for (const { metric_1, metric_2 } of correlationPairs) {
// https://uwdata.github.io/arquero/api/expressions
// https://observablehq.com/@mariodelgadosr/arquero-correlation-speed-test
rollupObj[
`${metric_1}${pairSeparator}${metric_2}`
] = `d => op.corr(d["${metric_1}"], d["${metric_2}"])`;
} // for
return rollupObj;
}
correlationAnalysis = aq
.from(data)
.rollup(rollupObj)
.fold(aq.all(), { as: ["Correlation Pair", "Correlation"] })
// https://uwdata.github.io/arquero/api/expressions#limitations
.derive(
{
metric_1: `d => op.split(d["Correlation Pair"],"${pairSeparator}")[0]`,
metric_2: `d => op.split(d["Correlation Pair"],"${pairSeparator}")[1]`
},
{ before: "Correlation Pair" }
)
.orderby(aq.desc("Correlation"))
maxMin = correlationAnalysis
.derive({
maxPair: op.first_value("Correlation Pair"),
minPair: op.last_value("Correlation Pair")
})
.rollup({
max: (d) => op.max(d.Correlation),
min: (d) => op.min(d.Correlation),
maxPair: op.max("maxPair"),
minPair: op.min("minPair")
})
analysisTextMax =
`Max correlation of ${maxMin.get("max",0).toFixed(6)} for ${maxMin.get("maxPair",0)}`;
analysisTextMin =
`Min correlation of ${maxMin.get("min",0).toFixed(6)} for ${maxMin.get("minPair",0)}`;
viewof correlationsRadioButtons = {
const minMaxPairs = maxMin
.get("maxPair", 0)
.split(pairSeparator)
.concat(maxMin.get("minPair", 0).split(pairSeparator));
const maxMinFilter = `(datum.metric_1 == '${minMaxPairs[0]}' && datum.metric_2 == '${minMaxPairs[1]}') ||
(datum.metric_1 == '${minMaxPairs[2]}' && datum.metric_2 == '${minMaxPairs[3]}')`;
const isolation = [
"true",
maxMinFilter,
"datum.Correlation > 0.5",
"datum.Correlation < - 0.5"
];
const subtitle = [null, `${analysisTextMax}; ${analysisTextMin}`, null, null];
const correlations = ["All", "Max/Min", "> 0.5", "< -0.5"];
const corrMap = new Map(
correlations.map((corr, i) => [
corr,
{ isolation: isolation[i], subtitle: subtitle[i] }
])
);
return Inputs.radio(corrMap, {
label: "Correlations",
value: corrMap.get("All")
});
}
// https://vega.github.io/vega/docs/schemes/#diverging
viewof colorScheme = {
const schemes = new Map([
["Blue-Orange", "blueorange"],
["Brown-Blue-Green", "brownbluegreen"],
["Puple-Green", "purplegreen"],
["Pink-Yellow-Green", "pinkyellowgreen"],
["Purple-Orange", "purpleorange"],
["Red-Blue", "redblue"],
["Red-Grey", "redgrey"],
["Red-Yellow-Blue", "redyellowblue"],
["Red-Yellow-Green", "redyellowgreen"],
["Spectral", "spectral"]
]);
return Inputs.select(schemes, { label: "Color Scheme", value: "spectral" });
}
// Name of the metric in the correlation analysis can be overriden when importing
metricName = undefined
// The heatmap's title can be overriden when importing
title = undefined
// A metric's lookup table can be overriden. See stockCompany for an example of a lookupTable key/value format
lookupTable = undefined
heatMap = {
// Concatenate correlationAnalysis to itself, switching metric_1/metric_2 for the heatmap.
// https://uwdata.github.io/arquero/api/verbs#concat
// https://uwdata.github.io/arquero/api/verbs#derive
const concatenation = correlationAnalysis.concat(
correlationAnalysis.derive({
metric_2: ({ metric_1 }) => metric_1,
metric_1: ({ metric_2 }) => metric_2
})
);
let thisMetricName = metricName;
let thisTitle = title;
let thisLookupTable = lookupTable;
if (data._fromNotebook) {
thisMetricName == "DJIA Stock";
thisTitle = "DJIA 2020 Correlation Analysis";
thisLookupTable = stockCompany;
} // if
thisMetricName = thisMetricName || "Metric";
thisTitle = thisTitle || "Metric Correlation Analysis";
thisLookupTable =
thisLookupTable ||
[
...new Map(
[]
.concat(
correlationAnalysis._data.metric_1.data,
correlationAnalysis._data.metric_2.data
)
.map((d) => [d, d])
).keys()
].map((d) => ({ metric: d, field: d }));
const lookupFields = Object.keys(thisLookupTable[0]);
const lookup = vl
.lookupData(thisLookupTable)
.key(lookupFields[0])
.fields([lookupFields[1]]);
return vl
.markBar()
.title({
text: thisTitle,
subtitle: correlationsRadioButtons.subtitle
})
.data(concatenation)
.transform(
vl.lookup("metric_1").from(lookup).as(`${lookupFields[0]} 1`),
vl.lookup("metric_2").from(lookup).as(`${lookupFields[0]} 2`)
)
.encode(
vl.y().fieldN("metric_1").title(thisMetricName),
vl.x().fieldN("metric_2").title(thisMetricName),
vl.color().fieldQ("Correlation").scale({ scheme: colorScheme }),
vl.tooltip([
`${lookupFields[0]} 1`,
`${lookupFields[0]} 2`,
{ field: "Correlation", format: "r" }
]),
vl
.opacity()
.if(correlationsRadioButtons.isolation, vl.value(1))
.value(0.09)
)
.render();
}
Purpose-built for displays of data
Observable is your go-to platform for exploring data and creating expressive data visualizations. Use reactive JavaScript notebooks for prototyping and a collaborative canvas for visual data exploration and dashboard creation.
