Published unlisted
Edited
Dec 29, 2019
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
applyAnnotations(labels, chart) // to remove annotations, comment out this line and re-run above `chart` cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
data = {
// Upload another file to this notebook and replace the name below to process data
return d3.csvParse(await FileAttachment('trident_10.26.2019.csv').text())
.map(row => {
const date = new Date(row.unix_timestamp * 1000);
return {...row, date};
})
.filter(datum => +datum.temp1 > 0 && +datum.temp2 > 0)
}
Insert cell
rawDataByNodeId = nodeIds.map(id => {
return data
.filter(datum => id === datum.nodeID)
.sort((a, b) => {
if (+a.unix_timestamp > +b.unix_timestamp) return 1;
if (+a.unix_timestamp < +b.unix_timestamp) return -1;
if (+a.millisSinceHeatPulse > +b.millisSinceHeatPulse) return 1;
if (+a.millisSinceHeatPulse < +b.millisSinceHeatPulse) return -1;
})
})
Insert cell
dataSeparatedByHeatPulse = {
return rawDataByNodeId.map(rawData => {
let indexOfCurrentHeatPulse = 0;
let dataBinnedByHeatPulse = [];

for (let [index, datum] of rawData.entries()) {
if (index === 0) dataBinnedByHeatPulse[0] = [datum];

const prevIndex = index - 1;

if (prevIndex > 0) {
// Conditions for determining a new heat pulse bucket should be created:
// - if the millisSinceHeatPulse of the current datum is less than the millisSinceHeatPulse of the
// previous datum
// - if there was a data interruption (i.e., the unix_timestamp of the current datum is >3 seconds
// apart from the unix_timestamp of the previous datum), count these as distinct heat pulse buckets

if (
+datum.millisSinceReferenceTemp < +rawData[prevIndex].millisSinceReferenceTemp
// || +datum.unix_timestamp - 1800 > +rawData[prevIndex].unix_timestamp
) {
indexOfCurrentHeatPulse++;
dataBinnedByHeatPulse[indexOfCurrentHeatPulse] = [datum];
} else {
dataBinnedByHeatPulse[indexOfCurrentHeatPulse].push(datum);
}
}
}
return dataBinnedByHeatPulse;
})
}
Insert cell
heatRatios = dataSeparatedByHeatPulse.map((heatPulses, nodeIndex) => heatPulses.map((heatPulseData, pulseIndex) => {
const referenceTempData = heatPulseData.filter(datum => +datum.millisSinceReferenceTemp < 10000);
const referenceTemp1 = z.pipe([
z.parseNums(['temp1']),
z.getCol('temp1'),
z.mean()
])(referenceTempData);
const referenceTemp2 = z.pipe([
z.parseNums(['temp2']),
z.getCol('temp2'),
z.mean()
])(referenceTempData);
const temp2temp1Diff = referenceTemp2 - referenceTemp1;
return heatPulseData.map(datum => {
const temp1Diff = +datum.temp1 - referenceTemp1;
const temp2Diff = +datum.temp2 - referenceTemp2;
const heatRatio = temp1Diff / temp2Diff;
return {
...datum,
temp2temp1Diff,
referenceTemp1,
referenceTemp2,
temp1Diff,
temp2Diff,
heatRatio,
finalTemp1Recorded: heatPulseData[heatPulseData.length - 1].temp1,
finalTemp2Recorded: heatPulseData[heatPulseData.length - 1].temp2,
finalTempRecordedTime: heatPulseData[heatPulseData.length - 1].rtcUnixTimestamp,
referenceTempData
}
})
}))
Insert cell
meanHeatRatios = heatRatios.map(dataPerNode => dataPerNode.map(heatPulseData => {
const timestampForMeanHeatRatio = heatPulseData[0].unix_timestamp - Math.round(heatPulseData[0].millisSinceReferenceTemp / 1000 + 10);
const nodeID = heatPulseData[0].nodeID;
// Adjust for timezone
const date = new Date(timestampForMeanHeatRatio * 1000);
const heatRatioCalculationWindowStart = heatRatioWindowStart;
const heatRatioCalculationWindowEnd = heatRatioWindowEnd;
const heatPulseDataUsedForCalculatingHr = heatPulseData.filter(
datum => +datum.millisSinceReferenceTemp > 10000
&& +datum.millisSinceHeatPulse > heatRatioCalculationWindowStart
&& +datum.millisSinceHeatPulse < heatRatioCalculationWindowEnd
// make sure the temperature hasn't dropped back down to the baseline reference temp
&& +datum.temp1Diff > 0
&& +datum.temp2Diff > 0
);
let meanhr = z.pipe([
z.getCol('heatRatio'),
z.mean()
])(heatPulseDataUsedForCalculatingHr);
const meanHumidity = z.pipe([
z.getCol('outsideHumidity'),
z.mean()
])(heatPulseData);
const meanSoilMoisture = 1.0 / (z.pipe([
z.getCol('soilMoisture'),
z.mean()
])(heatPulseData));
const meanAmbientTemp = z.pipe([
z.getCol('outsideTemp'),
z.mean()
])(heatPulseData);
const vpd = (1.0 - (meanHumidity / 100)) * (0.611 * Math.pow(10, ((7.5 * meanAmbientTemp) / (237 + meanAmbientTemp))));
return {
unix_timestamp: timestampForMeanHeatRatio,
unixtimestamp: timestampForMeanHeatRatio,
data: [...heatPulseDataUsedForCalculatingHr],
meanhr,
meanHumidity,
meanSoilMoisture,
meanAmbientTemp,
vpd,
nodeID,
heatPulseDataPointsUsed: heatPulseDataUsedForCalculatingHr.length,
date
}
})
// .filter(datum => datum.data.length > 10)
// .filter(
// datum =>
// nodePicker.includes(datum.nodeID)
// && datum.date.getTime() >= startDateTime.getTime() &&
// datum.date.getTime() <= endDateTime.getTime()
// )
)
Insert cell
selectedMeanHeatRatios = meanHeatRatios[nodeIds.indexOf(nodePicker)]
Insert cell
render_linechart = function(data, xAxis, yAxis, lineFunction, debug) {
/* Return a DOM node with containing a line chart.

Arguments:
data: a list of data objects (dictionaries or iterables)
xAxis: a d3.axis
yAxis: a d3.axis
lineFunction: a function used to extract values from each datapoint and render line segments.
debug: variable that forces the chart to redraw every time this step reruns
*/
const svg = d3.select(DOM.svg(width, height));
svg.append("text")
.attr("x", width / 2)
.attr("y", margin.top)
.attr("dy", "1em")
.attr("font-weight", "bold")
// .attr("font-size", 14)
.style("text-anchor", "middle")
.text("Mean Heat Ratios, 10-24-2019 to 10-26-2019");
svg.append("g")
.call(xAxis);
svg.append("text")
.attr("x", width / 2)
.attr("y", height - (margin.bottom / 2))
.attr("dy", "1em")
.attr("font-size", 14)
.style("text-anchor", "middle")
.text("Date");

svg.append("g")
.call(yAxis);
svg.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 0)
.attr("x", -(height / 2))
.attr("dy", "1em")
.attr("font-size", 14)
.style("text-anchor", "middle")
.text("Mean Heat Ratio");
svg.append("path")
.datum(data.filter(line.defined()))
.attr("fill", "none")
.attr("stroke", "#3172bc")
.attr("stroke-dasharray", "4 2")
.attr("d", lineFunction);
svg.append("path")
.datum(data)
.attr("fill", "none")
.attr("stroke", "#3172bc") // Enigma Blue
.attr("stroke-width", 1.5)
.attr("stroke-linejoin", "round")
.attr("stroke-linecap", "round")
.attr("d", lineFunction)
return svg.node();
}
Insert cell
annotations_display = {
let collection = labels.collection()
if (collection) {
return collection.annotations
} else {
return "Manually rerun this cell to check the annotation marked by annotation_number"
}
}
Insert cell
labelAnnotations = {
return [
{
// If you don't provide a custom "type" attribute in your options dictionary, ,
// the default type in the getAnnotations function will be used.
note: {
label: "Sunrise",
title: "10-25-2019 7:28AM"
},
type: d3.annotationCalloutCircle,
data: {date: new Date(1572013680000), meanhr: getClosestMeanHr(selectedMeanHeatRatios, 1572013680000)},
dx: -10,
dy: -140,
subject: {
radius: 10
}
},
{
// If you don't provide a custom "type" attribute in your options dictionary, ,
// the default type in the getAnnotations function will be used.
note: {
label: "Midday depression",
title: "10-25-2019 ~1:15PM"
},
type: d3.annotationCalloutCircle,
data: {date: new Date(1572034426000), meanhr: getClosestMeanHr(selectedMeanHeatRatios, 1572034426000)},
dx: -1,
dy: -186,
subject: {
radius: 51
}
},
{
// If you don't provide a custom "type" attribute in your options dictionary, ,
// the default type in the getAnnotations function will be used.
note: {
label: "Sunset",
title: "10-25-2019 6:19PM"
},
type: d3.annotationCalloutCircle,
data: {date: new Date(1572052740000), meanhr: getClosestMeanHr(selectedMeanHeatRatios, 1572052740000)},
dx: 13,
dy: -100,
subject: {
radius: 10
}
}
];
}
Insert cell
getAnnotations = function(annotationList, x, y, debug){
/*
Return a list of d3.annotation objects
Arguments
annotationList - a list of dictionaries used to configure each annotation
x: a d3.scale
y: a d3.scale
parseTime: function for turning strings into js dates
formatTime: function for turning js date into a string
debug: a boolean flag
*/
chart // reactive hack to force annotations to render whenever the chart is redrawn
let makeLabelAnnotations = d3.annotation()
.editMode(debug) // GLOBAL VARIABLE
.type(d3.annotationLabel) // Adjust this arg to adjust the default annotation styling.
.accessors({
x: d => x(d.date),
y: d => y(d.meanhr)
})
.accessorsInverse({
date: d => x.invert(d.x),
meanhr: d => y.invert(d.y)
})
.annotations(annotationList)
return makeLabelAnnotations
}
Insert cell
function getClosestMeanHr(data, timestampMs) {
let closestInterval = Infinity;
let closestTimestamp;
for (let datum of data) {
if (Math.abs((+datum.unix_timestamp * 1000) - timestampMs) < closestInterval) {
closestInterval = Math.abs((+datum.unix_timestamp * 1000) - timestampMs)
closestTimestamp = datum.unix_timestamp
}
}
return data.filter(datum => datum.unix_timestamp === closestTimestamp)[0].meanhr;
}
Insert cell
// This function is responsible for converting the list of dictionaries into actual d3.annotation objects
labels = getAnnotations(labelAnnotations, x, y, DEBUG)
Insert cell
applyAnnotations = function(annotations, target) {
/*Draws d3.annotation objects onto a designated DOM node */
d3.select(target)
.append("g")
.attr("class", "annotation-group")
.call(annotations)
return annotations
}
Insert cell
DEBUG = {
return (CHECKED == "toggle")
}
Insert cell
line = d3.line()
.defined(d => !isNaN(d.meanhr))
.x(d => x(d.date))
.y(d => y(d.meanhr))
Insert cell
xAxis = g => g
.attr("transform", `translate(0,${height - margin.bottom})`)
.attr("class", "x-axis")
.call(d3.axisBottom(x)
.ticks(width / 80)
.tickSizeOuter(0))
Insert cell
yAxis = g => g
.attr("transform", `translate(${margin.left},0)`)
.attr("class", "y-axis")
.call(d3.axisLeft(y))
.call(g => g.select(".domain").remove())
.call(g => g.select(".tick:last-of-type text").clone()
.attr("x", 3)
.attr("text-anchor", "start")
.attr("font-weight", "bold")
.text(data.meanhr))
Insert cell
x = d3.scaleTime()
.domain(d3.extent(selectedMeanHeatRatios, d => d.date))
.range([margin.left, width - margin.right])
Insert cell
y = d3.scaleLinear()
.domain(d3.extent(selectedMeanHeatRatios, d => d.meanhr)).nice()
.range([height - margin.bottom, margin.top])
Insert cell
nodeIds = {
let ids = new Set();
data.forEach(datum => {
if (datum.nodeID !== undefined) ids.add(datum.nodeID)
});
return Array.from(ids);
}
Insert cell
margin = ({top: 20, right: 30, bottom: 40, left: 60})
Insert cell
height = 500
Insert cell
Insert cell
d3 = {
const bundle = Object.assign({}, d3_base, d3_svg_annotation);
return bundle;
}
Insert cell
d3_base = require("d3")
Insert cell
d3_svg_annotation = require("d3-svg-annotation")
Insert cell
vegalite = require("@observablehq/vega-lite")
Insert cell
import {vl} from '@vega/vega-lite-api'
Insert cell
z = require('https://bundle.run/zebras@0.0.11')
Insert cell
import {checkbox, date, time, radio, number} from "@jashkenas/inputs"
Insert cell
import {printTable} from '@uwdata/data-utilities'
Insert cell

One platform to build and deploy the best data apps

Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.
Learn more