The avatar for @andyreagan
Andy Reagan
Senior Data Scientist, MassMutual Lecturer, UC Berkeley MIDS -- PhD Applied Math, Computational Story Lab, UVM MS Applied Math, UVM BS Math, Virginia Tech
Public
Apple Health & LivingWELL
Edited
Apr 14, 2023
Importers
6
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
activitySummary = {
if (dataElements === "Records") {
return recentRecordsPoints;
} else if (dataElements === "Activity Summary") {
return activitySummaryPoints;
} else {
return workoutPointsDaily;
}
}
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
activitySummaryFormatted = healthjson.HealthData.ActivitySummary.map((d) => ({
activeEnergyBurned: +d.activeEnergyBurned,
date: new Date(d.dateComponents)
}))
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
recentRecords = healthjson.HealthData.Record.map((d) => ({
...d,
// value: +d.value,
// startDate: new Date(d.startDate),
startDate: dayjs(d.startDate).utc(true),
// endDate: new Date(d.endDate),
endDate: dayjs(d.endDate).utc(true),
// creationDate: new Date(d.creationDate)
creationDate: dayjs(d.creationDate).utc(true)
})).filter(
(d) =>
true && // d.type === "HKQuantityTypeIdentifierBasalEnergyBurned"
d.startDate >= start // new Date(2022, 11, 25)
)
Insert cell
recentBasal = recentRecords.filter(
(d) => d.type === "HKQuantityTypeIdentifierBasalEnergyBurned"
)
Insert cell
Inputs.table(recentBasal)
Insert cell
recentActive = recentRecords.filter(
(d) => d.type === "HKQuantityTypeIdentifierActiveEnergyBurned"
)
Insert cell
recentSleep = recentRecords.filter(d => d.type === "HKCategoryTypeIdentifierSleepAnalysis")
Insert cell
basalDevices = [...d3.group(recentRecords, (d) => d.sourceName).keys()]
Insert cell
Insert cell
viewof device = Inputs.select(basalDevices, {value: basalDevices[0], label: "Device"})
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
recentSleepOffset = recentSleep.map((d) => ({
...d,
startDate: d.startDate.add(12, "hour"),
endDate: d.endDate.add(12, "hours")
}))
Insert cell
Insert cell
Insert cell
Insert cell
bmr_devices = [...d3.group(recentBasal.filter(d => !d.sourceName.includes('iPhone')), d => d.sourceName).keys()]
Insert cell
iPhone_bmr_source = recentBasal.filter((d) => d.sourceName.includes("iPhone"))
.length > 0
? recentBasal.filter((d) => d.sourceName.includes("iPhone"))[0].sourceName
: null
Insert cell
recentBasalDevice = recentBasal.filter((d) => d.sourceName === bmr_device)
Insert cell
recentBasalperSec = d3.sum(recentBasalDevice, (d) => +d.value) /
d3.sum(recentBasalDevice, (d) =>
d.endDate - d.startDate < 1000
? 1440 * 60
: (dayjs(d.endDate) - dayjs(d.startDate)) / 1000
)
Insert cell
recent_bmr = recentBasalperSec * 60 * 1440
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
activitySummaryPoints = healthjson.HealthData.ActivitySummary.filter(
(d) => dayjs(d.dateComponents) >= start
).map((d) => ({
...d,
points:
(((+d.activeEnergyBurned / recent_bmr) * 24) / MET_THRESHOLD) *
MAX_POINTS_ACTIVITIES,
date: new Date(d.dateComponents)
}))
Insert cell
recent_bmr
Insert cell
Insert cell
sleep_devices = [...d3.group(recentSleep.filter(d => !d.sourceName.includes('iPhone')), d => d.sourceName).keys()]
Insert cell
Insert cell
Insert cell
Insert cell
point_lookup_fun = function(lookup, sleep_minutes) {
const found = lookup.find(d => d.sleep_minutes === sleep_minutes)
return !found ? 0 : lookup.filter(d => d.sleep_minutes === sleep_minutes)[0].points
}
Insert cell
Insert cell
recentSleepPoints = {
return recentSleep
.filter(
(d) =>
d.sourceName === sleep_device &&
d.value.includes("HKCategoryValueSleepAnalysisAsleep")
)
.map((d) => ({
...d,
sleep_minutes: Math.round((d.endDate - d.startDate) / 1000 / 60),
points: point_lookup_fun(
SLEEP_LOOKUP,
Math.round((d.endDate - d.startDate) / 1000 / 60)
)
}));
}
Insert cell
Insert cell
sleepSummaryPoints = d3
.rollups(
recentSleep
.filter(
(d) =>
d.sourceName === sleep_device &&
d.value.includes("HKCategoryValueSleepAnalysisAsleep")
)
.map((d) => ({
...d,
offsetStartDate: d.startDate.add(12, "hours"),
sleep_minutes: (d.endDate - d.startDate) / 1000 / 60
}))
.map((d) => ({
...d,
offsetStartDay: new Date(
d.offsetStartDate.$y,
d.offsetStartDate.$M,
d.offsetStartDate.$D
)
})),
(v) => Math.round(d3.sum(v, (d) => d.sleep_minutes)),
(d) => d.offsetStartDay
)
.map((d) => ({
date: d[0],
sleep_minutes: d[1],
points: point_lookup_fun(SLEEP_LOOKUP, d[1])
}))
Insert cell
Insert cell
Insert cell
dates = {
let ranges = Array();
let currentDate = dayjs(start);

const closeTime = dayjs();
const amount = 1;
const unit = 'day';

while (currentDate.isBefore(closeTime) || currentDate.isSame(closeTime)) {
currentDate = currentDate.add(amount, unit);
ranges.push(currentDate);
}
return ranges;
}
Insert cell
all_together = dates.map((d) => ({
date: d,
date_formatted: d.format("YYYY-MM-DD"),
weekyear: d.weekYear(),
week: d.week(),
weekday: d.day(),
date_date: new Date(d),
activity: activitySummary.find(
(x) => x.dateComponents === d.format("YYYY-MM-DD")
),
sleep: sleepSummaryPoints.find(
(x) => dayjs(x.date).format("YYYY-MM-DD") === d.format("YYYY-MM-DD")
)
}))
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
all_together_long = activitySummaryPoints
.map(d => ({...d, date: d.dateComponents, type: 'activity'}))
.concat(
sleepSummaryPoints
.map(d => ({...d, date: dayjs(d.date).format('YYYY-MM-DD'), type: 'sleep'}))
)
Insert cell
Insert cell
Insert cell
Insert cell
all_together_cumulative = {
let tmp = Array(all_together.length);
let i = 0;
tmp[i] = { ...all_together[i] };

// previous
const prevCumSlpValid = 0;
const prevCumActValid = 0;
const prevCumSlpInval = 0;
const prevCumActInval = 0;
const prevCumValid = prevCumSlpValid + prevCumActValid;
// current
const curSlp = all_together[i].sleep ? all_together[i].sleep.points : 0;
const curAct = all_together[i].activity ? all_together[i].activity.points : 0;
const curCum = curSlp + curAct;
// new
const newCumSlp = prevCumSlpValid + curSlp;
const newCumAct = prevCumActValid + curAct;
const newCum = prevCumValid + curCum; // or, newCumSlp + newCumAct

// could just set curFractionValid to 0 if we have any previous invalid
const newCumValid = d3.min([newCum, MAX_POINTS]);
const pointsNeeded = d3.max([newCumValid - prevCumValid, 0]);
const curFractionValid = pointsNeeded / curCum;
const curSlpValid = curSlp * curFractionValid;
const curSlpInval = curSlp * (1 - curFractionValid);
const curActValid = curAct * curFractionValid;
const curActInval = curAct * (1 - curFractionValid);

// set current values
tmp[i].curSlpValid = curSlpValid;
tmp[i].curSlpInval = curSlpInval;
tmp[i].curActValid = curActValid;
tmp[i].curActInval = curActInval;

// set cumulative values
tmp[i].cumSlpValid = prevCumSlpValid + curSlpValid;
tmp[i].cumSlpInval = prevCumSlpInval + curSlpInval;
tmp[i].cumActValid = prevCumActValid + curActValid;
tmp[i].cumActInval = prevCumActInval + curActInval;

for (let i = 1; i < all_together.length; i++) {
tmp[i] = { ...all_together[i] };
// previous
const prevCumSlpValid = tmp[i].weekday == 0 ? 0 : tmp[i - 1].cumSlpValid;
const prevCumActValid = tmp[i].weekday == 0 ? 0 : tmp[i - 1].cumActValid;
const prevCumSlpInval = tmp[i].weekday == 0 ? 0 : tmp[i - 1].cumSlpInval;
const prevCumActInval = tmp[i].weekday == 0 ? 0 : tmp[i - 1].cumActInval;

// console.log(
// i,
// prevCumSlpValid,
// prevCumActValid,
// prevCumSlpInval,
// prevCumActInval
// );

const prevCumValid = prevCumSlpValid + prevCumActValid;
// current
const curSlp = all_together[i].sleep ? all_together[i].sleep.points : 0;
const curAct = all_together[i].activity
? all_together[i].activity.points
: 0;
const curCum = curSlp + curAct;
// new
const newCumSlp = prevCumSlpValid + curSlp;
const newCumAct = prevCumActValid + curAct;
const newCum = prevCumValid + curCum; // or, newCumSlp + newCumAct

// could just set curFractionValid to 0 if we have any previous invalid
const newCumValid = d3.min([newCum, MAX_POINTS]);
const pointsNeeded = d3.max([newCumValid - prevCumValid, 0]);
const curFractionValid = curCum > 0 ? pointsNeeded / curCum : 1;
const curSlpValid = curSlp * curFractionValid;
const curSlpInval = curSlp * (1 - curFractionValid);
const curActValid = curAct * curFractionValid;
const curActInval = curAct * (1 - curFractionValid);

// console.log(
// i,
// prevCumSlpValid,
// prevCumActValid,
// prevCumSlpInval,
// prevCumActInval,
// prevCumValid,
// curSlp,
// curAct,
// curCum,
// newCumSlp,
// newCumAct,
// newCum
// );

// set current values
tmp[i].curSlpValid = curSlpValid;
tmp[i].curSlpInval = curSlpInval;
tmp[i].curActValid = curActValid;
tmp[i].curActInval = curActInval;

// set cumulative values
tmp[i].cumSlpValid = prevCumSlpValid + curSlpValid;
tmp[i].cumSlpInval = prevCumSlpInval + curSlpInval;
tmp[i].cumActValid = prevCumActValid + curActValid;
tmp[i].cumActInval = prevCumActInval + curActInval;
}
return tmp;
}
Insert cell
Insert cell
Insert cell
all_together_cumulative_long = {
let tmp = Array();
for (let i = 0; i < all_together.length; i++) {
if (all_together_cumulative[i].cumActValid > 0) {
tmp.push({
...all_together_cumulative[i],
type: "activity",
status: "_valid",
points: all_together_cumulative[i].curActValid,
cum_points: all_together_cumulative[i].cumActValid
});
}
if (all_together_cumulative[i].cumActInval > 0) {
tmp.push({
...all_together_cumulative[i],
type: "activity",
status: "invalid",
points: all_together_cumulative[i].curActInval,
cum_points: all_together_cumulative[i].cumActInval
});
}
if (all_together_cumulative[i].cumSlpValid > 0) {
tmp.push({
...all_together_cumulative[i],
type: "sleep",
status: "_valid",
points: all_together_cumulative[i].curSlpValid,
cum_points: all_together_cumulative[i].cumSlpValid
});
}
if (all_together_cumulative[i].cumSlpInval > 0) {
tmp.push({
...all_together_cumulative[i],
type: "sleep",
status: "invalid",
points: all_together_cumulative[i].curSlpInval,
cum_points: all_together_cumulative[i].cumSlpInval
});
}
}
return tmp;
}
Insert cell
Insert cell
healthjsonrecent = ({
HealthData: {
...healthjson.HealthData,
Record: recentRecords,
Workout: healthjson.HealthData.Workout.filter(
(d) => dayjs(d.startDate) >= start
),
ActivitySummary: activitySummaryPoints
}
})
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Plot.plot({
width: width,
marginBottom: 50,
marginLeft: 100,
x: {
tickRotate: 45
},
color: {
type: "linear",
legend: true
},
marks: [
Plot.barX(
healthjsonrecent.HealthData.Record.filter(
(d) => d.type === "HKQuantityTypeIdentifierActiveEnergyBurned"
),
{
x1: (d) => new Date(d.startDate),
y: "sourceName",
fill: (d) => +d.value,
// sort: (d) => new Date(d.startDate)
// y2: (d) => +d.value + 0.1,
x2: (d) => new Date(d.endDate)
}
)
]
})
Insert cell
Insert cell
Insert cell
Insert cell
filter_range = [
start_filter,
Math.abs(dayjs(start).diff(dayjs(), "minute")) - end_filter
]
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
meanCalMinBasal = d3.mean(
healthjsonrecent.HealthData.Record.filter((d) =>
d.type.includes("HKQuantityTypeIdentifierBasalEnergyBurned")
).map(
(d) => (+d.value / ((dayjs(d.endDate) - dayjs(d.startDate)) / 1000)) * 60
)
)
Insert cell
meanCalMinBasalTimeWeighted = (d3.sum(
healthjsonrecent.HealthData.Record.filter((d) =>
d.type.includes("HKQuantityTypeIdentifierBasalEnergyBurned")
),
(d) => +d.value
) /
d3.sum(
healthjsonrecent.HealthData.Record.filter((d) =>
d.type.includes("HKQuantityTypeIdentifierBasalEnergyBurned")
),
(d) => (dayjs(d.endDate) - dayjs(d.startDate)) / 1000
)) *
60
Insert cell
Insert cell
Insert cell
recentRecordsPoints = d3
.rollups(
healthjsonrecent.HealthData.Record.filter(
(d) =>
d.type.includes("HKQuantityTypeIdentifierActiveEnergyBurned") &&
((+d.value / ((dayjs(d.endDate) - dayjs(d.startDate)) / 1000)) * 60) /
(recent_bmr / 1440) >=
MVPA_threshold - 1
),
(v) => d3.sum(v, (d) => +d.value),
(d) => dayjs(d.startDate).format("YYYY-MM-DD")
)
.map((d) => ({
dateComponents: d[0],
date: new Date(d[0]),
calories: d[1],
points: (((d[1] / recent_bmr) * 24) / MET_THRESHOLD) * MAX_POINTS_ACTIVITIES
}))
Insert cell
recentRecordsMETs = healthjsonrecent.HealthData.Record.filter((d) =>
d.type.includes("HKQuantityTypeIdentifierActiveEnergyBurned")
)
.map((d) => ({
...d,
calories: +d.value,
seconds: (dayjs(d.endDate) - dayjs(d.startDate)) / 1000
}))
.map((d) => ({
...d,
minutes: d.seconds / 60
}))
.map((d) => ({
...d,
MET: d.calories / d.minutes / (recent_bmr / 1440),
points:
(((d.calories / recent_bmr) * 24) / MET_THRESHOLD) * MAX_POINTS_ACTIVITIES
}))
Insert cell
Insert cell
Plot.plot({
width: width,
// color: {
// type: "categorical",
// legend: true
// },
x: { grid: true, textAnchor: "end" },
y: { type: "symlog" },
marks: [
Plot.rectY(
recentRecordsMETs,
Plot.binX(
{ y: "sum" },
{ x: "MET", y: "minutes", thresholds: d3.range(0, 19, 1) }
)
)
]
})
Insert cell
Insert cell
Plot.plot({
width: width,
// color: {
// type: "categorical",
// legend: true
// },
x: { grid: true, textAnchor: "end" },
y: { type: "symlog" },
marks: [
Plot.rectY(
recentRecordsMETs,
Plot.binX(
{ y: "sum" },
{ x: "MET", y: "minutes", thresholds: d3.range(0, 19, 0.2) }
)
)
]
})
Insert cell
Insert cell
Plot.plot({
width: width,
// color: {
// type: "categorical",
// legend: true
// },
x: { grid: true, textAnchor: "end" },
y: { type: "linear" },
marks: [
Plot.rectY(
recentRecordsMETs,
Plot.binX(
{ y: "sum" },
{ x: "MET", y: "calories", thresholds: d3.range(0, 20, 1) }
)
)
]
})
Insert cell
Insert cell
Plot.plot({
width: width,
// color: {
// type: "categorical",
// legend: true
// },
x: { grid: true, textAnchor: "end" },
y: { type: "linear", domain: [0, 2500] },
marks: [
Plot.rectY(
recentRecordsMETs,
Plot.binX(
{ y: "sum" },
{ x: "MET", y: "points", thresholds: d3.range(0, 20, 0.2) }
)
)
]
})
Insert cell
Insert cell
Plot.plot({
width: width,
// color: {
// type: "categorical",
// legend: true
// },
x: { grid: true, textAnchor: "end" },
y: { type: "linear" },
marks: [
Plot.rectY(
recentRecordsMETs,
Plot.binX(
{ y: "sum" },
{
x: "MET",
y: "calories",
thresholds: d3.range(0, 20, 1),
// fill: "MET",
cumulative: true
}
)
)
]
})
Insert cell
Insert cell
Plot.plot({
width: width,
// color: {
// type: "categorical",
// legend: true
// },
// x: { grid: true, textAnchor: "end" },
// y: { type: "linear" },
marks: [
Plot.rectY(
recentRecordsMETs,
Plot.binX(
{ y: "sum" },
{
x: "MET",
y: "points",
thresholds: d3.range(0, 20, 0.2),
// fill: "MET",
cumulative: true
}
)
)
]
})
Insert cell
Insert cell
Plot.plot({
width: width,
// color: {
// type: "categorical",
// legend: true
// },
// x: { grid: true, textAnchor: "end" },
// y: { type: "linear" },
marks: [
Plot.rectY(
recentRecordsMETs,
Plot.binX(
{ y: "sum" },
{
x: "MET",
y: "points",
thresholds: d3.range(0, 20, 0.2),
// fill: "MET",
cumulative: -1
}
)
)
]
})
Insert cell
Insert cell
workoutPoints = healthjsonrecent.HealthData.Workout.map((d) => ({
...d,
calBurned: d.WorkoutStatistics
? // if we have stats
Array.isArray(d.WorkoutStatistics)
? // if it's an array
d3.sum(
d.WorkoutStatistics.filter(
(y) => y.type === "HKQuantityTypeIdentifierActiveEnergyBurned"
),
(d) => +d.sum
)
: // not an array, try the object type
d.WorkoutStatistics.type === "HKQuantityTypeIdentifierActiveEnergyBurned"
? // yes, the object is the right type:
+d.WorkoutStatistics.sum
: // no, not the right object type
0
: // no stats: 0
0,
calBasal: d.WorkoutStatistics
? // if we have stats
Array.isArray(d.WorkoutStatistics)
? // if it's an array
d3.sum(
d.WorkoutStatistics.filter(
(y) => y.type === "HKQuantityTypeIdentifierBasalEnergyBurned"
),
(d) => +d.sum
)
: // not an array, try the object type
d.WorkoutStatistics.type === "HKQuantityTypeIdentifierBasalEnergyBurned"
? // yes, the object is the right type:
+d.WorkoutStatistics.sum
: // no, not the right object type
0
: // no stats: 0
0
}))
.map((d) => ({
...d,
METworkoutBasal: d.calBasal === 0 ? null : d.calBurned / d.calBasal,
METmeanBasal: d.calBurned / (meanCalMinBasalTimeWeighted * +d.duration),
calpermin:
(+d.calBurned / ((dayjs(d.endDate) - dayjs(d.startDate)) / 1000)) * 60
}))
.map((d) => ({
...d,
points:
d.METmeanBasal > MVPA_threshold - 1
? (((d.calBurned / (meanCalMinBasalTimeWeighted * 1440)) * 24) /
MET_THRESHOLD) *
MAX_POINTS_ACTIVITIES
: 0
}))
Insert cell
workoutPointsDaily = d3
.rollups(
workoutPoints,
(v) => d3.sum(v, (d) => d.points),
(d) => dayjs(d.startDate).format("YYYY-MM-DD")
)
.map((d) => ({
dateComponents: d[0],
date: new Date(d[0]),
points: d[1]
}))
Insert cell
Insert cell
Insert cell
MET_THRESHOLD = 35
Insert cell
MAX_POINTS_ACTIVITIES = 1000
Insert cell
MAX_POINTS = 1000
Insert cell
Insert cell
Insert cell
Insert cell
Insert cell
import { toc } from "@matthewkenny/toc"
Insert cell
import { dayjs } from "@andyreagan/dayjs-loader"
Insert cell
healthjsonrecent
Insert cell
healthjson
Insert cell
import { healthjson } with { zipfile as zipfile } from "@andyreagan/apple-health-xml-parse"
Insert cell
import { createJsonDownloadButton } from "@trebor/download-json"
Insert cell
import { interval, rangeSlider } from "@mootari/range-slider@1781"
Insert cell
import { blankInput } from "@andyreagan/blank-input"
Insert cell
Insert cell

One platform to build and deploy the best data apps

Experiment and prototype by building visualizations in live JavaScript notebooks. Collaborate with your team and decide which concepts to build out.
Use Observable Framework to build data apps locally. Use data loaders to build in any language or library, including Python, SQL, and R.
Seamlessly deploy to Observable. Test before you ship, use automatic deploy-on-commit, and ensure your projects are always up-to-date.
Learn more