28:null 29:["$","title",null,{"children":"Derwent River data - RiverSense Project - Correlations | Charles Boisvert | Observable"}] 2e:T794,function prepareCorrelPair(collection,fieldX,fieldY) { // inputs: // collection (array): times and values of a series of measurements; // fieldX, fieldY (string): names of the measurements to process // return (array of {"time":Date, fieldX:float, fieldY:float}): values prepared for correlation, with times in intervals of 12 minutes; // where fieldX or fieldY data is missing for the time null values are inserted const subsetX = collection.filter((o)=>(o.field==fieldX)) const subsetY = collection.filter((o)=>(o.field==fieldY)) const result = [] let indexX=0, indexY=0, valueX, valueY, dateTime, previousDateTime, dateTimeX, dateTimeY while ((indexX(15*60*1000) } function add12minutes(dateTime) { // in progress const d = (new Date(dateTime)).getTime() const dplus12 = (new Date (d + 12*60*1000)).toISOString() return dplus12 } } 2f:T142c,/* sets the data in suitable format for Vega charts and calculate moving average stats one pass to transform each line into objects for: temperature, do2, moving average of each object has the date in ISO format */ function prepare_data(table) { let result = [] // for 24-hr moving averages // moving average indexes and running totals let temperatureResultSubset = [] // subset of result with the recent temperature objects to average let DOResultSubset = [] // ibid. DO2 let countDO = 0 let sumTemperature = 0 let sumDO = 0 table.forEach(processLine) return result function ISODate(date) { // input date DD/MM/YYYY const year = date.substring(6,10) const month = date.substring(3,5) const day = date.substring(0,2) return `${year}-${month}-${day}` } function over24Hours(beginDateTime,endDateTime) { const begin = new Date(beginDateTime) const end = new Date(endDateTime) return (end-begin)>(24*60*60*1000) } function processLine(line) { const temperature = line["Temperature"] const do2 = line["mg/L DO"] const recordTemp = (temperature!="") const recordO2 = (do2!="") if (recordTemp || recordO2 ) { const dateStg = line["Date"] const timeStg = line['Time'] const dateTime = `${ISODate(dateStg)} ${timeStg}` if (recordTemp) { const temp = parseFloat(temperature) const countTemperature = temperatureResultSubset.length if (countTemperature>=minThresholdReadingsCount && over24Hours(temperatureResultSubset[0].DateTime,dateTime)) { temperatureResultSubset = temperatureResultSubset.slice(1) // next avg, all but the first const movingAvgTemp = sumTemperature/countTemperature // use countT/2 to position the mvg average in the middle of the window; countT to position it at the end (average of the past day) //const midRangeTempSubset = temperatureResultSubset[countTemperature]-1 const midRangeTempSubset = temperatureResultSubset[Math.floor(countTemperature/2)] const objMovingAvgTemp = { field: "Temperature - 24hr moving average (C)", DateTime: midRangeTempSubset.DateTime, value: movingAvgTemp, Date: midRangeTempSubset.Date } result.push(objMovingAvgTemp) const detrendedTemp = midRangeTempSubset.value-movingAvgTemp const objDetrendedTemp = { field: "Detrended temperature variation (C)", DateTime: midRangeTempSubset.DateTime, value: detrendedTemp, Date: midRangeTempSubset.Date } result.push(objDetrendedTemp) // debug data: uncomment to add to the result //const objMovingAvgDebug = // { field: "moving avg debug", DateTime: movingAvgTempDateTime, // countTemp: countTemperature, sumTemp: sumTemperature, // startDate:temperatureResultSubset[0].DateTime, // over24Hours: over24Hours(temperatureResultSubset[0].DateTime,dateTime) } //result.push(objMovingAvgDebug) sumTemperature -= temperatureResultSubset[0].value } const objTemp = { field: "Temperature (C)", DateTime: dateTime, value: temp, Date: dateStg } temperatureResultSubset.push(objTemp) sumTemperature += temp result.push(objTemp) } if (recordO2) { const dissO = parseFloat(do2) const countDO = DOResultSubset.length if (countDO>=minThresholdReadingsCount && over24Hours(DOResultSubset[0].DateTime,dateTime)) { DOResultSubset = DOResultSubset.slice(1) // next avg, all but the first const movingAvgDO = sumDO/countDO // use countDO/2 to position the mvg average in the middle of the window; countDO to position it at the end (average of the past day) // const midRangeDOSubset = DOResultSubset[countDO]-1 const midRangeDOSubset = DOResultSubset[Math.floor(countDO/2)] const objMovingAvgDO = { field: "Dissolved O2 - 24hr moving average (mg/l)", DateTime: midRangeDOSubset.DateTime, value: movingAvgDO, Date: midRangeDOSubset.Date } result.push(objMovingAvgDO) const detrendedDO = midRangeDOSubset.value-movingAvgDO const objDetrendedDO = { field: "Detrended dissolved O2 variation (mg/l)", DateTime: midRangeDOSubset.DateTime, value: detrendedDO, Date: midRangeDOSubset.Date } result.push(objDetrendedDO) sumDO -= DOResultSubset[0].value } const objDO = { field:"Dissolved O2 (mg/l)", DateTime: dateTime, value: dissO, Date: dateStg } DOResultSubset.push(objDO) sumDO += dissO result.push(objDO) } } } } 30:T436,// Adapted from: https://stackoverflow.com/questions/15886527/javascript-library-for-pearson-and-or-spearman-correlations function pearsonCorrel(correlPair,shift=0) { // takes data in the form of a "correlPair", array of objects {date:_, fieldX:_, fieldY:_} // dates should be every 12 minutes, and fieldX and fieldY are null when data is missing. // calculations are adapted so that null values are ignored, not treated as 0 (!) const xArray = correlPair.map((item)=>(item.fieldX)) const yArray = correlPair.map((item)=>(item.fieldY)) let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0, sumY2 = 0, N = 0 const length = xArray.length = yArray.length = Math.min(xArray.length, yArray.length) let index=shift while (indexThis work is licensed under CC BY-SA 4.0 $\"\"$ $\"\"$ $\"\"$

","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined"},{"id":1130,"value":"A preliminary analysis of river data, to explore the possibilities of analysing detailed information on dissolved oxygen levels. The data was collected with the help of Aquasensor (https://aquasensor.co.uk/sensor).\n\nTwo sets of the data were analysed in this early work, both collected from the same site. The first, collected from August to October 2023, and the second, from August to December 2024.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":97,"value":"dataSource23 = FileAttachment(\"riverdata2024-01-19@2.csv\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1093,"value":"dataSource24 = FileAttachment(\"sensor22-2024-07-31 to 2024-12-24@2.csv\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1163,"value":"{\n return vl.markLine()\n .data(PData23.filter((l)=>((l.field==\"Temperature (C)\")||(l.field==\"Dissolved O2 (mg/l)\"))))\n .encode(\n vl.x().fieldT('DateTime').timeUnit(\"monthdatehoursminutes\").title(\"Date\"),\n vl.y().fieldQ('value').title('Value').scale({ domain: [2, 16] }),\n vl.color().fieldN(\"field\")\n )\n .width(600)\n .title(\"Temperature (C) and Dissolved oxygen (ppm) from August to October 2023\")\n .render()\n}\n\n\n\n\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":970,"value":"The line graphs of temperature and dissolved oxygen clearly show daily variations of both, dissolved oxygen levels being higher at night in the cold water, and lower in the warmer day. The inverse relationship can also be seen over time.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":1308,"value":"{\n return vl.markLine()\n .data(PData24.filter((l)=>((l.field==\"Temperature (C)\")||(l.field==\"Dissolved O2 (mg/l)\"))))\n .encode(\n vl.x().fieldT('DateTime').timeUnit(\"monthdatehoursminutes\").title(\"Date\"),\n vl.y().fieldQ('value').title('Value').scale({ domain: [2, 16] }),\n vl.color().fieldN(\"field\")\n )\n .width(600)\n .title(\"Temperature (C) and Dissolved oxygen (ppm) from August to December 2024\")\n .render()\n}\n\n\n\n\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1390,"value":"Scattergraphs show this relationship very clearly, Although it is much more apparent in the later (2024) data. The difference is due to the position of the sensor: in 2023 the prototype sensor was fixed to steps close to the river bank, where eddies and debris influenced dissolved oxygen levels; in 2024 it was attached to a float and moved further into the flow of the river.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":74,"value":"{\n return vl\n .data(scatterData23) // data.map((l)=>({temp:l[\"Temperature\"],do2:l[\"mg/L DO\"]})))\n .encode(\n vl.x().fieldQ('fieldX').title(\"Temperature (C)\").scale({ domain: [2, 16] }), // was 'temp'\n vl.y().fieldQ('fieldY').title(\"Dissolved oxygen (ppm)\").scale({ domain: [4, 16] }), // was 'DO2'\n )\n .mark(\"point\",{size:1, opacity: 0.3 })\n .width(600)\n .title(\"Scattergraph of dissolved oxygen (ppm) against temperture (C), 2023\")\n .render()\n}\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1295,"value":"{\n return vl\n .data(scatterData24) // data.map((l)=>({temp:l[\"Temperature\"],do2:l[\"mg/L DO\"]})))\n .encode(\n vl.x().fieldQ('fieldX').title(\"Temperature (C)\").scale({ domain: [2, 16] }), // was 'temp'\n vl.y().fieldQ('fieldY').title(\"Dissolved oxygen (ppm)\").scale({ domain: [4, 16] }), // was 'DO2'\n )\n .mark(\"point\",{size:1, opacity: 0.3 })\n .width(600)\n .title(\"Scattergraph of dissolved oxygen (ppm) against temperture (C), 2024\")\n .render()\n}\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":985,"value":"Correlations between temperature are DO2 confirm this, at ${pearsonCorrel(scatterData23,0)} for the 2023 data and ${pearsonCorrel(scatterData24,0)} in the 2024 period.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":475,"value":"viewof range23 = interval([0, datesList23.length-1], {\n label: 'Select the dates to graph a section of the 2023 data',\n step: 1,\n format: ([start, end]) => `${datesList23[start]} … ${datesList23[end]}`\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1414,"value":"Dissolved oxygen and temperature. Use the Selector above to narrow the period; 3 to 9 September neatly shows the daily periodic variations of temp and DO2, and the delay of DO compared to temperature; 13 to 23 October shows a warming period ending in release of water from Ladybower and a storm.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":1350,"value":"{\n return vl.markLine()\n .data(filteredData23.filter((l)=>((l.field==\"Temperature (C)\")||(l.field==\"Dissolved O2 (mg/l)\"))))\n .encode(\n vl.x().fieldT('DateTime').timeUnit(\"monthdatehoursminutes\").title(\"Date\"),\n vl.y().fieldQ('value').title('Value').scale({ domain: [2, 16] }),\n vl.color().fieldN(\"field\")\n )\n .width(600)\n .title(\"Dissolved oxygen and temperature, storm between 13 to 23 October 2023\")\n .render()\n}\n\n\n\n\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1361,"value":"The fine granularity of the data enables a detailed view of variations over time. The one-week period of 3 to 9 September 2024 shows a clear phase shift between the periodic variations of temperature and dissolved oxygen levels: temperature varies over the day, and dissolved oxygen follows these temperature changes with several hours delay.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":1406,"value":"A cross-correlation of temperature and DO2 confirms this phase shift.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":1238,"value":"{\n return vl\n .data(XCorrel23)\n .encode(\n vl.x().fieldQ('shift').title(\"Shift (hours)\"),\n vl.y().fieldQ('correl').title(\"Correlation\").scale({domain:[-0.58, -0.46]}),\n )\n .mark(\"point\")\n .width(600)\n .title(\"Cross-correlation of temperature and dissolved oxygen, 2023 data\")\n .render()\n}\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1337,"value":"{\n return vl\n .data(XCorrel24)\n .encode(\n vl.x().fieldQ('shift').title(\"Shift (hours)\"),\n vl.y().fieldQ('correl').title(\"Correlation\").scale({domain:[-0.93, -0.9]}),\n )\n .mark(\"point\")\n .width(600)\n .title(\"Cross-correlation of temperature and dissolved oxygen, 2024 data\")\n .render()\n}\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1328,"value":"In 2023, the correlation between temperature and amounts of dissolved oxygen is strongest (at ${pearsonCorrel(scatterData23,28)}) after a delay of ${Math.floor(28/5)} hours ${(28%5)*12} minutes; while in 2024 the shift is lesser, but similar, with a best correlation of ${pearsonCorrel(scatterData24,26)} after ${Math.floor(26/5)} hours ${(26%5)*12} minutes.","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":1071,"value":"These early results are unsurprising, but demonstrate the potential of working with fine-grained data in a river basin like the Derwent.","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined"},{"id":646,"value":"**Sample of filtered data:**","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":259,"value":"md`---------------`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":17,"value":"md `Below: libraries, functions, boring operations`\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":285,"value":"md`Sample - unprocessed data`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":14,"value":"printTable(data24.slice(0, 5))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":901,"value":"

","pinned":false,"hidden":false,"mode":"html","author":"$undefined","summary":"$undefined"},{"id":1356,"value":"Data after transformation and filtered by date ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined"},{"id":48,"value":"printTable(filteredData23.slice(229,241))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":67,"value":"md `Select and transpose the data`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1230,"value":"XCorrel23 = crossCorrel(scatterData23, 50)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1285,"value":"XCorrel24 = crossCorrel(scatterData24, 50)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":799,"value":"scatterData23 = (prepareCorrelPair(PData23,\"Temperature (C)\",\"Dissolved O2 (mg/l)\")) //.filter((o)=>(o.fieldX==null))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1282,"value":"scatterData24 = (prepareCorrelPair(PData24,\"Temperature (C)\",\"Dissolved O2 (mg/l)\")) //.filter((o)=>(o.fieldX==null))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":719,"value":"$2e","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1217,"value":"function crossCorrel(XYPair,windowSize) {\n // inputs:\n // XYPair (Array): series {\"time\":T, fieldX:Vx, fieldY:Vy} of values prepared for correlation\n // windowSize (int): the number of correlations \n // return (array of {correl: float, shift: float}): correlations shifted by 0, 1, ..., windowSize\n const result = []\n for (var index=0; index=min && i<=max\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":43,"value":"filteredData23 = PData23.filter(byDates(datesList23,range23[0],range23[1])) // dates is user selection","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1261,"value":"filteredData24 = PData24.filter(byDates(datesList24,range[0],range[1])) // dates is user selection","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":51,"value":"PData24 = prepare_data(data24)\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":1267,"value":"PData23 = prepare_data(data23)\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":588,"value":"temp_series = []","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":601,"value":"minThresholdReadingsCount = 5","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":57,"value":"$2f","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":664,"value":"$30","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":316,"value":"md`Vega lite: plots`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":2,"value":"import {vl} from '@vega/vega-lite-api'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":317,"value":"md`Data Utils from Uni of Washington State; printTable is great, but unique filter inefficient (https://jsperf.com/unique-through-set-vs-loop/1)`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":12,"value":"import {printTable} from '@uwdata/data-utilities'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":324,"value":"md`D3 for csv import`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":8,"value":"d3 = require(\"d3\");","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":325,"value":"md`Form input functions. Thank you https://observablehq.com/@mootari`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":203,"value":"import {select} from '@jashkenas/inputs'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":471,"value":"import {interval} from '@mootari/range-slider'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"}],"files":"$W31","stdlib":"1","titling":"auto"}} 2d:"$2c:body"