3e:null 3f:["$","title",null,{"children":"EDNA data over time | akbstone | Observable"}] 45:Te69,createTimeSeries = ({ dateGroup = 'months', timeColumn = 'time', categoryColumn='family', categoryLimit, filterFn, categoryFilterFn, valueColumn='sampleSizeValue', calcMethod='sumOfValueColumn', excludeUnclassified = false })=>{ var day = 1000*60*60*24 var month = 30 * day function padStr(str){ return String(str).length < 2 ? `0${str}` : str } function getValue(row){ return valueColumn === 'count' ? 1 : row[valueColumn] } const calcMethods = { sum: (categoryRows, totalSumOfValue, filteredTotalSumOfValue)=>{ return d3.sum(categoryRows.map(getValue)) }, percentageOfAll: (categoryRows, totalSumOfValue, filteredTotalSumOfValue)=>{ console.log(categoryRows) return +((d3.sum(categoryRows.map(getValue))/totalSumOfValue)*100).toFixed(2) }, percentageOfVisualized: (categoryRows, totalSumOfValue, filteredTotalSumOfValue) => { return +((d3.sum(categoryRows.map(getValue))/filteredTotalSumOfValue)*100).toFixed(2) } } const dateGroupFns = { months:(d)=> `${d.getUTCFullYear()}-${padStr(d.getUTCMonth())}-01T00:00:00Z`, years:d=>`${d.getUTCFullYear()}`, seasonalMonths:d=>`2001-${padStr(d.getUTCMonth() + 1)}-15T00:00:00Z` } const resolutionsMap = { months: day * 30, years: day * 365, seasonalMonths: day*30 } const preFilter = false const fixedData = rawData.map(d=>{ return { ...d, [categoryColumn]: d[categoryColumn] || ( excludeUnclassified ? null : 'Unclassified at selected taxon level' ) } }).filter(d=>d[categoryColumn] !== null) const dataToUse = preFilter ? (filterFn ? fixedData.filter(filterFn) : fixedData.slice()) : fixedData.slice() const allByCategory = _.groupBy(dataToUse,d=>d[categoryColumn]) categoryFilterFn = categoryFilterFn || (()=>true) const categories = Object.keys(allByCategory) .map(cat=>({label:cat,records:allByCategory[cat]})) .sort((a,b)=>b.records.length - a.records.length) .filter(categoryFilterFn) .slice(0,categoryLimit || -1) .map(c=>c.label) const byDate = _.groupBy(dataToUse,d=>dateGroupFns[dateGroup](d[timeColumn])); if(dateGroup === 'seasonalMonths'){ const first = '2001-01-15T00:00:00Z' const last = '2001-12-15T00:00:00Z' if(!byDate[first]){ byDate[first] = [] } if(!byDate[last]){ byDate[last] = [] } } const out = Object.keys(byDate).map(timeCode=>{ console.log('byCategory', _.groupBy(byDate[timeCode],d=>d[categoryColumn])) const tot = d3.sum(byDate[timeCode].map(getValue)) const binData = preFilter ? byDate[timeCode] : filterFn ? byDate[timeCode].filter(filterFn) : byDate[timeCode].slice() const byCategory = _.groupBy(binData,d=>d[categoryColumn]) const filteredTot = d3.sum(categories.map(c=>(byCategory[c] || []).map(getValue)).flat(Infinity)) //return Object.keys(byCategory).filter(category=>categories.indexOf(category) >= 0).map(category=>{ return categories.map(category=>{ return { time: new Date(timeCode), // dateGroup === 'seasonalMonths' ? new Date(timeCode).getUTCMonth() : label: category, a: byCategory[category], sum: byCategory[category] ? d3.sum(byCategory[category].map(d=>d[valueColumn])) : 0, tot, filteredTot, value: byCategory[category] ? (calcMethods[calcMethod](byCategory[category], tot, filteredTot ) || null) : null } }) }).flat(Infinity).sort((a,b)=>{ return +a.time-+b.time }) out.byCategory = _.groupBy(out,r=>r.label) out.interval = dateGroup.match(/month/i) ? 'months' : 'years' return out }46:Tf8b,makePlot = ({data, time = 'time', category = 'label', value = 'value', title='Chart title', dateFormat, yLabel, wideLabels = false})=>{ const xChartOptions = dateFormat ? { tickFormat: dateFormat, } : {} const chart2 = Plot.plot({ width, height: 400, marginLeft: wideLabels ? 100 : 50, marginRight: 50, color: { legend: true, columns: '180px', range: d3.schemeTableau10 }, y: { grid: true, label: yLabel }, x: { ...xChartOptions, align: 'left' }, marks: [ Plot.rectY(data, {x: time, y: value, interval: data.interval, fill: category}), Plot.ruleY([0]) ] }) const chartSel = d3.select(chart2) const colorMap = {} chartSel.selectAll('div [class$=-swatch]').each(function(t){ const color = d3.select(this).select('svg').attr('fill') const label = d3.select(this).select('div').text() colorMap[label] = color }) if(dateFormat){ if(chartSel.selectAll('g[aria-label="x-axis tick label"] text').nodes().length > 12){ const firstTick = chartSel.selectAll('g[aria-label="x-axis tick label"] text').filter((d,i)=>i < 1) const xPos = +firstTick.attr('transform').match(/(0|[1-9]\d*)(\.\d+)?/g)[0] firstTick.remove() chartSel.selectAll('g[aria-label="x-axis tick label"] text').each(function(t){ const n = d3.select(this) const xy = n.attr('transform').match(/(0|[1-9]\d*)(\.\d+)?/g) n.attr('transform',`translate(${+xy[0] + 6 - xPos/2},${xy[1]})`) }) } } // ${chart} // display:grid;min-height:300px;grid-template-rows: repeat(4, minmax(0, 1fr));grid-auto-flow: column;gap: 15px;overflow:auto; return html`

${title}

${chart2}

${Object.keys(data.byCategory).map(c=> // Plot.line({data:data.byCategory[c]}).plot({height:100}) html`

${c}

${ Plot.plot({ height:120, width:350, x: xChartOptions, y: { label: yLabel, tickSize: 0 }, marginLeft: wideLabels ? 100 : 50, marks: [ Plot.lineY(data.byCategory[c], { x: time, y: value, fillOpacity: 0, strokeWidth: 3, interval: data.interval, stroke: colorMap[c], curve: 'catmull-rom' }), Plot.dotY(data.byCategory[c], { x: time, y: value, interval: data.interval, stroke: colorMap[c], fill: colorMap[c] }), Plot.ruleY([0]) ] })}

` )}

${Inputs.table(data)}

`; }47:T1055,makeInteractive = ({filterOn = 'phylum', categoryColumn = 'genus', valueColumn, preFilterFn, categoryFilterFn, dateGroup = 'months'}) => { const params = new URL(location.href) const chartWrap = html`

` const stateLinkWrap = html`

` const list = ['All'].concat(_.uniq(rawData.filter(d=>d[filterOn]).map(d=>d[filterOn])).sort((a,b)=>a.localeCompare(b))) //const input = Inputs.select(list, {label: `Select ${filterOn}`}) const categoryColumnInput = Inputs.select(["genus","family","order","class","phylum","kingdom"],{ label: 'Grouping column', value: params.searchParams.get('cc') || categoryColumn }) const categoryLimitInput = Inputs.select([5,10,20,30,40,'All'],{ label: 'Number of category groups to include', value: params.searchParams.get('i') || 10 }) const calcMethodMap = new Map([ ['Percentage of all', 'percentageOfAll'], ['Percentage of visualized','percentageOfVisualized'], ['Sum', 'sum'] ]) const calcMethodInput = Inputs.select(calcMethodMap,{ label: 'Calculation method', value: params.searchParams.get('c') || 'percentageOfAll' }) const valueColumnMap = new Map([ ['Sample size', 'sampleSizeValue'], ['Organism quantity', 'organismQuantity'], ['Row count', 'count'] ]) const valueColumnInput = Inputs.select(valueColumnMap,{ label: 'Column to use for visualization', value: params.searchParams.get('v') || 'sampleSizeValue' }) const timeBinInput = Inputs.select(new Map([ ['Years','years'], ['Months','months'], ['Seasonal statistic: months', 'seasonalMonths'] ]),{ label: 'Time bin', value: params.searchParams.get('t') || 'years' }) const excludeUnclassifiedInput = Inputs.toggle({ label: "Exclude unclassified at selected taxon level", value: params.searchParams.get('e') === 'false' ? false : true }) const rebuildChart = ()=>{ //const index = input.value //const selected = index === 'All' ? null : index const categoryColumnValue = categoryColumnInput.value const categoryLimit = categoryLimitInput.value === 'All' ? null : categoryLimitInput.value const calcMethod = calcMethodInput.value const valueColumn = valueColumnInput.value const excludeUnclassified = excludeUnclassifiedInput.value const dateGroup = timeBinInput.value const chart = makePlot({ title: `By ${categoryColumnValue} binned on ${dateGroup}`, dateFormat: dateGroup === 'seasonalMonths' ? (d)=>d.toLocaleString('default', { month: 'short' }) : null, yLabel: `${Object.fromEntries(Array.from(valueColumnMap).map(d=>d.slice().reverse()))[valueColumn]} ${calcMethod.match(/^percent/) ? '(%)' : ''}`, wideLabels: calcMethod === 'sum', data:createTimeSeries({ categoryColumn: categoryColumnValue, valueColumn, dateGroup, categoryLimit, categoryFilterFn, filterFn:d=> (typeof preFilterFn === 'function' ? preFilterFn(d) : true) && /*(!selected || d[filterOn] === selected) &&*/ d[categoryColumnValue], calcMethod, valueColumn, excludeUnclassified, dateGroup }) }) chartWrap.innerHTML = '' chartWrap.append(chart) stateLinkWrap.innerHTML = `Save application state in url (reloads page)` } categoryColumnInput.oninput = (e) => { rebuildChart() } categoryLimitInput.oninput = (e) => { rebuildChart() } calcMethodInput.oninput = (e) => { rebuildChart() } valueColumnInput.oninput = (e) => { rebuildChart() } excludeUnclassifiedInput.oninput = (e) => { rebuildChart() } timeBinInput.oninput = (e) => { rebuildChart() } rebuildChart() const form = html `${stateLinkWrap}` //${input} return html`${form}${chartWrap}` }48:[] 43:{"id":"64225f088d745a5e","slug":null,"title":"EDNA data over time","thumbnail":"43de18df5a8bd36ffc97a52e70a7c2598120cc16db66003f2193f22ed2e18561","access_level":"public","create_time":"$D2023-06-21T16:06:51.895Z","update_time":"$D2026-09-29T22:18:10.720Z","view_time":null,"first_public_time":"$D2023-06-21T16:06:51.895Z","has_unpublished_edits":null,"fork":false,"stars":0,"starred":false,"owner":{"type":"team","id":"e2bf871ed39979f5","login":"akbstone","name":"","avatar_url":"https://avatars-next.observableusercontent.com/5f1198be755feb49abf2a4a66e9ba1c5387eab18c8da8e0bc8bc865def49f441","tier":"starter_2024","bio":"","following":false},"version":617,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"d3206b376defd402","login":"akbstone","name":"","avatar_url":"https://avatars-next.observableusercontent.com/5f1198be755feb49abf2a4a66e9ba1c5387eab18c8da8e0bc8bc865def49f441","tier":null,"bio":"","following":false}],"members":[{"type":"individual","id":"d3206b376defd402","login":"akbstone","name":"","avatar_url":"https://avatars-next.observableusercontent.com/5f1198be755feb49abf2a4a66e9ba1c5387eab18c8da8e0bc8bc865def49f441","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":4,"collection":null,"type":"notebook","body":{"titling":"auto","title":"EDNA data over time","cells":[{"id":0,"value":"# EDNA data over time\n\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":218,"value":"plot = makeInteractive({\n filterOn:'phylum',\n categoryColumn:'scientificName',\n valueColumn:'sampleSizeValue',\n dateGroup:'seasonalMonths',\n categoryFilterFn:(c)=>c.label && c.label !== 'null'\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":9,"value":"# Get data from geoserver","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":11,"value":"rawData = {\n const geojson = await(await fetch('https://data.axds.co/gs/cencoos_edna/wms?service=WFS&version=1.1.0&request=GetFeature&outputFormat=application/json&typeName=cencoos_edna:monterey_bay_edna&srsName=EPSG:4326')).json()\n const features = geojson.features.map(f=>({...f.properties,time:new Date(f.properties.eventDate)}));\n return features;\n //const bySpecies = _.groupBy(features,d=>d.scientificName)\n //const topSpecies = Object.keys(bySpecies)\n // .map(k=>({label:k,data:bySpecies[k]}))\n // .sort((a,b)=>b.data.length - a.data.length)\n // .map(o=>o.label)\n //return features.filter(f=>topSpecies.indexOf(f.scientificName) >= 0)\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":40,"value":"# Functions","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":227,"value":"filterData = (filterFn)=>{\n\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":317,"value":"insertNulls = ({data,resolution,makeRow,xAcc})=>{\n var withNulls = data.slice()\n var copy = withNulls.slice()\n var allCt = 0\n \n copy.forEach((d,i)=>{\n if(i < (copy.length - 1)){\n var next = copy[i + 1]\n allCt += 1\n var nextX = xAcc(next)\n var currX = xAcc(copy[i])\n if (\n +nextX - +currX > resolution * 2\n ) {\n var x = +currX + (nextX - currX);\n var newRow = makeRow(x)\n withNulls.splice(+allCt, 0, newRow);\n allCt += 1;\n }\n \n }\n })\n\n return withNulls\n\n \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":13,"value":"$45","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"$46","pinned":false,"hidden":true,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":161,"value":"$47","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":42,"value":"# External dependencies","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":44,"value":"// import {StackedAreaChart} from \"@d3/normalized-stacked-area-chart/2\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":76,"value":"import {Swatches} from \"@d3/color-legend\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","tool":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W48","stdlib":"1"}} 44:"$43:body"