2e:null 2f:["$","title",null,{"children":"Universal testing/sentinel surveillance correlation analysis | Marco Q | Observable"}] 34:T6aa,plot = { const p = Plot.plot({ title: 'raw', width: width - 30, y: { grid: true }, marks: [ // Plot.rectY([viewof span], { // x1: d => new Date((d).value[0]), // x2: d => new Date((d).value[1]), // y1: 0, // y2: 100, // fill: "rgba(241, 246, 251, .7)" // }), Plot.line(timeseries, { z: 'source', x: "date", y: "value", stroke: d => colorMap[d['source']], filter: d => ['flunet','unisuv'].includes(d.source) }), Plot.line(timeseries, { z: 'source', x: "date", y: "value", stroke: d => colorMap[d['source']], filter: d => d.source === 'upd_flunet', strokeDasharray: '2,4', }), Plot.ruleX([new Date((viewof span).value[0])], { stroke: "rgba(0, 0, 0, 0.3)", strokeWidth: 1 }), Plot.ruleX([new Date((viewof span).value[1])], { stroke: "rgba(0, 0, 0, 0.3)", strokeWidth: 1 }), Plot.rectY([viewof span], { x1: d => new Date(Date.parse(country[1][0])), x2: d => new Date((d).value[0]), y1: 0, y2: 100, fill: "rgba(241, 246, 251, .7)" }), Plot.rectY([viewof span], { x1: d => new Date((d).value[1]), x2: d => new Date(Date.parse(country[1][1])), y1: 0, y2: 100, fill: "rgba(241, 246, 251, .7)" }), // Plot.text(timeseries, Plot.selectLast({ // x: "date", // y: "value", // z: "source", // text: "source", // textAnchor: "start", // dx: 3 // })) ] }) yield p; fullscreen(p, { center: true }); }35:T4a2,## Methodology We explored SARS-CoV-2 testing data from universal surveillance (obtained from WHO) and FluNet sentinel data sets, including only those countries with at least 12 continuous weeks (3 months) of data shared between the 2 data sets. Indeed, several countries were present only in one dataset or the weeks in common where too few to properly compute a correlation measure. The chosen correlation measure (to compare the percent positivity for SARS-CoV-2) is the Pearson correlation coefficient (PCC) which is the most used in several studies on correlation analysis. For each country, it is reported the PCC and its p-value considering the influence of lag (to shift the FluNet sentinel signal by a variable number of weeks) and smoothing (to average the signal of one week with the one of the previous and following weeks). Furthermore, the correlation between the 2 signals could have changed over time and, thus, it is possible to choose any time window selecting two dates thanks to the slider. The analysis was performed using Python libraries: scipy, numpy, datetime and pandas for the analysis of data sets and matplotlib and seaborn for the visualisation. 36:[] 32:{"id":"929fed484603a107","slug":"testing-surveillance-correlation","title":"Universal testing/sentinel surveillance correlation analysis","thumbnail":"e73285982e4904d946ec2f82ae86c6d55dac2425184e8cb549a9965175696e55","access_level":"public","create_time":"$D2022-12-13T14:33:57.219Z","update_time":"$D2022-12-13T16:22:53.990Z","view_time":null,"first_public_time":"$D2022-12-13T14:33:57.219Z","has_unpublished_edits":null,"fork":true,"stars":0,"starred":false,"owner":{"type":"team","id":"6e14342e91f52f30","login":"marcoqu","name":"Marco Q","avatar_url":"https://avatars-next.observableusercontent.com/155641707a4e7e52e419f477fc6b9524cd23ffea34a3369a48a2243a30f5a179","tier":"starter_2024"},"version":483,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"508a005ebab6ccf4","login":"marcoqu","name":"Marco Q","avatar_url":"https://avatars-next.observableusercontent.com/155641707a4e7e52e419f477fc6b9524cd23ffea34a3369a48a2243a30f5a179","tier":null}],"members":[{"type":"individual","id":"508a005ebab6ccf4","login":"marcoqu","name":"Marco Q","avatar_url":"https://avatars-next.observableusercontent.com/155641707a4e7e52e419f477fc6b9524cd23ffea34a3369a48a2243a30f5a179","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"Universal testing/sentinel surveillance correlation analysis","cells":[{"id":4,"value":"# Universal testing/sentinel surveillance correlation analysis","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":471,"value":"**Correlation between percent positivity from universal SARS-CoV-2 testing and percent positivity from sentinel surveillance testing as reported to FluNet**\n\n ","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":474,"value":"---","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":466,"value":"### Authentication","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":450,"value":"viewof user = Inputs.text({ label: \"Username\", value: \"\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":453,"value":"viewof pass = Inputs.password({ label: \"Password\", value: \"\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":469,"value":"### Input","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":51,"value":"viewof country = {\n const response = await fetch(`https://${url}/countries`, { headers: headers});\n const countriesObj = await response.json();\n return Inputs.select(Object.entries(countriesObj), { format: ([country, dates]) => country, label: \"country\" })\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":6,"value":"viewof span = {\n const start = new Date(Date.parse(country[1][0])).getTime()\n const end = new Date(Date.parse(country[1][1])).getTime()\n return interval([start, end], {\n label: \"time range\",\n min: start,\n max: end,\n step: 5,\n format: ([start, end]) => `${new Date(start).toLocaleDateString(\"en-US\")} ... ${new Date(end).toLocaleDateString(\"en-US\")}`,\n})\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":245,"value":"viewof submit = Inputs.button(\"Update\", { label: \" \"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":193,"value":"### Plot","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":267,"value":"viewof ts = Inputs.checkbox(new Map([[\"Sentinel surveillance\", \"flunet\"], [\"Universal surveillance\", \"unisuv\"], [\"Adjusted sentinel surveillance\", \"upd_flunet\"]]), { value: ['flunet', 'unisuv', 'upd_flunet'], format: x => html`${x[0]}`})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":77,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":195,"value":"html`\n\n\n \n \n\n\n \n \n\n\n \n \n\n\n \n \n\n

Lag:	${raw.correlation.lag}
Moving average:	${raw.correlation.movingaverage}
Pearson:	${raw.correlation.pearson}
Pvalue:	${raw.correlation.pvalue}

\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":385,"value":"$35","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":387,"value":"## Data sources\nFor sentinels surveillance we used data provided by sentinels on FluNet, the web-based tool for influenza surveillance, while the universal surveillance data come from Our World in Data (to be confirmed)","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":188,"value":"## Data and API calls","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","output":"","database":"$undefined"},{"id":237,"value":"viewof url = Inputs.text({ label: \"Server URL\", value: \"who.api.datainterfaces.org\"})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":456,"value":"headers = {\n const h = new Headers();\n h.set('Authorization', 'Basic ' + btoa(user + \":\" + pass));\n return h;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":319,"value":"colorMap = ({\n 'flunet': 'red',\n 'unisuv': 'orange',\n 'upd_flunet': 'red'\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"raw = {\n submit;\n const label = country[0];\n const [start, end] = (viewof span).value.map(t => new Date(t).toISOString().slice(0, 10));\n const response = await fetch(`https://${url}/timeseries/${label}/${start}/${end}`, { headers: headers});\n try {\n return await response.json();\n } catch (e) {\n return e\n }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":81,"value":"timeseries = {\n const list = Object.entries(raw.timeseries).flatMap(([k, entries]) => {\n return entries.map(entry => ({ date: new Date(entry[0]), source:k, value:entry[1] }))\n })\n list.sort((a, b) => a.date.getTime()-b.date.getTime())\n return list.filter(a => ts.includes(a.source))\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":31,"value":"import {interval} from '@mootari/range-slider@1781'","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":290,"value":"import {fullscreen} from \"@fil/fullscreen\"\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W36","stdlib":"1","titling":"auto"}} 33:"$32:body"