26:null 27:["$","title",null,{"children":"Generic notebook to analyse 1D averaged electron density profiles | Davi Ortega | Observable"}] 2c:Ta2d,viewof myPlot = { const minMax = getMinMax(dataParsed) const padding = 0.01 return vega( { "width": 100, "height": 500, "data": {values: dataParsed}, layer: [ { "mark": "rule", "data": { "values": [ { "p": `${features.th}` } ] }, "encoding": { "x": { "aggregate": "sum", "field": "p", "type": "quantitative", "scale": {"domain": [minMax.minValue - padding, minMax.maxValue + padding]}, }, "color": {"value": "red"}, "size": {"value": 0.5} } }, { "mark": "rule", "encoding": { "x": { "aggregate": "mean", "field": "y", "type": "quantitative", "scale": {"domain": [minMax.minValue - padding, minMax.maxValue + padding]}, }, "color": {"value": "green"}, "size": {"value": 0.5} } }, { "mark": { "type": "line", "interpolate": 'cardinal', 'orient': 'horizontal' }, "encoding": { "y": { "field": "x", "type": "quantitative", "axis": { "title": "position (pixel)" } }, "x": { "field": 'y', "type": "quantitative", "grid": false, "scale": {"domain": [minMax.minValue - padding, minMax.maxValue + padding]}, "axis": { "title": 'pixel value normalized' } }, color: {'value': 'black'} }, }, { "mark": { "type": "point", 'orient': 'horizontal', "filled": true }, "encoding": { "y": { "field": "x", "type": "quantitative", "axis": { "title": "position (pixel)" } }, "x": { "field": 'y', "type": "quantitative", "grid": false, "scale": {"domain": [minMax.minValue - padding, minMax.maxValue + padding]}, "axis": { "title": 'pixel value normalized' } }, color: {'value': 'black'}, opacity: { value: 0.3 } }, } ] } ) }2d:Ta42,viewof myPlot2 = { const minMax = getMinMax(avgNorm.vegaFriendly) const padding = 0.01 return vega( { "width": 100, "height": 500, "data": {values: avgNorm.vegaFriendly}, layer: [ { "mark": "rule", "data": { "values": [ { "p": `${features.th}` } ] }, "encoding": { "x": { "aggregate": "sum", "field": "p", "type": "quantitative", "scale": {"domain": [minMax.minValue - padding, minMax.maxValue + padding]}, }, "color": {"value": "red"}, "size": {"value": 0.5} } }, { "mark": "rule", "encoding": { "x": { "aggregate": "mean", "field": "y", "type": "quantitative", "scale": {"domain": [minMax.minValue - padding, minMax.maxValue + padding]}, }, "color": {"value": "green"}, "size": {"value": 0.5} } }, { "mark": { "type": "line", "interpolate": 'cardinal', 'orient': 'horizontal' }, "encoding": { "y": { "field": "x", "type": "quantitative", "axis": { "title": "position (pixel)" } }, "x": { "field": 'y', "type": "quantitative", "grid": false, "scale": {"domain": [minMax.minValue - padding, minMax.maxValue + padding]}, "axis": { "title": 'pixel value normalized' } }, color: {'value': 'black'} }, }, { "mark": { "type": "point", 'orient': 'horizontal', "filled": true }, "encoding": { "y": { "field": "x", "type": "quantitative", "axis": { "title": "position (pixel)" } }, "x": { "field": 'y', "type": "quantitative", "grid": false, "scale": {"domain": [minMax.minValue - padding, minMax.maxValue + padding]}, "axis": { "title": 'pixel value normalized' } }, color: {'value': 'black'}, opacity: { value: 0.3 } }, } ] } ) }2e:T5e5,makePlot = (myData, align = true, ylabel = 'y value', yaxis = 'y', avg, filterPath = []) => { const allDataLayer = { "mark": { "type": "point", "interpolate" : "monotone" }, "transform": [ { calculate: `${align} ? datum.x + datum.shift : datum.x`, as: "x" } ], "encoding": { "x": { "field": "x", "type": "quantitative", "axis": { "title": "position (pixel)" } }, "y": { "field": yaxis, "type": "quantitative", "axis": { "title": ylabel } }, color: {"field": "path", "type": "nominal", legend: false }, opacity: {value: 0.3} } } const avgDataLayer = { "mark": { "type": "line", "interpolate" : "monotone" }, "transform": [ { calculate: `${align} ? datum.x + datum.shift : datum.x`, as: "x" } ], "encoding": { "x": {"field": "x", "type": "quantitative"}, "y": {"aggregate": "mean", field: "y", "type": "quantitative"}, color: { value: 'red', width: 20, legend: false } } } const allLayers = [allDataLayer] if (avg) allLayers.push(avgDataLayer) const parsedData = myData.filter((dataPoints) => { return filterPath.length === 0 ? true : filterPath.indexOf(dataPoints.path) !== -1 }) return vega( { "width": width - 100, "data": {values: myData}, layer: allLayers } ) }2f:[] 2a:{"id":"6399ff6b27e2465b","slug":"generic-notebook-to-analyse-1d-averaged-electron-density-p/2","title":"Generic notebook to analyse 1D averaged electron density profiles","thumbnail":"1b28a0079e47bbbf2af238b702652adccd353af69a7e98cea5a472053207ce53","access_level":"public","create_time":"$D2018-10-11T10:52:58.428Z","update_time":"$D2019-05-23T20:41:20.092Z","view_time":null,"first_public_time":"$D2019-05-28T23:33:01.476Z","has_unpublished_edits":null,"fork":{"id":"3d350add05a8a6c7","slug":"generic-notebook-to-analyse-1d-averaged-electron-density-p","title":"Generic notebook to analyse 1D averaged electron density profiles","owner":{"id":"92ee311a44ee7505","login":"daviortega","name":"Davi Ortega"}},"stars":0,"starred":false,"owner":{"type":"team","id":"92ee311a44ee7505","login":"daviortega","name":"Davi Ortega","avatar_url":"https://avatars-next.observableusercontent.com/5a0983d94c9bb9e0c1cc49b659f092376349a9ed0fb1967a9c8d98d4ea67af61","tier":"starter_2024"},"version":2100,"publish_version":2100,"role":null,"authored":false,"authors":[{"type":"individual","id":"61e7ac5783b2cc17","login":"daviortega","name":"Davi Ortega","avatar_url":"https://avatars-next.observableusercontent.com/5a0983d94c9bb9e0c1cc49b659f092376349a9ed0fb1967a9c8d98d4ea67af61","tier":null}],"members":[{"type":"individual","id":"61e7ac5783b2cc17","login":"daviortega","name":"Davi Ortega","avatar_url":"https://avatars-next.observableusercontent.com/5a0983d94c9bb9e0c1cc49b659f092376349a9ed0fb1967a9c8d98d4ea67af61","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":0,"collection":null,"type":"notebook","body":{"title":"Generic notebook to analyse 1D averaged electron density profiles","cells":[{"id":0,"value":"md`# Generic notebook to analyse 1D averaged electron density profiles\n\nThis is an analysis dashboard for 1D electron density profiles. The averaged profiles are generated by the package _sideview-profile-average_ on gitlab.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1798,"value":"viewof inputFiles = file(\n {\n title: \"Electron density profiles\",\n description: \"Upload your JSON formated profile\",\n accept: \".json\",\n multiple: true,\n }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":907,"value":"data = {\n const results = []\n for (let i = 0; i < inputFiles.length; i++)\n results.push(JSON.parse(`${await Files.text(inputFiles[i])}`))\n return results\n}\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1128,"value":"arrayToJson = (inputData) => {\n const parsed = []\n inputData.forEach((profile, i) => {\n profile.forEach((datapoint, j) => {\n parsed.push({x: j, y: parseFloat(datapoint), path: i})\n })\n })\n return parsed\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1130,"value":"dataParsed = {\n const averages = []\n data.forEach((profile) => {\n averages.push(profile.average.data)\n })\n return arrayToJson(averages)\n} ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":940,"value":"md`These are the averaged profiles collected from the tomogram themselves.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":912,"value":"$2c","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1628,"value":"features = {\n return {\n 'aw': 37,\n 'daw': 0.75,\n 'im': 50,\n 'dim': 0.5,\n 'om': 130,\n 'dom': 1,\n 'sl': 36,\n 'dsl': 0.5,\n 'l1': 41,\n 'dl1': 1,\n 'l2': 45,\n 'dl2': 1,\n 'l3': 32.2,\n 'dl3': 0,\n 'pxSize': 1.973,\n 'th': 0.46\n }\n}","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1262,"value":"md` ## Table with distance measurements to the CheA/CheW layer\n\n| Feature | Distance from CheA/CheW (nm ${tex`\\pm`} uncertainty)\n| - | :-: |\n| Inner membrane (IM) | ${represent(distances2AW.im.value, 2 * distances2AW.im.error)} nm |\n| Layer 3 (L3) | ${represent(distances2AW.l3.value, 2 * distances2AW.l3.error)} nm |\n| Layer 2 (L2) | ${represent(distances2AW.l2.value, 2 * distances2AW.l2.error)} nm |\n| Layer 1 (L1) | ${represent(distances2AW.l1.value, 2 * distances2AW.l1.error)} nm |\n| Signaling Layer (SL) | ${represent(distances2AW.sl.value, 2 * distances2AW.sl.error)} nm |\n\n_pixel size of the tomogram: ${features.pxSize} nm/pixel_<\\small>\n***\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2008,"value":"distances2AW = {\n const obj = {\n im: {\n value: Math.abs(features.im - features.aw) * features.pxSize,\n error: calcErrorSub(features.dim, features.daw) * features.pxSize\n },\n sl: {\n value: Math.abs(features.sl - features.aw) * features.pxSize,\n error: calcErrorSub(features.dsl, features.daw) * features.pxSize\n },\n l1: {\n value: Math.abs(features.l1 - features.aw) * features.pxSize,\n error: calcErrorSub(features.dl1, features.daw) * features.pxSize\n },\n l2: {\n value: Math.abs(features.l2 - features.aw) * features.pxSize,\n error: calcErrorSub(features.dl2, features.daw) * features.pxSize\n },\n l3: {\n value: Math.abs(features.l3 - features.aw) * features.pxSize,\n error: calcErrorSub(features.dl3, features.daw) * features.pxSize\n },\n }\n return obj\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2010,"value":"calcErrorSub = (d1, d2) => {\n return Math.sqrt( Math.pow(d1, 2) + Math.pow(d2, 2) )\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":2001,"value":"represent = (value, error) => {\n const e = error.toPrecision(2)\n const f = e.split('.')[1].length\n return tex`${value.toFixed(f)} \\pm ${e}`\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1076,"value":"md`## Processing the data\nWe should normalize the pixel values based on all data collected in these profiles. But first let's take a look at the distribution of values, just to make sure that there are no outliers.`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1086,"value":"{\n const padding = 3\n const minMax = getMinMax(dataParsed)\n \n return vega(\n {\n \"width\": width,\n \"data\": {\n values: dataParsed\n },\n layer: [\n {\n \"mark\": \"tick\",\n \"encoding\": {\n \"x\": {\n \"field\": \"y\",\n \"type\": \"quantitative\",\n \"scale\": {\"domain\": [minMax.minValue - padding, minMax.maxValue + padding]},\n \"axis\": {\n \"title\": \"pixel value\"\n }\n },\n 'y' : {\n 'field': 'path',\n 'type': 'ordinal',\n },\n 'color': { 'field': 'path' }\n }\n }\n ]\n })\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1180,"value":"md`There are two ways to normalize these:\n\n1) Global normalization, where all data is normalized based on global maximum and minimum. \n2) Each profile is normalized based on it's own value of maximum and minimum.\n\nThe first is simpler and second is safer, let's do the second`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1123,"value":"getMinMax = (allData) => {\n const maxValue = Math.max(...allData.map(a => a.y))\n const minValue = Math.min(...allData.map(a => a.y))\n return {maxValue, minValue}\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1120,"value":"dataNorm = {\n const results = []\n data.forEach((profile, i) => {\n averages.push(profile.average.data)\n })\n const minMax = getMinMax(dataParsed)\n dataParsed.forEach((value) => {\n const newValue = JSON.parse(JSON.stringify(value))\n newValue.norm = ((newValue.y - minMax.minValue)/(minMax.maxValue - minMax.minValue))\n results.push(newValue)\n })\n return results\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1223,"value":"md`... let's make another plot where the redline is the average of all data`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1225,"value":"makePlot(dataNorm, false, 'pixel value (normalized)', 'y', false)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1864,"value":"md `## Getting the average out.\n\nWe will need to plot this somewhere else, so let's calculate the average ourselves and export.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1306,"value":"getAvgNorm = (dataProfile) => {\n const result = []\n const paths = new Set()\n dataProfile.forEach((dataPoint) => {\n paths.add(dataPoint.path)\n while (dataPoint.x > result.length - 1) {\n result.push(0)\n }\n result[dataPoint.x] += dataPoint.norm\n })\n const numPaths = paths.size\n const vegaFriendly = []\n for (let i = 0; i < result.length; i++) {\n vegaFriendly.push(\n {\n x: i,\n y: result[i] /numPaths\n }\n )\n result[i] /= numPaths\n }\n return {vegaFriendly, result}\n}\n\n","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1298,"value":"avgNorm = getAvgNorm(dataNorm)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1337,"value":"md `Now let's plot that:`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1342,"value":"$2d","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1867,"value":"md` ## Measuring uncertanties","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":943,"value":"md `## Aligning the profiles.\n\nIt might be the case that the profiles can be slighly alined. This is not true ofr the original data I used but it might be for yours.\n\nLet's use a cross-correlation-based algorithm to align the profiles. This comes from the field of signal processing. I will skip the theory for now, but the idea is that by the maximum of cross-correlation between two signals is equivalent to the shift between them.\n\nTo calculate the cross-correlation between two signals, we will use FFT.\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1487,"value":"makePlot(dataS, false, 'y value', 'y', true, [])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":753,"value":"makePlot(dataS, true, 'y value', 'y', true, [])","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":20,"value":"$2e","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":623,"value":"dataS = prepData(dataNorm, 0)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":998,"value":"{\n const shifts = new Set()\n const diff = new Set()\n dataS.forEach((point) => {\n if (point.shift !== 0)\n diff.add(point.path)\n shifts.add(point.shift)\n })\n return { shifts, diff : Array.from(diff).toString() }\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1043,"value":"getShift(dataNorm, 1, 0) ","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":675,"value":"prepData = (myData, ref) => {\n const paths = []\n const final = []\n \n myData.forEach((item) => {\n if (!(item.path in paths))\n paths.push(item.path)\n final.push(item)\n })\n\n let shifted = []\n \n paths.forEach((p) => {\n const shift = getShift(final, ref, p)\n for ( let i = 0, N = final.length; i < N; i++) {\n if (final[i].path === p) {\n final[i].shift = shift[1]\n shifted.push(final[i])\n }\n }\n })\n \n return shifted\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":639,"value":"shiftData = (data, path, shift) => {\n const final = []\n data.forEach((item) => final.push(item))\n final.filter((d) => d.path === path).forEach((point, i) => {\n final[i].x += shift\n })\n return final\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":5,"value":"d3 = require('d3')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":250,"value":"getShift = (d, i, j) => {\n const sig1 = d.filter((item) => item.path === i)\n const sig2 = d.filter((item) => item.path === j)\n \n const s1 = ndarray(sig1.map((d) => d.norm))\n const s2 = ndarray(sig2.map((d) => d.norm))\n \n const xc = xcorr(s1, s2)\n \n \n return [xc, ops.argmax(xc) - Math.floor(xc.size / 2)]\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":281,"value":"xcorr = (x1, x2) => { \n const size = x1.shape[0] + x2.shape[0] - 1\n \n const s1 = zeros([size])\n const si1 = zeros([size])\n const s2 = zeros([size])\n const si2 = zeros([size])\n \n ops.assign(s1.hi.apply(s1, x1.shape), x1)\n ops.assign(s2.hi.apply(s2, x2.shape), x2.step(-1))\n \n fft(1, s1, si1)\n fft(1, s2, si2)\n cops.muleq(s1, si1, s2, si2)\n \n fft(-1, s1, si1)\n return s1\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":252,"value":"ndarray = require('https://bundle.run/ndarray@1.0.18')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":264,"value":"fft = require('https://bundle.run/ndarray-fft@1.0.3')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":260,"value":"zeros = require('https://bundle.run/zeros@1.0.0')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":271,"value":"ops = require('https://bundle.run/ndarray-ops@1.2.2')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":594,"value":"cops = require('https://bundle.run/ndarray-complex@1.0.3')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1542,"value":"// Taken from https://beta.observablehq.com/@mbostock/hello-vega-embed\nvega = {\n const v = window.vega = await require(\"vega@3\");\n const vl = window.vl = await require(\"vega-lite@2\");\n const ve = await require(\"vega-embed@3\");\n async function vega(spec, options) {\n const div = document.createElement(\"div\");\n div.value = (await ve(div, spec, options)).view;\n return div;\n }\n vega.changeset = v.changeset;\n return vega;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1554,"value":"vega_style = html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1796,"value":"import {file} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"},{"id":1823,"value":"// JSON = require('JSON')","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W2f","stdlib":"1","titling":"auto"}} 2b:"$2a:body"