2b:null 2c:["$","title",null,{"children":"ARGOS NGS QC Summary Data | Joe Gergely | Observable"}] 31:T435,html`

${qc_data.length} ngs Read Files

${countUniqueKeys("organism_name")} Organisms

${countUniqueKeys("lineage")} Strains

${countUniqueKeys("genome_assembly_id")} Genome Assemblies

${countUniqueKeys("instrument")} NGS Platforms

`32:T563,contrast_plot = (nfield, cfield,plot_title) => ({ "data": {"values": stratify_data (qc_data, nfield, cfield)}, "resolve": {"scale": {"x": "shared"}}, //"title" : plot_title, "hconcat" : [{ "mark": { "type": "rect", }, //height : 200, "encoding": { "y": {"field": "type", "type": "nominal", "axis" : {"title" : cfield, "titleFontSize": 14, "labelFontSize" : 12, "labelFont" : "courier"}}, "x": { "field": "value", "type": "quantitative", "bin": {"maxbins":60}, "scale": {"type" : "linear"}, "axis" : {"title" : nfield, "titleFontSize": 14}, }, "color": { "aggregate": "count", "type": "quantitative", "title" : "Count", "legend" : {"orient" : "bottom"}, "scale" : {"type" : "symlog","scheme": "purples"} } } }, { "mark": { "type": "boxplot", //"extent": "min-max" }, //height : 200, "encoding": { "y": {"field": "type", "type": "nominal", axis: {"title" : cfield, "titleFontSize": 14, "labelFontSize" : 12, "labelFont" : "courier"}}, "x": { "field": "value", "type": "quantitative", "scale": {"zero": false, "type" : "linear"}, "axis" : {"title" : null, "grid" : false, "titleFontSize": 14}, }, "color" : {"value" : "purple"} } }, ] })33:T734,variant_density = (data, field, label)=>{ label = label || field; return vegalite ({ "config": { "style": { "cell": { "stroke": "transparent" } } }, //"resolve": {"scale": {"x": "independent"}}, //"title" : title ? title[i] : null, "data": { "values": _.filter (data, (c)=>_.isNumber (c[field])) }, "layer" : [ { "width": 800, "height": 60, "transform":[{ "density": field, "bandwidth": 0 }], "mark": {type: "area", "opacity" : 0.2, line : true, clip: true}, "encoding": { "x": { "field": "value", "grid" : null, "title": label, "type": "quantitative", "scale" : {"type" : "linear"}, "axis": {"grid": false, "titleFontSize" : 14, "labelFontSize" : 12} }, "y": { "field": "density", "type": "quantitative", "title" : "Density (AU)", "axis": {"grid": false, "titleFontSize" : 14} }, "color" : {"value" : "grey"}, } }, { "mark": {type: "tick", "opacity" : 0.5, clip: true, tooltip : {"content" : "data"}, size : 15, thickness: 2}, "transform":[{ "calculate": "-0", "as": "placement" }], "encoding": { "x": { "field": field, "grid" : null, //"title": "AV frequency", "type": "quantitative", "axis": {"grid": false} }, "y" : { "type": "quantitative", "field" : "placement" }, "color" : {"value" : "firebrick"}, } } ] } ); }34:Td46,html` `35:[] 2f:{"id":"33112ca3d3a2cd80","slug":null,"title":"ARGOS NGS QC Summary Data","thumbnail":"c0716325d2e8ee2ca89441175afcea9d153a4e2c027388a66a1bc81435ab5ae1","access_level":"public","create_time":"$D2022-08-12T14:40:29.205Z","update_time":"$D2022-10-18T16:28:10.408Z","view_time":null,"first_public_time":"$D2022-12-09T18:16:22.799Z","has_unpublished_edits":null,"fork":true,"stars":0,"starred":false,"owner":{"type":"team","id":"adea462ffc6621bd","login":"jgergely","name":"Joe Gergely","avatar_url":"https://avatars-next.observableusercontent.com/4b84f0039622847d0f14f75e71a85d7627752823928611c25cd0d895ac6f2c18","tier":"starter_2024"},"version":1556,"publish_version":null,"role":null,"authored":false,"authors":[{"type":"individual","id":"6fd125c00a304828","login":"jgergely","name":"Joe Gergely","avatar_url":"https://avatars-next.observableusercontent.com/4b84f0039622847d0f14f75e71a85d7627752823928611c25cd0d895ac6f2c18","tier":null}],"members":[{"type":"individual","id":"6fd125c00a304828","login":"jgergely","name":"Joe Gergely","avatar_url":"https://avatars-next.observableusercontent.com/4b84f0039622847d0f14f75e71a85d7627752823928611c25cd0d895ac6f2c18","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"lapsed":null,"subscribed":null,"forks":2,"collection":null,"type":"notebook","body":{"titling":"auto","title":"ARGOS NGS QC Summary Data","cells":[{"id":4,"value":"# ARGOS NGS QC Summary Data\n","pinned":false,"hidden":false,"mode":"md","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"","database":"$undefined"},{"id":1223,"value":"viewof data_source = Inputs.select ([\"HIVE\",\"Crandall\"])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":507,"value":"viewof dataset = Inputs.select(listOfOrganisms, {label: \"Organism\", format: x => x.label + \" (\" + x.count + \")\", value: listOfOrganisms.find(t => t.tag === \"Salmonella_enterica\")})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":926,"value":"$31","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1198,"value":"md`### Individual record data`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1202,"value":"html`

`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":878,"value":"qc_data = await d3.csv (\"https://raw.githubusercontent.com/Jgergely11/test2/main/.github/workflows/ngsQC_A.csv\",d3.autoType)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":908,"value":"listOfOrganisms = _.chain (qc_data).countBy ((d)=>d.organism_name).map ((d,k)=>({'label' : k.split ('_').join (' '), 'tag' : k, 'count' : d})).value()","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":937,"value":"selectedOrganism = _.filter (qc_data, (d)=>d.organism_name == dataset.tag)","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1086,"value":"numerical_fields = _.filter (_.keys (selectedOrganism[0]), (d)=> {\n const number_count = _.countBy (selectedOrganism, (c)=>_.isNumber (c[d]) ? 1 : 0);\n return (number_count[1] || 0) > (number_count[0] || 0);\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1129,"value":"categorical_fields = _.filter (_.keys (selectedOrganism[0]), (d)=> {\n const number_count = _.countBy (selectedOrganism, (c)=>_.isNumber (c[d]) ? 1 : 0);\n const cat_count = _.size(_.countBy (selectedOrganism, (c)=>c[d]));\n return (number_count[1] || 0) < (number_count[0] || 0) && cat_count <= 10 && cat_count >= 2;\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":504,"value":"data_sources = [\n {'url' : 'https://raw.githubusercontent.com/aglucaci/BUSTEDS-MH/main/tables/Table_EMPIRICAL_UNMASKED_SELECTOME_BUSTEDS_and_BUSTEDS-MH.csv', 'label' : \"Selectome\"},\n {'url': \"https://raw.githubusercontent.com/aglucaci/BUSTEDS-MH/main/tables/Table_13Datasets_BUSTEDS_and_BUSTEDS-MH.csv\", \"label\" : \"13 datasets\"},\n {'url' : \"https://raw.githubusercontent.com/aglucaci/BUSTEDS-MH/main/tables/Table_SIMULATIONS-SHULTZ-SACKTON_BUSTEDS_and_BUSTEDS-MH.csv\", \"label\" : \"Null simulations\"}\n]","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":31,"value":"stratify_data = (data, nfield, cfield) => {\n return _.filter (_.map (data, (c)=> {\n return {'value' : c[nfield], 'type' : c[cfield]}\n }), (d)=>_.isNumber (d.value));\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1152,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":626,"value":"extract_omega_distributions = (d)=> {\n let result = [];\n\n _.each ([\"BUSTEDS\",\"BUSTEDS-MH\"], (model)=> {\n _.each ([[\"w1\",\"p1\"],[\"w2\",\"p2\"],[\"w3\",\"p3\"]], (rate,i)=> {\n let record = {};\n record.gene = d.gene;\n record.type = model;\n record.value = d[model + \"|\" + rate[0]];\n record.weight = d[model + \"|\" + rate[1]];\n record.index = i;\n result.push (record);\n });\n });\n return result;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":265,"value":"binByGene = (data, p, aggregator, filter) => {\n p = p || 0.05;\n aggregator = aggregator || ((d)=>0);\n filter = filter || ((d)=>true);\n const data_2 = _.chain (data).groupBy ((d)=>d.Gene).mapValues ((d)=>_.groupBy (d, (c)=>c.Method)).mapValues (d=>_.mapValues (d, (c)=>c[0])).value();\n //return data_2;\n let result = [];\n _.each (data_2, (d, g)=> {\n let rec = {};\n rec.gene = g;\n if ((\"BUSTEDS\" in d) && (\"BUSTEDS-MH\" in d)) {\n if (filter (d)) {\n rec.ps = (d[\"BUSTEDS\"][\"LRT p-value\"] <= p ? \"+\" : \"-\") + (d[\"BUSTEDS-MH\"][\"LRT p-value\"] <= p ? \"+\" : \"-\");\n aggregator (d, rec);\n result.push (rec);\n }\n }\n });\n return result;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":148,"value":"twoByTwo = (m11,m12,m21,m22) => {\n const N = m11 + m12 + m21 + m22;\n const fmtP = d3.format (\".3p\");\n const formatP = (d)=>\"\" + d + \" (\" + fmtP (d/N) + \")\";\n return html`\n \n \n \n \n \n

	MH-	MH+	Total
SH-	${formatP(m11)}	${formatP(m12)}	${formatP(m11+m21)}
SH+	${formatP(m21)}	${formatP(m22)}	${formatP(m21+m22)}
Total	${formatP(m11+m21)}	${formatP(m12+m22)}	${N}

\n Cohen's κ = ${cohen_kappa (m11, m12, m21, m22)}\n `;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":247,"value":"cohen_kappa = (m11, m12, m21, m22) => {\n const N = m11 + m12 + m21 + m22; \n const p_0 = (m11 + m22) / N;\n const p_e = ((m11 + m12) * (m11 + m21) + (m21 + m22) * (m12 + m22)) / N / N;\n return (p_0 - p_e) / (1 - p_e);\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":197,"value":"getCounts = (d,p)=> {\n p = p || 0.05;\n return [_.filter (d, (c)=>c[\"BUSTEDS|LRT p-value\"] >p && c[\"BUSTEDS-MH|LRT p-value\"] >p).length,\n _.filter (d, (c)=>c[\"BUSTEDS|LRT p-value\"] >p && c[\"BUSTEDS-MH|LRT p-value\"] <=p).length,\n _.filter (d, (c)=>c[\"BUSTEDS|LRT p-value\"] <=p && c[\"BUSTEDS-MH|LRT p-value\"] > p).length,\n _.filter (d, (c)=>c[\"BUSTEDS|LRT p-value\"] <=p && c[\"BUSTEDS-MH|LRT p-value\"] <= p).length];\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":407,"value":"compareEstimates = (data, field, field_2) => (\n{\n \"data\": {\"values\": data},\n \"width\" : 400,\n \"height\" : 400,\n \"mark\": \"rect\",\n \"encoding\": {\n \"x\": {\"field\": field, \"type\": \"quantitative\", \"scale\" : {\"type\" : \"linear\"}, \"axis\" : {\"title\" : field, \"titleFontSize\" : 18, \"tickCount\" : 10}, \"bin\": {\"maxbins\":50}},\n \"y\": {\"field\": field_2, \"type\": \"quantitative\", \"scale\" : {\"type\" : \"linear\"},\"axis\" : {\"tickCount\" : 10, \"title\" : field_2, \"titleFontSize\" : 18}, \"bin\": {\"maxbins\":50}},\n \"size\" : {\"value\" : 5},\n \"color\": {\n \"aggregate\": \"count\",\n \"type\": \"quantitative\",\n \"title\" : \"Count\",\n \"legend\" : {\"orient\" : \"bottom\"},\n \"scale\" : {\"type\" : \"symlog\",\"scheme\": \"purples\"}\n }\n }\n }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":693,"value":"plot_omega = (d, label)=> (\n {\n \"data\": {\n \"values\": d\n },\n \"width\": 200,\n //\"height\": 400,\n \"layer\" : [{\n \"mark\": {\n \"type\": \"circle\",\n \"opacity\": 0.8,\n \"stroke\": \"black\",\n \"strokeWidth\": 1\n },\n \"encoding\": {\n \"x\": {\n \"field\": \"value\",\n \"type\": \"quantitative\",\n \"axis\": {\"grid\": false, \"title\" : label ? label : false, \"titleFontSize\" : 14, \"tickCount\" : 5},\n \"scale\" : {\"type\" : \"symlog\"}\n },\n \"color\" : {\"value\" : \"darkgrey\"},\n \"size\": {\n \"field\": \"weight\",\n \"type\": \"quantitative\",\n \"title\": \"Proportion\",\n \"legend\": false,//{\"clipHeight\": 50, \"orient\": \"top\"},\n \"scale\": {\"rangeMax\": 400}\n },\n }},{\n \"mark\": {\n \"type\": \"rule\",\n \"color\": \"red\",\n \"strokeWidth\": \"2\",\n },\n \"encoding\": {\n \"x\": {\n \"datum\": {\"expr\": \"1\"},\n \"type\": \"quantitative\",\n }\n }\n }\n \n \n ],\n \"config\": {\n \"style\": {\n \"cell\": {\n \"stroke\": \"transparent\"\n }\n }\n }\n}\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":65,"value":"vegalite = require (\"vega-embed\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":950,"value":"simpleHistogram = (key)=> {\n return histogram(\n _.map (selectedOrganism, (d)=>d[key]),\n {\n thresholds: 50,\n //domainline: { size: 1, stroke: \"#1e90ff\" },\n \"width\" : 80,\n \"height\" : 20,\n \"color\" : {\"line\" : \"black\", \"fill\" : \"silver\"},\n \"highlights\" : {\"start\" : null, \"end\" : null, \"min\" : null, \"max\" : null},\n //\"datum\" : {\"value\" : pval, \"color\" : \"firebrick\"},\n \"scaling\" : \"zero\",\n \"margin\" : {\"horizontal\" : .04, \"vertical\" : .125}\n }\n );\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":980,"value":"countUniqueKeys = (k,d)=>_.size (_.countBy (d || qc_data, (c)=>c[k]))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":948,"value":"import {line, bar, area, histogram, ticks, range} from \"@hamzaamjad/tiny-charts\"","pinned":true,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":1025,"value":"$33","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":928,"value":"html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":930,"value":"html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"},{"id":933,"value":"$34","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined","providerMetadata":"$undefined","attachments":"$undefined","output":"$undefined","database":"$undefined"}],"files":"$W35","stdlib":"1"}} 30:"$2f:body"