2b:null 2c:["$","title",null,{"children":"Combined view of sites which may be experiencing convergence in N501Y clades | Sergei Pond | Observable"}] 31:T1114,joint_MEME_list = { let rows = []; const classify = (set, n)=> { if (set) { if (annotation[n] && annotation[n].evo) { return _.map ([...set], (r)=> { return [r, annotation[n].evo[r] || 0]; }); } } return set; } _.each (joint_site_list[0], (info, site)=> { //console.log (site); let c = +site; let this_row = {}; if (site in reverseLookup == false) { c++; } this_row.n = site; if (!reverseLookup[c]) return; this_row.coord = reverseLookup[c][0][0]; let is_sig = _.find (signatures, (d)=>d.site == +site); let sig_site = false; if ('meme' in info || is_sig) { this_row.tags = {}; if ('meme' in info) { _.each ([...info['meme']], (clade)=>this_row.tags[clade] = {'meme' : 1}); } if (is_sig) { _.each (is_sig, (yn,clade) => { if (yn == 1) { if (clade in this_row.tags == false) { this_row.tags[clade] = {}; } this_row.tags[clade].sig = 1; sig_site = true; } }); } if (info['meme-full']) { _.each ([...info['meme-full']], (clade)=> { if (clade in this_row.tags == false) { this_row.tags[clade] = {}; } this_row.tags[clade] = {'meme-full' : 1}; }); } _.each (info['subs'], (subs,clade) => { //if (!this_row.tags[clade]) { if (this_row.coord == "S/484") { subs.add ("K"); } _.each ([...subs], (aa)=> { if (aa != '-') { _.each (info['subs'], (subs2,clade2)=> { if (clade2 != clade) { if (subs2.has (aa)) { if (clade in this_row.tags == false) { this_row.tags[clade] = {}; } this_row.tags[clade]['subs'] = clade2; } } }); } }); // } }); if ((sig_site || _.size (this_row.tags) > 1) && info["subs"]) { this_row ["V1"] = this_row.tags ["V1"]; this_row ["V2"] = this_row.tags ["V2"]; this_row ["V3"] = this_row.tags ["V3"]; this_row ["V1S"] = classify(info["subs"]["V1"],this_row.n); this_row ["V2S"] = classify(info["subs"]["V2"],this_row.n); this_row ["V3S"] = classify(info["subs"]["V3"],this_row.n); rows.push (this_row); } } }); /*let classify_site = (s)=> { let site_class = {}; if (s) { if (s.bMEME.p <= qv) { site_class ['E'] = s.bMEME.br; } if (s.bFEL.p <= qv) { if (s.bFEL.a < s.bFEL.b) { site_class ['P'] = 1; } else { site_class ['N'] = 1; } } if (s.bCFEL.q <= 0.2) { site_class ['X'] = 1; } if (s.FADE) { let fd = _.filter (_.toPairs(s.FADE), (p)=>p[1] >= 100); if (fd.length) { site_class['D'] = fd; } } } return site_class; } let site_freqs = (s)=> { let site_f = []; if (s) { site_f = _.toPairs(_.find (_.toPairs (s.aa), (d)=>d[0] != 'Reference')[1]); } return site_f; } let ref_freqs = (s)=> { let site_f = []; if (s) { site_f = _.toPairs(_.find (_.toPairs (s.aa), (d)=>d[0] == 'Reference')[1]); } return site_f; } _.each (joint_MEME, (d)=> { let c = d; let this_row = {}; if (d in reverseLookup == false) { c++; } this_row.n = d; this_row.coord = reverseLookup[c][0][0]; _.each (analysis_info, (info,k)=> { this_row[k] = classify_site (info[d]); this_row[k+"F"] = freq_string(site_freqs (info[d])); if (k == "V1") { this_row ["REF_F"] = freq_string(ref_freqs (info[d])); } }); rows.push (this_row); });*/ return rows; }32:Te3b,joint_site_list = { let site_list = {}; let sub_count = {}; let internal_branches_only = options.indexOf ("ib") >= 0; let internal_selection_only = options.indexOf ("is") >= 0; let meme_use_pv = options.indexOf ("pv") < 0; let filter_gaps = options.indexOf ("gaps") >= 0; let exclude_sites = {}; if (filter_gaps) { _.each (analysis_info, (data, clade)=> { exclude_sites[clade] = {}; _.each (data, (segment)=> { _.each (segment.subs, (subs, site)=>{ if (_.size (subs) >= 5 && _.filter (subs, (value)=> value == '---').length / _.size (subs) > 0.65) { exclude_sites[clade][site] = 1; } }); }); }); } else { _.each (analysis_info, (data, clade)=>exclude_sites[clade] = {}); } let add_clade_to_meme = (site, clade, tag)=> { tag = tag || "meme"; if (site in site_list == false) { site_list[site] = {}; } if (!site_list[site][tag]) { site_list[site][tag] = new Set(); } site_list[site][tag].add (clade); } if (meme_use_pv) { _.each (analysis_info_combined.sites, (data, clade)=> { _.each (data, (info, site)=> { if ("bMEME" in info) { if (info.bMEME.p <= pv || (!internal_selection_only && info.lMEME.p <= pv)) { if (!exclude_sites[clade][site]) { add_clade_to_meme(site, clade); } } } }); }); } _.each (analysis_info, (data, clade)=> { sub_count [clade] = 0; _.each (data, (segment)=> { if (meme_use_pv == false) { _.each (segment.tree_tags, (tag)=> { if (tag[0].length && tag[1] && tag[0] != "Reference") { _.each (tag[3], (site)=> { if (!exclude_sites[clade][site]) add_clade_to_meme(site, clade); }); if (!internal_selection_only) { _.each (tag[4], (site)=> { if (!exclude_sites[clade][site]) add_clade_to_meme(site, clade, 'meme-all'); }); } } }); } _.each (segment.subs, (subs, site)=>{ if (exclude_sites[clade][site]) return; _.each (subs, (codon, node)=> { if (!segment.tree_tags[node]) return; if (internal_branches_only && segment.tree_tags[node][1] == false) return; if (segment.tree_tags[node][0].length && segment.tree_tags[node][0] != "Reference") { if (site in site_list == false) { site_list[site] = {}; } if (!site_list[site]['subs']) { site_list[site]['subs'] = {}; } if (!site_list[site]['subs'][clade]) { site_list[site]['subs'][clade] = new Set(); } if (codon != '---' ) { sub_count[clade]++; } site_list[site]['subs'][clade].add (translation_table[codon]); } }); }); }); }); return [site_list,sub_count]; }33:[] 2f:{"id":"f423b9161a2d2b7b","slug":"combined-view-of-sites-which-may-be-experiencing-convergen","title":"Combined view of sites which may be experiencing convergence in N501Y clades","thumbnail":"352ba439f20fdb810de2fd2ed990993c1e2fbee596ff484cec40abd6e8b59a97","access_level":"public","create_time":"$D2021-02-05T14:20:42.376Z","update_time":"$D2021-04-24T21:58:24.101Z","view_time":null,"first_public_time":"$D2021-02-05T17:15:05.474Z","fork":{"id":"427dc44aef36d06d","slug":"n501y-sites","title":"Combined view of sites under selection in N501Y clades","owner":{"id":"9bf992815a7b66f2","login":"spond","name":"Sergei Pond"}},"stars":0,"starred":false,"owner":{"type":"team","id":"9bf992815a7b66f2","login":"spond","name":"Sergei Pond","avatar_url":"https://avatars-next.observableusercontent.com/23c8c466a3ef27f1444c95a9ed8e52bcf08888403b6d4efbd039319b0761c13d","tier":"starter_2024"},"version":668,"publish_version":668,"role":null,"authored":false,"authors":[{"type":"individual","id":"f439ea11e6786e12","login":"spond","name":"Sergei Pond","avatar_url":"https://avatars-next.observableusercontent.com/23c8c466a3ef27f1444c95a9ed8e52bcf08888403b6d4efbd039319b0761c13d","tier":null}],"members":[{"type":"individual","id":"f439ea11e6786e12","login":"spond","name":"Sergei Pond","avatar_url":"https://avatars-next.observableusercontent.com/23c8c466a3ef27f1444c95a9ed8e52bcf08888403b6d4efbd039319b0761c13d","tier":null,"role":"editor"}],"comments":[],"custom_thumbnail":false,"trashed":false,"subscribed":null,"forks":3,"collection":null,"type":"notebook","body":{"title":"Combined view of sites which may be experiencing convergence in N501Y clades","cells":[{"id":0,"value":"md`# Combined view of sites which may be experiencing convergence in N501Y clades`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":568,"value":"viewof data_from = select ({\n options : [\"February 2021\", \"April 2021\"],\n title: \"Use analysis from: \"\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":573,"value":"viewof options = checkbox({\n title: \"Filtering options\",\n options: [\n { value: \"ib\", label: \"Convergent substitutions must occur along internal branches\" },\n { value: \"is\", label: \"Selection signal for internal branches only\" },\n { value: \"pv\", label: \"Use MEME EBF-flagged branches as evidence of selection\" },\n { value: \"gaps\", label: \"Filter sites with excessive gaps\"}\n ],\n submit: false\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":626,"value":"viewof pv = text ({\n value : 0.05,\n title: \"p-value for MEME\",\n submit: true\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":211,"value":"md`\n${joint_MEME_list.length} sites found.\n\n1. Signature OR Pos selection with MEME on internal tree branch AND one or more of the following for other lineages\n 1. Pos selection with MEME\n 2. Convergent mutations on internal tree branches to other variant signature\n 3. Convergent mutations on internal tree branches in different variant lineage`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":520,"value":"legend({\n color: color,\n title: \"Evolutionary Probability\",\n width: 900,\n marginLeft: 50\n})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":136,"value":"table(\n joint_MEME_list,\n {\n maxHeight:2000,\n limit: 500,\n enableFilter: false,\n enableCSVDownload: false,\n columns: [{ \n key: 'n',\n name: 'Nucleotide',\n }, \n { \n key: 'coord',\n name: 'Site',\n },\n { \n key: 'V1',\n name: 'V1',\n render: (val)=>site_string(val)\n },\n { \n key: 'V2',\n name: 'V2',\n render: (val)=>site_string(val)\n },\n { \n key: 'V3',\n name: 'V3',\n render: (val)=>site_string(val)\n },\n { \n key: 'V1S',\n name: 'V1 subs to',\n render: (val)=>freq_string(val)\n },\n { \n key: 'V2S',\n name: 'V2 subs to',\n render: (val)=>freq_string(val)\n },\n { \n key: 'V3S',\n name: 'V3 subs to',\n render: (val)=>freq_string(val)\n }\n ]\n }\n)","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":157,"value":"md`Selection legend\n 1. : Signtature mutation\n 1. : MEME support\n 2. : convergent subsitutions to another lineage (no MEME support)\n 2. : convergent subsitutions to another lineage AND MEME support\n\n`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":33,"value":"md`### Computed`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":463,"value":"pct_fmt = d3.format (\".2p\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":145,"value":"freq_string = (f)=> {\n \n if (f) {\n if (_.isArray (f)) {\n return _.map (f, (d)=> {\n if (d[0] == '-') {\n return \"\"; \n } \n return \"\" + d[0] + \"\";\n }).join (\"\");\n } else {\n f.delete (\"-\");\n return [...f].join (\",\");\n }\n }\n return \"\";\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":470,"value":"color = d3.scaleThreshold ().domain ([1e-6,1e-4,0.01,0.05,0.5]).range (['red','yellow','orange','purple','blue','green'])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":490,"value":"colorText = d3.scaleThreshold ().domain ([1e-6,1e-4,0.01,0.05,0.5]).range (['black','black','black','white','white','white'])","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":174,"value":"site_string = (d)=> {\n let res = [];\n if (d) {\n if ('sig' in d) {\n res.push (''); \n }\n if ('meme' in d) {\n if ('subs' in d) {\n res.push ('');\n } else {\n res.push ('');\n }\n } else {\n if ('subs' in d) {\n res.push ('');\n }\n }\n \n }\n return res.join ('');\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":104,"value":"$31","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":552,"value":"convergent_site_set = {\n let csl = {};\n _.each (joint_MEME_list, (s)=> {\n _.each (['V1','V2','V3'], (clade)=> {\n if (s[clade] && s[clade].subs) {\n if (s.n in csl == false) {\n csl[s.n] = {};\n }\n csl[s.n][clade] = new Set (_.map (s[s[clade].subs + \"S\"], (d)=>d[0]));\n \n \n }\n });\n });\n return csl;\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":547,"value":"convergent_site_list = {\n let sub_count = {};\n _.each (analysis_info, (data, clade)=> {\n sub_count [clade] = 0;\n _.each (data, (segment)=> {\n _.each (segment.subs, (subs, site)=>{\n _.each (subs, (codon, node)=> {\n if (!segment.tree_tags[node]) return;\n if (segment.tree_tags[node][0].length && segment.tree_tags[node][0] != \"Reference\") {\n if (codon != '---') {\n if ( site in convergent_site_set && clade in convergent_site_set[site] && convergent_site_set[site][clade].has (translation_table[codon]))\n sub_count[clade]++;\n } \n }\n });\n });\n });\n });\n return sub_count;\n \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":36,"value":"$32","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":68,"value":"qvalues = (pvalues,cut)=> {\n cut = cut || qv;\n return _.chain (pvalues).sortBy ((d)=>d[1]).map ((d,i)=>[d[0], d[1], d[1] * pvalues.length/(i+1)]).filter ((d)=>(d[1] <= cut)).value();\n};","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":53,"value":"site2gene = (site)=> {\n if (site in reverseLookup || (+site + 1) in reverseLookup) {\n let lkup = reverseLookup[site] || reverseLookup[+site + 1];\n if (lkup.length > 2) {\n return lkup[1][0].split (\"/\")[0];\n }\n return lkup[0][0].split (\"/\")[0];\n }\n return \"\";\n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":391,"value":"_.mapValues (analysis_info, (v)=>_.mapValues (v, (g)=>g.busted && g.busted.p <= 0.05))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":396,"value":"_.mapValues (analysis_info, (v)=>_.mapValues (v, (g)=>g.relax && g.relax.p <= 0.05))","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":20,"value":"md`### Data`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":30,"value":"qv = 0.01","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":23,"value":"analysis_info_combined = {\n let base_url = \"https://data.hyphy.org/web/SARS-CoV-2/clades/\";\n let info;\n \n if (data_from == \"April 2021\") {\n info = [{label: \"V1\", value: 'B.1.1.7-202104'},\n { label: \"V2\", value: 'B.1.351-202104'},\n { label: \"V3\", value: 'P.1-202104'}];\n } else {\n info = [{label: \"V1\", value: 'B117-Feb2020'},\n { label: \"V2\", value: 'B1351-Feb2020'},\n { label: \"V3\", value: 'P1-Feb2020'}];\n }\n \n let annotations = {},\n site_info = {};\n \n for (let i = 0; i < info.length; i++) {\n annotations[info[i].label] = await d3.json (base_url + info[i].value + \"-summary.json\");\n site_info[info[i].label] = await d3.json (base_url + info[i].value + \"-annotation.json\");\n }\n \n return {\n summary : annotations,\n sites : site_info\n }; \n}","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":606,"value":"analysis_info = analysis_info_combined.summary","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":340,"value":"signatures= _.map ( [[\"S/18\",0,1,1],[\"S/20\",0,0,1],[\"S/26\",0,0,1],[\"S/80\",0,1,0],[\"S/138\",0,0,1],[\"S/190\",0,0,1],[\"S/215\",0,1,0],[\"S/417\",0,1,1],[\"S/484\",0,1,1],[\"S/501\",1,1,1],[\"S/570\",1,0,0],[\"S/655\",0,0,1],[\"S/681\",1,0,0],[\"S/701\",0,1,0],[\"S/716\",1,0,0],[\"S/982\",1,0,0],[\"S/1027\",0,0,1],[\"S/1118\",1,0,0],[\"S/1176\",0,0,1],[\"ORF1a/265\",0,1,0],[\"ORF1a/1001\",1,0,0],[\"ORF1a/1188\",0,0,1],[\"ORF1a/1655\",0,1,0],[\"ORF1a/1708\",1,0,0],[\"ORF1a/1795\",0,0,1],[\"ORF1a/2230\",1,0,0],[\"ORF1a/3353\",0,1,0],[\"ORF1b/1264\",0,0,1],[\"ORF3a/57\",0,1,0],[\"ORF3a/171\",0,1,0],[\"ORF3a/253\",0,0,1],[\"E/71\",0,1,0],[\"ORF8/92\",0,0,1],[\"N/3\",1,0,0],[\"N/80\",0,0,1],[\"N/205\",0,1,0],[\"N/235\",1,0,0]], (s)=>{return {'site':+jsonCodonLookup[s[0].toUpperCase()],'V1': s[1], 'V2' : s[2], 'V3' : s[3]}})","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":425,"value":"annotation = _.mapValues (await d3.json ('https://raw.githubusercontent.com/veg/SARS-CoV-2/compact/data/comparative-annotation.json'), (d)=>{\n if (d.evo) {\n d.evo = _.chain (d.evo).toPairs().map ((d)=>[translation_table[d[0]], d[1]]).groupBy ((d)=>d[0]).mapValues((v)=>d3.sum (v, (c)=>c[1])).value();\n }\n return d;\n});","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":5,"value":"md`### Libraries`","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":522,"value":"import {legend} from \"@d3/color-legend\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":171,"value":"html``","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":239,"value":"import {translation_table} from \"@spond/usual-vs-unusual-mutation-analyses\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":14,"value":"import { table } from \"@gampleman/table\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":47,"value":"import {reverseLookup} from \"@spond/sc2-genes\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":17,"value":"import {select,checkbox, autoSelect, text} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":343,"value":"import {jsonCodonLookup} from \"@spond/sars_cov_2_sites\"","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":8,"value":"d3 = require (\"d3\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"},{"id":11,"value":"_ = require (\"lodash\")","pinned":false,"hidden":false,"mode":"ojs","author":"$undefined","summary":"$undefined"}],"files":"$W33","stdlib":"1","titling":"auto"}} 30:"$2f:body"