30:T4ba,md`# SD-PIRC sequence QA Performing phylogeny and distance based consistency checks on SD-PIRC sequence data. Because we have multiple sequences for **${node_info['multiples'].length}** out of **${_.keys(node_info['id']).length}** individuals (either as replicates done at different labs, or sampled at different time-points) they can be used to run some simple QA on sequence data. The expectation is that **multiple sequences from the same individual should**: 1. Cluster in the joint phylogeny. 2. Be linked (≤1.5%) to each other in the TN93 distance space. If either of those is broken (1 is easier to break than 2, because TN93 linked sequences may not be neatly clustered in the phylogeny), this could be due to one of three things (possibly more, of course, but these are at the top of my list) 1. Sequence issues -- mislabeling, sample mix-up, etc (more likely _a priori_) 2. A known (or missed) instance of dual infection, where sequences at different time points are legitimately different (less likely than 1) 3. A sequence that managed to rapidly evolve in a monoinfected host (**very** unlikely) The following table summarizes the results of tests done on the **current** database build. `31:T8e4,render = {const svg = d3.select(DOM.svg(+width, +height)); var t = tree (d3.layout.newick_parser ("(1,2)")).svg(svg) .size([+height, +width]) .layout(); var highlight_color = (n) => { if ( n.name.indexOf (patient) >= 0) { return "selected"; } return null; }; t = tree (d3.layout.newick_parser(newick_string)) .svg(svg) .size([+height, +width]) //.branch_name (d => {return "";}) .style_edges (function (element, data) { if (d3.layout.phylotree.is_leafnode (data.target)) { const group = highlight_color (data.target); if (group) { element.style ("stroke", "red"); element.style ("stroke-width", "5px"); } return; } else { //console.log (data.target); if (data.target.bootstrap_values >= bootstrap_cutoff) { element.style ("stroke-width", "5px"); return; } } element.style ("stroke", null); element.style ("stroke-width", null); }) .style_nodes (function (element, data) { if (d3.layout.phylotree.is_leafnode (data)) { const group = highlight_color (data); if (group) { element.style ("fill", "red"); } return; } element.style ("fill", null); }).layout(); t.traverse_and_compute ((n) => { if (!d3.layout.phylotree.is_leafnode (n)) { if ("collapsed" in n) { delete n["collapsed"]; } n.tagged_view = false; } }); t.traverse_and_compute ((n) => { if (highlight_color (n)) { n.tagged_view = true; } if (n.tagged_view && n.parent) { n.parent.tagged_view = true; } }); t.traverse_and_compute ((n) => { if (!d3.layout.phylotree.is_leafnode (n) && !n.tagged_view && n.parent && n.parent.tagged_view) { //console.log (n); n.collapsed = true; } }); t.placenodes().layout(); return svg.node();}32:T90e,node_info = { var sequences_by_id = {}; var id_list = {}; var tree_parser = tree (d3.layout.newick_parser(newick_string)); tree_parser.traverse_and_compute (n => { if (d3.layout.phylotree.is_leafnode (n)) { var id = n.name.split ('|')[0]; id_list[id] = id_list[id] ? 1 + id_list[id] : 1; if (!(id in sequences_by_id)) { sequences_by_id[id] = []; } sequences_by_id[id].push (n.name); } }); var monophyletic_results = {}; _.each (id_list, (v, k) => { if (v > 1) { tree_parser.traverse_and_compute ( (n) => {n.count_below = 0; n.tagged_below = 0;}); var check_pass = true; tree_parser.traverse_and_compute ( (n) => { if (d3.layout.phylotree.is_leafnode (n)) { if (n.name.indexOf (k) == 0 && n.name.indexOf ('consensus') < 0) { n.tagged_below = 1; } n.count_below = 1; } if (n.parent) { n.parent.count_below += n.count_below; n.parent.tagged_below += n.tagged_below; } if (!d3.layout.phylotree.is_leafnode (n)) { if (+n["bootstrap_values"] >= bootstrap_cutoff) { if ( n.tagged_below > 0 && n.count_below > n.tagged_below && n.tagged_below < v && tree_parser.branch_length()(n) >= min_monophyly_length) { check_pass = false; } } } } ); monophyletic_results[k] = check_pass; } else { monophyletic_results[k] = null; } }); var linked = {}; _.each (sequences_by_id, (seqs, id) => { var pairs = []; for (var i = 0; i < seqs.length; i++) { for (var j = i+1; j < seqs.length; j++) { pairs.push ([seqs[i], seqs[j], (seqs[i] + "|" + seqs[j] in tn93) || (seqs[j] + "|" + seqs[i] in tn93)]); } } if (pairs.length) { linked[id] = pairs; } }); return { id : id_list, multiples : _.filter (_.map (id_list, (n,k) => {return [k,n];}), (n) => { return n[1] > 1;}), seqs : sequences_by_id, monophyly : monophyletic_results, tn93 : linked } }33:[] 2e:{"title":"SD-PIRC sequence QA","cells":[{"id":0,"value":"$30","pinned":false,"hidden":false,"mode":"ojs"},{"id":85,"value":"md`**Table 1** Summary of quality control metrics (see above) for the **${node_info['multiples'].length}** individuals with multiple bulk sequences data. Also shown is the _a priori_ provided dual infection status. See the **Patient level view** section below for more details on individual patients`","pinned":false,"hidden":false,"mode":"ojs"},{"id":95,"value":"table(_.map (node_info['multiples'], (data) => {\n var result = {'OUR#' : data[0], 'Sequence count' : data[1], 'DI' : dual_status [data[0]], 'Cluster in tree' : node_info['monophyly'][data[0]], 'Link together' : _.filter (node_info['tn93'][data[0]], (d) => {return !d[2];}).length == 0, 'Checks passed' : [0,0]};\n \n _.each (['Cluster in tree','Link together'], (d) => {if (result[d] !== null) {\n result['Checks passed'][0] ++;\n if (result[d]) {\n result['Checks passed'][1] ++;\n }\n }});\n \n return result;\n}),{//nully: () => 'No data',\n limit: 500,\n columns: [{ \n key: 'OUR#',\n render : val => `${val}`\n }, \n 'Sequence count', \n 'DI',\n {key : 'Cluster in tree',\n render: val => `${render_icon(val)}`\n },\n {key : 'Link together',\n render: val => `${render_icon(val)}`\n },\n \n {key: 'Checks passed', render: val => `${val[1]} / ${val[0]}`}]\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":86,"value":"md`#### Patient level view`","pinned":false,"hidden":false,"mode":"ojs"},{"id":49,"value":"viewof height = slider({\n min: 600, \n max: 50000, \n value : 2000,\n step: 100, \n format: \".0f\",\n description: \"Tree viewer height in pixels\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":52,"value":"viewof width = slider({\n min: 600, \n max: 2000, \n value : 800,\n step: 100, \n format: \".0f\",\n description: \"Tree viewer width in pixels\"\n})","pinned":false,"hidden":false,"mode":"ojs"},{"id":75,"value":"viewof patient = DOM.select(_.map(node_info['multiples'], (v) => {return v[0];}))","pinned":false,"hidden":false,"mode":"ojs"},{"id":77,"value":"md`**Figure 1** Partially collapsed joint tree of bulk and NGS rt consensus data with sequences for **${patient}** highlighted. Branches with sufficient bootstrap support are highlighted with thicker lines`","pinned":false,"hidden":false,"mode":"ojs"},{"id":116,"value":"$31","pinned":false,"hidden":false,"mode":"ojs"},{"id":138,"value":"md`**Table 2** Information on TN93 linkage among multiple sequences from **${patient}**.`","pinned":false,"hidden":false,"mode":"ojs"},{"id":145,"value":"{\n return table(_.map (node_info['tn93'][patient], (n) => {return n;}), \n {\n columns: [{ \n key: '0',\n name : \"Seq 1\",\n \n }, \n { \n key: '1',\n name : \"Seq 2\",\n \n },\n { key : '2',\n name : 'Linked at 1.5%',\n render: val => `${render_icon(val)}`\n }\n ]\n });\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":8,"value":"md`### Dependencies from here on`","pinned":false,"hidden":false,"mode":"ojs"},{"id":24,"value":"_ = window._ = await require(\"lodash@4\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":26,"value":"d3 = {\n const d3 = window.d3 = await require(\"d3@3\");\n await require(\"phylotree@v0.2.0-alpha.1\").catch(() => {});\n return d3;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":21,"value":"d3v5 = require(\"https://d3js.org/d3.v5.min.js\")","pinned":false,"hidden":false,"mode":"ojs"},{"id":33,"value":"import {slider} from \"@jashkenas/inputs\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":29,"value":"bootstrap_cutoff = 0.9","pinned":false,"hidden":false,"mode":"ojs"},{"id":35,"value":"min_monophyly_length = 0.01","pinned":false,"hidden":false,"mode":"ojs"},{"id":19,"value":"import { table } from \"@gampleman/table\"","pinned":false,"hidden":false,"mode":"ojs"},{"id":11,"value":"render_icon = (value) => {\n if (value == 'warning') {\n return '';\n } \n if (value) {\n return '';\n }\n if (value == false) {\n return '';\n }\n\n return ''; \n }","pinned":false,"hidden":false,"mode":"ojs"},{"id":37,"value":"tree = {\n return d3.layout.phylotree()\n .options({\n 'left-right-spacing': 'fit-to-size', // fit to given size top-to-bottom\n 'top-bottom-spacing': 'fit-to-size', // fit to given size left-to-right\n 'selectable': true, // make nodes and branches not selectable\n 'collapsible': true, // turn off the menu on internal nodes\n 'transitions' : false, // turn off animations\n 'align-tips' : true\n })\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":43,"value":"newick_string = await d3v5.text ('https://dl.dropbox.com/s/f57lwoxn82104di/tree.nwk')","pinned":false,"hidden":false,"mode":"ojs"},{"id":125,"value":"tn93 = {\n var raw = await d3v5.csv (\"https://dl.dropbox.com/s/bj5m437fqo63bf0/distances_0.02.csv\"); \n var linked_pairs = {};\n _.each (raw, (d) => {\n if (d[\"Distance\"] <= 0.015) {\n linked_pairs[d[\"ID1\"] + \"|\" + d[\"ID2\"]] = 1;\n }\n });\n return linked_pairs;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":112,"value":"dual_status = {\n const ds = await d3v5.csv (\"https://dl.dropbox.com/s/1eo8jj2euo55416/dual.csv\");\n var dsa = {};\n _.each (ds, (v) => {console.log (v); dsa[v['ID']] = v['Status'];});\n return dsa;\n}","pinned":false,"hidden":false,"mode":"ojs"},{"id":54,"value":"$32","pinned":false,"hidden":false,"mode":"ojs"},{"id":41,"value":"html``\n\n\n","pinned":false,"hidden":false,"mode":"ojs"},{"id":39,"value":"html``","pinned":false,"hidden":false,"mode":"ojs"}],"files":"$W33","stdlib":"1","titling":"auto"}