All_families.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from opentree import OT\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "forwards.tsv  README.html  synonyms.tsv  taxonomy.tsv  version.txt\r\n"
     ]
    }
   ],
   "source": [
    "!ls ../ott3.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "taxonomy_file = \"../ott3.2/taxonomy.tsv\"\n",
    "assert os.path.exists(taxonomy_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "fam_dict = {}\n",
    "for lin in open(taxonomy_file):\n",
    "        lii=lin.split('\\t|\\t')\n",
    "        if len(lii[2].split(' ')) > 1:\n",
    "            pass\n",
    "        elif lii[2].endswith(\"aceae\"):\n",
    "            fam_dict[lii[2]]=lii\n",
    "        elif lii[2].endswith(\"idae\"):\n",
    "            fam_dict[lii[2]]=lii"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "maybe_not_fams = set()\n",
    "for fam in fam_dict:\n",
    "    if fam_dict[fam][3] == 'family':\n",
    "        pass\n",
    "    else:\n",
    "        maybe_not_fams.add(fam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "163"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(maybe_not_fams)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fams_ott_id = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "for fam in fam_dict:\n",
    "    if fam not in fams_ott_id:\n",
    "        ott_id = fam_dict[fam][0]\n",
    "        fams_ott_id[fam] = OT.synth_node_info(ott_id = ott_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "fams_node_id = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "for fam in fams_ott_id:\n",
    "    try:\n",
    "        nid = fams_ott_id[fam].response_dict.get('node_id')\n",
    "    except:\n",
    "        assert(str(fams_ott_id[fam].response) == '<Response [400]>')\n",
    "        nid=None\n",
    "    fams_node_id[fam] =  nid\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('fam_synth_node_info.json', 'w') as outfile:\n",
    "    json.dump(fams_node_id, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "node_ids = {}\n",
    "fams_in_tree = set()\n",
    "not_in_tree = set()\n",
    "for fam in fams_ott_id:\n",
    "    nid = fams_node_id[fam]\n",
    "    if nid:\n",
    "        if nid not in node_ids:\n",
    "            node_ids[nid] = set()\n",
    "        node_ids[nid].add(fam)\n",
    "        fams_in_tree.add(fam)\n",
    "    else:\n",
    "        not_in_tree.add(fam)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('fam_synth_node_info.csv', 'w') as outfile:\n",
    "    outfile.write(\"Family, ott_id, synthetic node id,notes\\n\")\n",
    "    for fam in fams_node_id:\n",
    "        ott_id = fam_dict[fam][0]\n",
    "        node_id = fams_node_id[fam]\n",
    "        notes = ''\n",
    "        if node_id != None:\n",
    "            if len(node_ids[node_id]) > 1:\n",
    "                taxa = \" - \".join(list(node_ids[node_id]))\n",
    "                notes = 'Family maps to same node as' + taxa\n",
    "        outfile.write(\"{},{}, {}, {}\\n\".format(fam, ott_id, node_id, notes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10104"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(fams_in_tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "fi = open(\"families_not_in_tree.tsv\",'w')\n",
    "fi.write('name\\t ottid, sources, flags\\n')\n",
    "for fam in not_in_tree:\n",
    "    fi.write(\"{}\\t{}\\t{}\\t{}\\n\".format(fam,fam_dict[fam][0],fam_dict[fam][-4],fam_dict[fam][-2]))\n",
    "\n",
    "fi.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "fi = open(\"families_in_tree.tsv\",'w')\n",
    "fi.write('name \\t ottid \\t sources \\t node_id \\t notes\\n')\n",
    "for fam in fams_in_tree:\n",
    "        ott_id = fam_dict[fam][0]\n",
    "        sources = fam_dict[fam][4]\n",
    "        node_id = fams_node_id[fam]\n",
    "        notes = ''\n",
    "        if node_id != None:\n",
    "            if len(node_ids[node_id]) > 1:\n",
    "                taxa = \" - \".join(list(node_ids[node_id]))\n",
    "                notes = 'Maps to same node as: ' + taxa\n",
    "        fi.write(\"{}\\t{}\\t{}\\t{}\\t{}\\n\".format(fam, ott_id, sources, node_id, notes))\n",
    "\n",
    "fi.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'worms:204,ncbi:47127,gbif:6876,irmng:115825'"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree = OT.synth_induced_tree(node_ids = list(node_ids.keys()), label_format=\"name\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['broken', 'newick', 'supporting_studies'])\n"
     ]
    }
   ],
   "source": [
    "tree.tree.write(path=\"allfam.tre\",schema=\"newick\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_problem_characters(tree, prob_char = \"():#\", replace_w = '?'):\n",
    "    problem_characters = set(prob_char)\n",
    "    for node in tree:\n",
    "        if node.taxon:\n",
    "            for char in problem_characters:\n",
    "                node.taxon.label = node.taxon.label.replace(char,replace_w)\n",
    "        elif node.label:\n",
    "            for char in problem_characters:\n",
    "                node.label = node.label.replace(char,replace_w)\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "local_tree = copy.deepcopy(tree.tree)\n",
    "\n",
    "remove_problem_characters(local_tree)\n",
    "local_tree.write(path=\"allfam.tre\",schema=\"newick\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Don't forget to cite your friendly phylogeneticists!\n",
    "studies = tree.response_dict['supporting_studies']\n",
    "cites = OT.get_citations(studies) #this can be a bit slow\n",
    "fi = open(\"all_famcites.txt\",\"w\")\n",
    "fi.write(cites)\n",
    "fi.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['5252602',\n",
       " '464323',\n",
       " 'Pasteurellaceae',\n",
       " 'no rank - terminal',\n",
       " 'silva:GM884484',\n",
       " 'Pasteurellaceae (silva:GM884484)',\n",
       " '',\n",
       " '\\n']"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fam_dict['Pasteurellaceae']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "rev_fam =  {'ott{}'.format(fam_dict[k][0]):k for k in fam_dict}\n",
    "\n",
    "\n",
    "for node in local_tree:\n",
    "    if node.taxon:\n",
    "        if node.taxon.label in node_ids:\n",
    "            fam = '-'.join(list(node_ids[node.taxon.label]))\n",
    "            node.taxon.label = \"{} broken - MRCA {}\".format(fam, node.taxon.label)\n",
    "    elif node.label:\n",
    "         if node.label in node_ids:\n",
    "            fam = '-'.join(list(node_ids[node.label]))\n",
    "            node.label = \"{} broken - MRCA {}\".format(fam, node.label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "local_tree.write(path=\"allfam_labeled.tre\",schema=\"newick\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#Alternate approach:\n",
    "Throw everything into it, and let some taxa be broken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = OT.synth_induced_tree(ott_ids=list(fams_ott_id.keys()),  label_format='name_and_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "broken = output.response_dict['broken']\n",
    "len(broken)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rev_fam =  {'ott{}'.format(v):k for k, v in fam_dict.items()}\n",
    "rev_broken = {}\n",
    "\n",
    "for k, v in broken.items():\n",
    "    fam = rev_fam[k]\n",
    "    if v not in rev_broken:\n",
    "        rev_broken[v] = [fam]\n",
    "    else:\n",
    "        rev_broken[v].append(fam)\n",
    "\n",
    "\n",
    "for node in output.tree:\n",
    "    if node.taxon:\n",
    "        if node.taxon.label in rev_broken:\n",
    "            fam = '-'.join(rev_broken[node.taxon.label])\n",
    "            node.taxon.label = \"{} broken - MRCA {}\".format(fam, node.taxon.label)\n",
    "    elif node.label:\n",
    "         if node.label in rev_broken:\n",
    "            fam = '-'.join(rev_broken[node.label])\n",
    "            node.label = \"{} broken - MRCA {}\".format(fam, node.label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We can check which taxa are in the tree\n",
    "synth_leaf_ott_ids = set()\n",
    "synth_node_ids = set()\n",
    "\n",
    "for node in output.tree:\n",
    "    if node.taxon:\n",
    "        synth_leaf_ott_ids.add(node.taxon.label.split()[-1])\n",
    "    if node.label:\n",
    "        synth_node_ids.add(node.label.split()[-1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "outfi = open('fam_info_synth_1.csv','w')\n",
    "outfi.write(\"fam_name, ott_id, broken_taxon, mrca_of_descendents, in_synth_tree\\n\")\n",
    "for fam_name in fam_dict:\n",
    "    ott_id = fam_dict[fam_name]\n",
    "    brok = \"-\"\n",
    "    mrca = \"-\"\n",
    "    in_synth_tree = 'False'\n",
    "    if ott_id != 'None':\n",
    "        ott_str = \"ott{}\".format(ott_id)\n",
    "        if ott_str in broken:\n",
    "            brok = \"T\"\n",
    "            mrca = broken[ott_str].strip()\n",
    "            if mrca in synth_leaf_ott_ids:\n",
    "                in_synth_tree = 'Leaf'\n",
    "            elif mrca in synth_node_ids:\n",
    "                in_synth_tree = 'Node'\n",
    "        if ott_str in synth_leaf_ott_ids:\n",
    "            in_synth_tree = 'Leaf'\n",
    "        elif ott_str in synth_node_ids:\n",
    "            in_synth_tree = 'Node'\n",
    "            \n",
    "    outfi.write(\"{},{},{},{},{}\\n\".format(fam_name, ott_id, brok, mrca, in_synth_tree))\n",
    "outfi.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}