tde_run.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pkg_resources \n",
    "from tdev2.readers.gold_reader import *\n",
    "from tdev2.readers.disc_reader import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tdev2.measures.ned import *\n",
    "from tdev2.measures.boundary import *\n",
    "from tdev2.measures.grouping import *\n",
    "from tdev2.measures.coverage import *\n",
    "from tdev2.measures.token_type import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prf2dict(dct, measurename, obj):\n",
    "    # save precision-recall-f of tde objects to dictionary\n",
    "    dct[measurename + '_P'] = obj.precision\n",
    "    dct[measurename + '_R'] = obj.recall\n",
    "    dct[measurename + '_F'] = obj.fscore\n",
    "    \n",
    "    return dct\n",
    "\n",
    "\n",
    "def compute_scores(gold, disc, measures=[]):\n",
    "    scores = dict()\n",
    "    \n",
    "    # Launch evaluation of each metric\n",
    "    if len(measures) == 0 or \"boundary\" in measures:\n",
    "        print('Computing Boundary...')\n",
    "        boundary = Boundary(gold, disc)\n",
    "        boundary.compute_boundary()\n",
    "        scores = prf2dict(scores, 'boundary', boundary)\n",
    "        \n",
    "    if len(measures) == 0 or \"grouping\" in measures:\n",
    "        print('Computing Grouping...')\n",
    "        grouping = Grouping(disc)\n",
    "        grouping.compute_grouping()\n",
    "        scores = prf2dict(scores, 'grouping', grouping)    \n",
    "        \n",
    "    if len(measures) == 0 or \"token/type\" in measures:\n",
    "        print('Computing Token and Type...')\n",
    "        token_type = TokenType(gold, disc)\n",
    "        token_type.compute_token_type()\n",
    "        scores['token_P'],scores['token_R'],scores['token_F'] = token_type.precision[0], token_type.recall[0], token_type.fscore[0]\n",
    "        scores['type_P'],scores['type_R'],scores['type_F'] = token_type.precision[1], token_type.recall[1], token_type.fscore[1]        \n",
    "        \n",
    "    if len(measures) == 0 or \"coverage\" in measures:\n",
    "        print('Computing Coverage...')\n",
    "        coverage = Coverage(gold, disc)\n",
    "        coverage.compute_coverage()\n",
    "        scores['coverage'] = coverage.coverage\n",
    "\n",
    "        \n",
    "    if len(measures) == 0 or \"coverageNS\" in measures:\n",
    "        print('Computing Coverage No Single...')\n",
    "        coverage = Coverage_NoSingleton(gold, disc)\n",
    "        coverage.compute_coverage()\n",
    "        scores['coverageNS'] = coverage.coverage\n",
    "\n",
    "        \n",
    "    if len(measures) == 0 or \"ned\" in measures:\n",
    "        print('Computing NED...')\n",
    "        ned = Ned(disc)\n",
    "        ned.compute_ned()\n",
    "        scores['ned'] = ned.ned\n",
    "        \n",
    "        \n",
    "    \n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('build/lib/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir('../../')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/korhan/Desktop/tez/zr_exps/zr_mDGS_r_c3r_PCA40_Wh_rhothr0.3_T0.5_D10_dx18_dy9_medthr0.4_Tscore0.5_R12_castthr8_trimthr1.25_B50_P8/postpairwise_cost0.0006181300604703248_olap0.25//master_graph.class\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "sys.path.append('/home/korhan/Dropbox/tez_scripts/')\n",
    "from tdev2.utils import zrexp2tde, sdtw2tde, narrow_gold, read_config\n",
    "\n",
    "wrd_path = 'tdev2/share/phoenix.wrd'\n",
    "phn_path = 'tdev2/share/phoenix.phn'\n",
    "cnf_path = 'config.json'\n",
    "\n",
    "# read_config(cnf_path)\n",
    "\n",
    "# exp_path = '/home/korhan/Desktop/zerospeech2017/track2/src/ZRTools/exp/'\n",
    "# exp_path = '/home/korhan/Desktop/tez/sdtw_exps/sdtw_pairs_A_0_341_c3right_PCA_Wh_diag_olapFalse_w8_L2_extend_r0.3_diag_thr0.7_loss_funceuclid_end_cut0/postpairwise_cost0.001_olap0.25/'\n",
    "exp_path = '/home/korhan/Desktop/tez/zr_exps/zr_mDGS_r_c3r_PCA40_Wh_rhothr0.3_T0.5_D10_dx18_dy9_medthr0.4_Tscore0.5_R12_castthr8_trimthr1.25_B50_P8/postpairwise_cost0.0006181300604703248_olap0.25//'\n",
    "\n",
    "disc_clsfile = sdtw2tde(exp_path)\n",
    "\n",
    "print(disc_clsfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tdev2.eval_sign import try_compute_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "kwargs = {\n",
    "#     'config_file' : 'config.json',\n",
    "    'config_file' : '/home/korhan/Dropbox/config/config_mdgs.json',\n",
    "         'njobs': 1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "exp_path = '/home/korhan/Desktop/tez/zr_exps/zr_mDGS_r_c3r_PCA40_Wh_rhothr0.3_T0.5_D10_dx18_dy9_medthr0.4_Tscore0.5_R12_castthr8_trimthr1.25_B50_P8/postpairwise_cost0.010509214885954384_olap0.25/'\n",
    "disc_clsfile = sdtw2tde(exp_path)\n",
    "\n",
    "wrd_path = 'tdev2/share/mdgsClean_both.wrd'\n",
    "phn_path = 'tdev2/share/mdgsClean_both.phn'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Config file read, ovth 100.0 ***\n",
      "/home/korhan/Dropbox/config/config_mdgs.json\n",
      "Discovered Class file read\n",
      "\n",
      "966 unique intervals, 494 clusters with 971 nodes found\n"
     ]
    }
   ],
   "source": [
    "# read_config(cnf_path)\n",
    "gold = Gold(wrd_path=wrd_path, phn_path=phn_path, **kwargs) \n",
    "disc = Disc(disc_clsfile, gold) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing Boundary...\n",
      "Computing Grouping...\n",
      "Number of grouping jobs: 1\n",
      "Computing Token and Type...\n",
      "Computing Coverage...\n",
      "Computing Coverage...\n",
      "Computing Coverage No Single...\n",
      "*** Config file read, ovth 100.0 ***\n",
      "Computing NED...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'boundary_P': 1.0,\n",
       " 'boundary_R': 0.0222,\n",
       " 'boundary_F': 0.0435,\n",
       " 'n_clus': 494,\n",
       " 'n_node': 971,\n",
       " 'grouping_P': 0.0221,\n",
       " 'grouping_R': 0.2239,\n",
       " 'grouping_F': 0.0401,\n",
       " 'token_P': 0.0704,\n",
       " 'token_R': 0.003,\n",
       " 'token_F': 0.0058,\n",
       " 'type_P': 0.046,\n",
       " 'type_R': 0.0093,\n",
       " 'type_F': 0.0154,\n",
       " 'coverage': 0.046,\n",
       " 'coverageNS': 0.0456,\n",
       " 'coverageNS_f': 0.029,\n",
       " 'ned': 0.8644}"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "try_compute_scores(gold, disc, measures=[], **kwargs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/home/korhan/Desktop/tez/zr_exps/zr_mDGS_r_c3r_PCA40_Wh_rhothr0.3_T0.5_D10_dx18_dy9_medthr0.4_Tscore0.5_R12_castthr8_trimthr1.25_B50_P8/postpairwise_cost0.0006181300604703248_olap0.25//master_graph.class'"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "disc_clsfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing Boundary...\n",
      "Computing Grouping...\n",
      "Number of grouping jobs: 1\n",
      "Computing Token and Type...\n",
      "Computing Coverage...\n",
      "Computing Coverage No Single...\n",
      "Computing NED...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'boundary_P': 0.5131578947368421,\n",
       " 'boundary_R': 0.005875086619866831,\n",
       " 'boundary_F': 0.011594830287040826,\n",
       " 'grouping_P': 0.5307443365695792,\n",
       " 'grouping_R': 0.45054945054945056,\n",
       " 'grouping_F': 0.48687383012976415,\n",
       " 'token_P': 0.20609579100145137,\n",
       " 'token_R': 0.0023389501078881917,\n",
       " 'token_F': 0.004625407166123778,\n",
       " 'type_P': 0.4444444444444444,\n",
       " 'type_R': 0.02924451665312754,\n",
       " 'type_F': 0.054878048780487805,\n",
       " 'coverage': 0.005818596691386195,\n",
       " 'coverageNS': 0.006113361801525603,\n",
       " 'ned': 0.5161290322580645,\n",
       " 'n_clus': 521,\n",
       " 'n_node': 689}"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores = compute_scores(gold, disc)\n",
    "scores['n_clus'] = len(disc.clusters)\n",
    "scores['n_node'] = len(disc.intervals)\n",
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### save disc intervals and clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'1': [('1420216_B_both_208220_210800',\n",
       "   1.0,\n",
       "   80.0,\n",
       "   ((0.0, 6.0, 'ICH1'),\n",
       "    (15.0, 21.0, '$NUM-EINER1A:1d'),\n",
       "    (29.0, 38.0, 'RASIEREN2'),\n",
       "    (39.0, 44.0, 'ICH1'),\n",
       "    (58.0, 62.0, 'VOR1G'),\n",
       "    (69.0, 75.0, '$NUM-EINER1A:1d')),\n",
       "   ('ICH1',\n",
       "    '$NUM-EINER1A:1d',\n",
       "    'RASIEREN2',\n",
       "    'ICH1',\n",
       "    'VOR1G',\n",
       "    '$NUM-EINER1A:1d')),\n",
       "  ('1420216_B_both_208220_210800',\n",
       "   50.0,\n",
       "   128.0,\n",
       "   ((58.0, 62.0, 'VOR1G'),\n",
       "    (69.0, 75.0, '$NUM-EINER1A:1d'),\n",
       "    (83.0, 91.0, 'RASIEREN2'),\n",
       "    (96.0, 98.0, 'ICH1'),\n",
       "    (107.0, 114.0, 'KLEIN3'),\n",
       "    (121.0, 129.0, '$GEST-OFF')),\n",
       "   ('VOR1G', '$NUM-EINER1A:1d', 'RASIEREN2', 'ICH1', 'KLEIN3', '$GEST-OFF'))],\n",
       " '2': [('1184756_A_both_283520_285920',\n",
       "   0.0,\n",
       "   71.0,\n",
       "   ((0.0, 4.0, 'GUT1'), (27.0, 36.0, 'SCHÖN1A'), (41.0, 120.0, 'SEHR-VIEL1')),\n",
       "   ('GUT1', 'SCHÖN1A', 'SEHR-VIEL1')),\n",
       "  ('1184756_A_both_283520_285920',\n",
       "   50.0,\n",
       "   119.0,\n",
       "   ((41.0, 120.0, 'SEHR-VIEL1'),),\n",
       "   ('SEHR-VIEL1',))],\n",
       " '3': [('1419931_B_both_187880_189900',\n",
       "   1.0,\n",
       "   52.0,\n",
       "   ((0.0, 101.0, '$GEST-NM-KOPFNICKEN1'),),\n",
       "   ('$GEST-NM-KOPFNICKEN1',)),\n",
       "  ('1419931_B_both_187880_189900',\n",
       "   50.0,\n",
       "   100.0,\n",
       "   ((0.0, 101.0, '$GEST-NM-KOPFNICKEN1'),),\n",
       "   ('$GEST-NM-KOPFNICKEN1',))],\n",
       " '4': [('1184756_B_both_837440_897560',\n",
       "   1804.0,\n",
       "   2172.0,\n",
       "   ((1817.0, 1825.0, 'LASSEN1'),\n",
       "    (1835.0, 1846.0, 'BRAUCHEN-NICHT1'),\n",
       "    (1855.0, 1862.0, 'FERTIG1B'),\n",
       "    (1868.0, 1873.0, '$GEST'),\n",
       "    (1885.0, 1887.0, 'NUR2B'),\n",
       "    (1891.0, 1895.0, 'NOCH4B'),\n",
       "    (1909.0, 1914.0, '$NUM-EINER1A:2'),\n",
       "    (1918.0, 1927.0, 'KOMMEND2'),\n",
       "    (1939.0, 1946.0, 'FERTIG2'),\n",
       "    (1956.0, 1967.0, 'VORBEI4'),\n",
       "    (1981.0, 1994.0, 'DANN1A'),\n",
       "    (2013.0, 2035.0, 'WEITER1B'),\n",
       "    (2038.0, 2048.0, 'WEG-FORT1'),\n",
       "    (2056.0, 2064.0, '$NAME'),\n",
       "    (2070.0, 2083.0, '$GEST-ÜBERLEGEN1'),\n",
       "    (2119.0, 2123.0, 'MEIN1'),\n",
       "    (2129.0, 2135.0, 'SCHATZ1'),\n",
       "    (2136.0, 2142.0, 'TAG-VOR1A'),\n",
       "    (2149.0, 2162.0, 'GEBURTSTAG5')),\n",
       "   ('LASSEN1',\n",
       "    'BRAUCHEN-NICHT1',\n",
       "    'FERTIG1B',\n",
       "    '$GEST',\n",
       "    'NUR2B',\n",
       "    'NOCH4B',\n",
       "    '$NUM-EINER1A:2',\n",
       "    'KOMMEND2',\n",
       "    'FERTIG2',\n",
       "    'VORBEI4',\n",
       "    'DANN1A',\n",
       "    'WEITER1B',\n",
       "    'WEG-FORT1',\n",
       "    '$NAME',\n",
       "    '$GEST-ÜBERLEGEN1',\n",
       "    'MEIN1',\n",
       "    'SCHATZ1',\n",
       "    'TAG-VOR1A',\n",
       "    'GEBURTSTAG5')),\n",
       "  ('1184756_B_both_914080_935880',\n",
       "   300.0,\n",
       "   644.0,\n",
       "   ((312.0, 322.0, 'THEMA1'),\n",
       "    (333.0, 345.0, 'TIER1A'),\n",
       "    (354.0, 372.0, 'ANZIEHEN-OBERKÖRPER1A'),\n",
       "    (382.0, 392.0, 'SOLL1'),\n",
       "    (400.0, 405.0, 'ICH1'),\n",
       "    (417.0, 428.0, 'BESTELLEN1A'),\n",
       "    (442.0, 447.0, 'EIS2'),\n",
       "    (458.0, 470.0, 'BÄR2'),\n",
       "    (489.0, 497.0, '$PROD'),\n",
       "    (515.0, 525.0, 'FAN2'),\n",
       "    (548.0, 555.0, '$PROD'),\n",
       "    (566.0, 570.0, 'MIT1A'),\n",
       "    (590.0, 616.0, 'KLEIDUNG3'),\n",
       "    (625.0, 635.0, 'DAZU1')),\n",
       "   ('THEMA1',\n",
       "    'TIER1A',\n",
       "    'ANZIEHEN-OBERKÖRPER1A',\n",
       "    'SOLL1',\n",
       "    'ICH1',\n",
       "    'BESTELLEN1A',\n",
       "    'EIS2',\n",
       "    'BÄR2',\n",
       "    '$PROD',\n",
       "    'FAN2',\n",
       "    '$PROD',\n",
       "    'MIT1A',\n",
       "    'KLEIDUNG3',\n",
       "    'DAZU1'))],\n",
       " '5': [('1184756_B_both_155020_292700',\n",
       "   6514.0,\n",
       "   6683.0,\n",
       "   ((6530.0, 6541.0, 'BAUCHSCHMERZEN1'),\n",
       "    (6556.0, 6557.0, 'ICH1'),\n",
       "    (6564.0, 6571.0, 'SCHAUEN2'),\n",
       "    (6580.0, 6587.0, 'NICHT3B'),\n",
       "    (6587.0, 6594.0, 'SCHLECHT1'),\n",
       "    (6604.0, 6609.0, 'KULTUR1A'),\n",
       "    (6616.0, 6618.0, '$INDEX1'),\n",
       "    (6628.0, 6640.0, 'ANDERS6'),\n",
       "    (6651.0, 6653.0, 'ICH1'),\n",
       "    (6666.0, 6671.0, 'NICHT3A'),\n",
       "    (6680.0, 6686.0, 'SO1')),\n",
       "   ('BAUCHSCHMERZEN1',\n",
       "    'ICH1',\n",
       "    'SCHAUEN2',\n",
       "    'NICHT3B',\n",
       "    'SCHLECHT1',\n",
       "    'KULTUR1A',\n",
       "    '$INDEX1',\n",
       "    'ANDERS6',\n",
       "    'ICH1',\n",
       "    'NICHT3A',\n",
       "    'SO1')),\n",
       "  ('1184756_B_both_560_4080',\n",
       "   1.0,\n",
       "   165.0,\n",
       "   ((0.0, 2.0, 'THEMA1'),\n",
       "    (5.0, 10.0, 'ANFANG1A'),\n",
       "    (15.0, 19.0, '$GEST-OFF'),\n",
       "    (33.0, 44.0, 'VERNUNFT1'),\n",
       "    (55.0, 57.0, '$INDEX2'),\n",
       "    (81.0, 87.0, '$GEST-AUFMERKSAMKEIT1'),\n",
       "    (93.0, 101.0, 'URLAUB8B'),\n",
       "    (111.0, 115.0, 'DIESE2'),\n",
       "    (120.0, 130.0, 'JAHR2B'),\n",
       "    (139.0, 146.0, 'NEIN1A')),\n",
       "   ('THEMA1',\n",
       "    'ANFANG1A',\n",
       "    '$GEST-OFF',\n",
       "    'VERNUNFT1',\n",
       "    '$INDEX2',\n",
       "    '$GEST-AUFMERKSAMKEIT1',\n",
       "    'URLAUB8B',\n",
       "    'DIESE2',\n",
       "    'JAHR2B',\n",
       "    'NEIN1A'))],\n",
       " '6': [('1184756_B_both_155020_292700',\n",
       "   2672.0,\n",
       "   2942.0,\n",
       "   ((2685.0, 2703.0, 'ODER1'),\n",
       "    (2711.0, 2727.0, '$GEST-ÜBERLEGEN1'),\n",
       "    (2736.0, 2742.0, 'KUGEL1B'),\n",
       "    (2744.0, 2749.0, '$GEST-ÜBERLEGEN1'),\n",
       "    (2770.0, 2852.0, '$ALPHA1:Y-I-N-G-Y-A-N-G'),\n",
       "    (2864.0, 2879.0, '$PROD'),\n",
       "    (2891.0, 2900.0, 'KUGEL1B'),\n",
       "    (2919.0, 2928.0, 'RUND-KUGEL2')),\n",
       "   ('ODER1',\n",
       "    '$GEST-ÜBERLEGEN1',\n",
       "    'KUGEL1B',\n",
       "    '$GEST-ÜBERLEGEN1',\n",
       "    '$ALPHA1:Y-I-N-G-Y-A-N-G',\n",
       "    '$PROD',\n",
       "    'KUGEL1B',\n",
       "    'RUND-KUGEL2')),\n",
       "  ('1184756_B_both_155020_292700',\n",
       "   3448.0,\n",
       "   3716.0,\n",
       "   ((3454.0, 3465.0, 'BESETZT1A'),\n",
       "    (3475.0, 3479.0, 'SCHON1A'),\n",
       "    (3485.0, 3487.0, '$INDEX1'),\n",
       "    (3493.0, 3498.0, 'KANN2A'),\n",
       "    (3506.0, 3520.0, 'KOCHEN3A'),\n",
       "    (3528.0, 3536.0, 'BESSER1'),\n",
       "    (3548.0, 3562.0, 'SALAT3'),\n",
       "    (3575.0, 3582.0, 'BUFFET1'),\n",
       "    (3605.0, 3608.0, 'ANFÜHRUNGSZEICHEN1'),\n",
       "    (3616.0, 3622.0, 'KARTOFFEL3A'),\n",
       "    (3630.0, 3633.0, 'SALAT3'),\n",
       "    (3645.0, 3660.0, 'NEBEN1A'),\n",
       "    (3667.0, 3672.0, 'SALAT3'),\n",
       "    (3679.0, 3682.0, 'BEREICH1A'),\n",
       "    (3693.0, 3698.0, 'UND2A'),\n",
       "    (3709.0, 3720.0, 'FLEISCH1B')),\n",
       "   ('BESETZT1A',\n",
       "    'SCHON1A',\n",
       "    '$INDEX1',\n",
       "    'KANN2A',\n",
       "    'KOCHEN3A',\n",
       "    'BESSER1',\n",
       "    'SALAT3',\n",
       "    'BUFFET1',\n",
       "    'ANFÜHRUNGSZEICHEN1',\n",
       "    'KARTOFFEL3A',\n",
       "    'SALAT3',\n",
       "    'NEBEN1A',\n",
       "    'SALAT3',\n",
       "    'BEREICH1A',\n",
       "    'UND2A',\n",
       "    'FLEISCH1B'))],\n",
       " '7': [('1184756_B_both_155020_292700',\n",
       "   2525.0,\n",
       "   2735.0,\n",
       "   ((2536.0, 2573.0, '$PROD'),\n",
       "    (2578.0, 2584.0, 'MIT1A'),\n",
       "    (2606.0, 2617.0, '$PROD'),\n",
       "    (2633.0, 2644.0, 'VERSCHIEDEN1'),\n",
       "    (2664.0, 2674.0, 'KREUZ1B'),\n",
       "    (2685.0, 2703.0, 'ODER1'),\n",
       "    (2711.0, 2727.0, '$GEST-ÜBERLEGEN1')),\n",
       "   ('$PROD',\n",
       "    'MIT1A',\n",
       "    '$PROD',\n",
       "    'VERSCHIEDEN1',\n",
       "    'KREUZ1B',\n",
       "    'ODER1',\n",
       "    '$GEST-ÜBERLEGEN1')),\n",
       "  ('1184756_B_both_155020_292700',\n",
       "   2701.0,\n",
       "   2911.0,\n",
       "   ((2711.0, 2727.0, '$GEST-ÜBERLEGEN1'),\n",
       "    (2736.0, 2742.0, 'KUGEL1B'),\n",
       "    (2744.0, 2749.0, '$GEST-ÜBERLEGEN1'),\n",
       "    (2770.0, 2852.0, '$ALPHA1:Y-I-N-G-Y-A-N-G'),\n",
       "    (2864.0, 2879.0, '$PROD'),\n",
       "    (2891.0, 2900.0, 'KUGEL1B')),\n",
       "   ('$GEST-ÜBERLEGEN1',\n",
       "    'KUGEL1B',\n",
       "    '$GEST-ÜBERLEGEN1',\n",
       "    '$ALPHA1:Y-I-N-G-Y-A-N-G',\n",
       "    '$PROD',\n",
       "    'KUGEL1B'))],\n",
       " '8': [('1184756_B_both_155020_292700',\n",
       "   2703.0,\n",
       "   2958.0,\n",
       "   ((2711.0, 2727.0, '$GEST-ÜBERLEGEN1'),\n",
       "    (2736.0, 2742.0, 'KUGEL1B'),\n",
       "    (2744.0, 2749.0, '$GEST-ÜBERLEGEN1'),\n",
       "    (2770.0, 2852.0, '$ALPHA1:Y-I-N-G-Y-A-N-G'),\n",
       "    (2864.0, 2879.0, '$PROD'),\n",
       "    (2891.0, 2900.0, 'KUGEL1B'),\n",
       "    (2919.0, 2928.0, 'RUND-KUGEL2'),\n",
       "    (2946.0, 2973.0, 'KUGEL1B')),\n",
       "   ('$GEST-ÜBERLEGEN1',\n",
       "    'KUGEL1B',\n",
       "    '$GEST-ÜBERLEGEN1',\n",
       "    '$ALPHA1:Y-I-N-G-Y-A-N-G',\n",
       "    '$PROD',\n",
       "    'KUGEL1B',\n",
       "    'RUND-KUGEL2',\n",
       "    'KUGEL1B')),\n",
       "  ('1184756_B_both_155020_292700',\n",
       "   3231.0,\n",
       "   3507.0,\n",
       "   ((3241.0, 3252.0, 'ESSEN1'),\n",
       "    (3263.0, 3276.0, 'TOLL1A'),\n",
       "    (3287.0, 3289.0, '$NUM-EINER1A:4'),\n",
       "    (3298.0, 3300.0, 'MÄDCHEN4'),\n",
       "    (3308.0, 3322.0, '$LIST1:3of4'),\n",
       "    (3330.0, 3332.0, '$NUM-EINER1A:1d'),\n",
       "    (3337.0, 3340.0, '$LIST1:3of4'),\n",
       "    (3348.0, 3351.0, 'SEIN-IHR1'),\n",
       "    (3359.0, 3372.0, 'VATER1'),\n",
       "    (3379.0, 3390.0, 'RESTAURANT1'),\n",
       "    (3393.0, 3400.0, 'ANFÜHRUNGSZEICHEN2'),\n",
       "    (3404.0, 3420.0, 'BESETZT1A'),\n",
       "    (3426.0, 3429.0, '$INDEX1'),\n",
       "    (3439.0, 3448.0, 'RESTAURANT1'),\n",
       "    (3454.0, 3465.0, 'BESETZT1A'),\n",
       "    (3475.0, 3479.0, 'SCHON1A'),\n",
       "    (3485.0, 3487.0, '$INDEX1'),\n",
       "    (3493.0, 3498.0, 'KANN2A')),\n",
       "   ('ESSEN1',\n",
       "    'TOLL1A',\n",
       "    '$NUM-EINER1A:4',\n",
       "    'MÄDCHEN4',\n",
       "    '$LIST1:3of4',\n",
       "    '$NUM-EINER1A:1d',\n",
       "    '$LIST1:3of4',\n",
       "    'SEIN-IHR1',\n",
       "    'VATER1',\n",
       "    'RESTAURANT1',\n",
       "    'ANFÜHRUNGSZEICHEN2',\n",
       "    'BESETZT1A',\n",
       "    '$INDEX1',\n",
       "    'RESTAURANT1',\n",
       "    'BESETZT1A',\n",
       "    'SCHON1A',\n",
       "    '$INDEX1',\n",
       "    'KANN2A'))],\n",
       " '9': [('1184756_B_both_155020_292700',\n",
       "   2477.0,\n",
       "   2739.0,\n",
       "   ((2474.0, 2489.0, 'UNGEFÄHR1'),\n",
       "    (2510.0, 2513.0, '$PROD'),\n",
       "    (2536.0, 2573.0, '$PROD'),\n",
       "    (2578.0, 2584.0, 'MIT1A'),\n",
       "    (2606.0, 2617.0, '$PROD'),\n",
       "    (2633.0, 2644.0, 'VERSCHIEDEN1'),\n",
       "    (2664.0, 2674.0, 'KREUZ1B'),\n",
       "    (2685.0, 2703.0, 'ODER1'),\n",
       "    (2711.0, 2727.0, '$GEST-ÜBERLEGEN1'),\n",
       "    (2736.0, 2742.0, 'KUGEL1B')),\n",
       "   ('UNGEFÄHR1',\n",
       "    '$PROD',\n",
       "    '$PROD',\n",
       "    'MIT1A',\n",
       "    '$PROD',\n",
       "    'VERSCHIEDEN1',\n",
       "    'KREUZ1B',\n",
       "    'ODER1',\n",
       "    '$GEST-ÜBERLEGEN1',\n",
       "    'KUGEL1B')),\n",
       "  ('1184756_B_both_155020_292700',\n",
       "   3314.0,\n",
       "   3589.0,\n",
       "   ((3308.0, 3322.0, '$LIST1:3of4'),\n",
       "    (3330.0, 3332.0, '$NUM-EINER1A:1d'),\n",
       "    (3337.0, 3340.0, '$LIST1:3of4'),\n",
       "    (3348.0, 3351.0, 'SEIN-IHR1'),\n",
       "    (3359.0, 3372.0, 'VATER1'),\n",
       "    (3379.0, 3390.0, 'RESTAURANT1'),\n",
       "    (3393.0, 3400.0, 'ANFÜHRUNGSZEICHEN2'),\n",
       "    (3404.0, 3420.0, 'BESETZT1A'),\n",
       "    (3426.0, 3429.0, '$INDEX1'),\n",
       "    (3439.0, 3448.0, 'RESTAURANT1'),\n",
       "    (3454.0, 3465.0, 'BESETZT1A'),\n",
       "    (3475.0, 3479.0, 'SCHON1A'),\n",
       "    (3485.0, 3487.0, '$INDEX1'),\n",
       "    (3493.0, 3498.0, 'KANN2A'),\n",
       "    (3506.0, 3520.0, 'KOCHEN3A'),\n",
       "    (3528.0, 3536.0, 'BESSER1'),\n",
       "    (3548.0, 3562.0, 'SALAT3'),\n",
       "    (3575.0, 3582.0, 'BUFFET1')),\n",
       "   ('$LIST1:3of4',\n",
       "    '$NUM-EINER1A:1d',\n",
       "    '$LIST1:3of4',\n",
       "    'SEIN-IHR1',\n",
       "    'VATER1',\n",
       "    'RESTAURANT1',\n",
       "    'ANFÜHRUNGSZEICHEN2',\n",
       "    'BESETZT1A',\n",
       "    '$INDEX1',\n",
       "    'RESTAURANT1',\n",
       "    'BESETZT1A',\n",
       "    'SCHON1A',\n",
       "    '$INDEX1',\n",
       "    'KANN2A',\n",
       "    'KOCHEN3A',\n",
       "    'BESSER1',\n",
       "    'SALAT3',\n",
       "    'BUFFET1'))]}"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "disc.clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open(exp_path + 'clusters_tde.json','w') as f:\n",
    "    json.dump(disc.clusters, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### visualize clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "def find_commons(labelslist):\n",
    "    # labelslist: list of dicts\n",
    "    common_labels = set()\n",
    "    for i in range(len(labelslist)):\n",
    "        for j in range(i+1,len(labelslist)):\n",
    "            if '$' not in labelslist[i] and  '$' not in labelslist[j]:\n",
    "                common_labels |= (labelslist[i] & labelslist[j])\n",
    "    \n",
    "    common_labels = list(common_labels)\n",
    "    if len(common_labels) == 0:\n",
    "        return 'none'\n",
    "    else:\n",
    "        return str(common_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cluster_id</th>\n",
       "      <th>file</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>common_labels</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1420216_B_both_208220_210800</td>\n",
       "      <td>1.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>none</td>\n",
       "      <td>(ICH1, $, RASIEREN2, ICH1, VOR1G, $)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1420216_B_both_208220_210800</td>\n",
       "      <td>50.0</td>\n",
       "      <td>128.0</td>\n",
       "      <td>none</td>\n",
       "      <td>(VOR1G, $, RASIEREN2, ICH1, KLEIN3, $)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>1184756_A_both_283520_285920</td>\n",
       "      <td>0.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>none</td>\n",
       "      <td>(GUT1, SCHÖN1A)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1184756_A_both_283520_285920</td>\n",
       "      <td>50.0</td>\n",
       "      <td>119.0</td>\n",
       "      <td>none</td>\n",
       "      <td>(SEHR-VIEL1,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>1419931_B_both_187880_189900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>52.0</td>\n",
       "      <td>none</td>\n",
       "      <td>($,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>493</td>\n",
       "      <td>1184756_B_both_155020_292700</td>\n",
       "      <td>5365.0</td>\n",
       "      <td>5470.0</td>\n",
       "      <td>none</td>\n",
       "      <td>(GLAUBEN2A, AB2, GLAUBEN2A, $, LOS-START1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>494</td>\n",
       "      <td>1431690_B_both_113760_272000</td>\n",
       "      <td>2784.0</td>\n",
       "      <td>2886.0</td>\n",
       "      <td>none</td>\n",
       "      <td>(ICH2, SEHEN1, FÜHREN1, ICH1, SEHEN-AUF1, AUF-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>494</td>\n",
       "      <td>1431690_B_both_113760_272000</td>\n",
       "      <td>5314.0</td>\n",
       "      <td>5402.0</td>\n",
       "      <td>none</td>\n",
       "      <td>($, NICHT3B)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>495</td>\n",
       "      <td>1184756_B_both_155020_292700</td>\n",
       "      <td>3132.0</td>\n",
       "      <td>3235.0</td>\n",
       "      <td>none</td>\n",
       "      <td>($, SCHÖN1B, THEATER6, BEREICH1A, GUT1, $)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>495</td>\n",
       "      <td>1184756_B_both_155020_292700</td>\n",
       "      <td>5376.0</td>\n",
       "      <td>5457.0</td>\n",
       "      <td>none</td>\n",
       "      <td>(AB2, GLAUBEN2A, $)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>971 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   cluster_id                          file   start     end common_labels  \\\n",
       "0           1  1420216_B_both_208220_210800     1.0    80.0          none   \n",
       "1           1  1420216_B_both_208220_210800    50.0   128.0          none   \n",
       "0           2  1184756_A_both_283520_285920     0.0    71.0          none   \n",
       "1           2  1184756_A_both_283520_285920    50.0   119.0          none   \n",
       "0           3  1419931_B_both_187880_189900     1.0    52.0          none   \n",
       "..        ...                           ...     ...     ...           ...   \n",
       "1         493  1184756_B_both_155020_292700  5365.0  5470.0          none   \n",
       "0         494  1431690_B_both_113760_272000  2784.0  2886.0          none   \n",
       "1         494  1431690_B_both_113760_272000  5314.0  5402.0          none   \n",
       "0         495  1184756_B_both_155020_292700  3132.0  3235.0          none   \n",
       "1         495  1184756_B_both_155020_292700  5376.0  5457.0          none   \n",
       "\n",
       "                                               labels  \n",
       "0                (ICH1, $, RASIEREN2, ICH1, VOR1G, $)  \n",
       "1              (VOR1G, $, RASIEREN2, ICH1, KLEIN3, $)  \n",
       "0                                     (GUT1, SCHÖN1A)  \n",
       "1                                       (SEHR-VIEL1,)  \n",
       "0                                                ($,)  \n",
       "..                                                ...  \n",
       "1          (GLAUBEN2A, AB2, GLAUBEN2A, $, LOS-START1)  \n",
       "0   (ICH2, SEHEN1, FÜHREN1, ICH1, SEHEN-AUF1, AUF-...  \n",
       "1                                        ($, NICHT3B)  \n",
       "0          ($, SCHÖN1B, THEATER6, BEREICH1A, GUT1, $)  \n",
       "1                                 (AB2, GLAUBEN2A, $)  \n",
       "\n",
       "[971 rows x 6 columns]"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clusters = disc.clusters\n",
    "\n",
    "all_clus = []\n",
    "for cid, clus in clusters.items():\n",
    "    clus_within = []\n",
    "    for fname, start, end, tokens, ngram in clus:\n",
    "        clus_within.append((fname,start,end,ngram))\n",
    "    tmp = pd.DataFrame(clus_within, columns=['file','start','end','labels'])\n",
    "    common_labels = find_commons(list(tmp.labels.apply(lambda x: set(x))))\n",
    "    tmp['common_labels'] = common_labels\n",
    "    tmp['cluster_id'] = cid\n",
    "    tmp = tmp[['cluster_id', 'file','start','end', 'common_labels', 'labels']]\n",
    "    all_clus.append(tmp)\n",
    "pd.concat(all_clus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cluster_id</th>\n",
       "      <th>file</th>\n",
       "      <th>start</th>\n",
       "      <th>end</th>\n",
       "      <th>common_labels</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>75</td>\n",
       "      <td>1177918_B_both_32720_40900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>['SAGEN1']</td>\n",
       "      <td>(NEIN1A, MOMENT2, SAGEN1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>75</td>\n",
       "      <td>1177918_B_both_32720_40900</td>\n",
       "      <td>56.0</td>\n",
       "      <td>139.0</td>\n",
       "      <td>['SAGEN1']</td>\n",
       "      <td>(SAGEN1, AUCH3A, DAZU1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>121</td>\n",
       "      <td>1177918_A_both_112820_172780</td>\n",
       "      <td>1.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>['JA2']</td>\n",
       "      <td>(JA2,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>121</td>\n",
       "      <td>1177918_A_both_112820_172780</td>\n",
       "      <td>56.0</td>\n",
       "      <td>132.0</td>\n",
       "      <td>['JA2']</td>\n",
       "      <td>(JA2,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>158</td>\n",
       "      <td>1184756_B_both_691540_694720</td>\n",
       "      <td>31.0</td>\n",
       "      <td>110.0</td>\n",
       "      <td>['SCHWEIZ1A']</td>\n",
       "      <td>(SCHWEIZ1A,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>158</td>\n",
       "      <td>1184756_B_both_691540_694720</td>\n",
       "      <td>80.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>['SCHWEIZ1A']</td>\n",
       "      <td>(SCHWEIZ1A,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>238</td>\n",
       "      <td>1431690_B_both_113760_272000</td>\n",
       "      <td>2787.0</td>\n",
       "      <td>2891.0</td>\n",
       "      <td>['SEHEN-AUF1']</td>\n",
       "      <td>(SEHEN1, FÜHREN1, ICH1, SEHEN-AUF1, AUF-PERSON1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>238</td>\n",
       "      <td>1431690_B_both_113760_272000</td>\n",
       "      <td>3224.0</td>\n",
       "      <td>3346.0</td>\n",
       "      <td>['SEHEN-AUF1']</td>\n",
       "      <td>(SEHEN-AUF2, SEHR6, ARMER1, ICH2, SEHEN-AUF1, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>253</td>\n",
       "      <td>1431690_B_both_113760_272000</td>\n",
       "      <td>2784.0</td>\n",
       "      <td>2885.0</td>\n",
       "      <td>['ICH1', 'SEHEN-AUF1']</td>\n",
       "      <td>(ICH2, SEHEN1, FÜHREN1, ICH1, SEHEN-AUF1, AUF-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>253</td>\n",
       "      <td>1431690_B_both_113760_272000</td>\n",
       "      <td>2917.0</td>\n",
       "      <td>3013.0</td>\n",
       "      <td>['ICH1', 'SEHEN-AUF1']</td>\n",
       "      <td>(ICH1, SEHEN-AUF2, ARMER1, SEHEN-AUF1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>292</td>\n",
       "      <td>1431690_B_both_113760_272000</td>\n",
       "      <td>2923.0</td>\n",
       "      <td>3022.0</td>\n",
       "      <td>['ARMER1', 'SEHEN-AUF1']</td>\n",
       "      <td>(ICH1, SEHEN-AUF2, ARMER1, SEHEN-AUF1, ICH1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>292</td>\n",
       "      <td>1431690_B_both_113760_272000</td>\n",
       "      <td>3233.0</td>\n",
       "      <td>3342.0</td>\n",
       "      <td>['ARMER1', 'SEHEN-AUF1']</td>\n",
       "      <td>(SEHR6, ARMER1, ICH2, SEHEN-AUF1, ICH2, TAUB-G...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>302</td>\n",
       "      <td>1178133_A_both_800_42600</td>\n",
       "      <td>617.0</td>\n",
       "      <td>690.0</td>\n",
       "      <td>['LANG-ZEIT4A']</td>\n",
       "      <td>(LANG-ZEIT4A,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>302</td>\n",
       "      <td>1178133_A_both_800_42600</td>\n",
       "      <td>664.0</td>\n",
       "      <td>736.0</td>\n",
       "      <td>['LANG-ZEIT4A']</td>\n",
       "      <td>(LANG-ZEIT4A,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>350</td>\n",
       "      <td>1432043_A_both_640_17800</td>\n",
       "      <td>741.0</td>\n",
       "      <td>822.0</td>\n",
       "      <td>['JA2']</td>\n",
       "      <td>(JA2, LETZTE1A)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>350</td>\n",
       "      <td>1432043_A_both_1291620_1298960</td>\n",
       "      <td>14.0</td>\n",
       "      <td>117.0</td>\n",
       "      <td>['JA2']</td>\n",
       "      <td>(JA2,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>356</td>\n",
       "      <td>1177918_B_both_32720_40900</td>\n",
       "      <td>196.0</td>\n",
       "      <td>279.0</td>\n",
       "      <td>['HAKEN1']</td>\n",
       "      <td>(WOLLEN-NICHT2, HAKEN1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>356</td>\n",
       "      <td>1177918_B_both_32720_40900</td>\n",
       "      <td>252.0</td>\n",
       "      <td>334.0</td>\n",
       "      <td>['HAKEN1']</td>\n",
       "      <td>(HAKEN1,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>375</td>\n",
       "      <td>1177918_B_both_266840_280140</td>\n",
       "      <td>202.0</td>\n",
       "      <td>287.0</td>\n",
       "      <td>['ICH1']</td>\n",
       "      <td>(FUSSBALL2, ICH1, SCHAUEN1, ICH1, NOCH4A)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>375</td>\n",
       "      <td>1177918_B_both_266840_280140</td>\n",
       "      <td>596.0</td>\n",
       "      <td>664.0</td>\n",
       "      <td>['ICH1']</td>\n",
       "      <td>(EINIGE1, ICH1, NIEMALS2B, ICH1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>411</td>\n",
       "      <td>1420216_B_both_76300_99480</td>\n",
       "      <td>600.0</td>\n",
       "      <td>686.0</td>\n",
       "      <td>['ICH1']</td>\n",
       "      <td>(ICH1, SCHREIBEN1E)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>411</td>\n",
       "      <td>1420216_B_both_313000_346940</td>\n",
       "      <td>1591.0</td>\n",
       "      <td>1688.0</td>\n",
       "      <td>['ICH1']</td>\n",
       "      <td>(ICH1,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>424</td>\n",
       "      <td>1178133_A_both_112880_133640</td>\n",
       "      <td>68.0</td>\n",
       "      <td>146.0</td>\n",
       "      <td>['PAPST1']</td>\n",
       "      <td>(PAPST1,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>424</td>\n",
       "      <td>1178133_A_both_800_42600</td>\n",
       "      <td>41.0</td>\n",
       "      <td>143.0</td>\n",
       "      <td>['PAPST1']</td>\n",
       "      <td>(PAPST1,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>446</td>\n",
       "      <td>1244581_A_both_198900_220380</td>\n",
       "      <td>923.0</td>\n",
       "      <td>1028.0</td>\n",
       "      <td>['ERFAHREN1A']</td>\n",
       "      <td>(ERFAHREN1A,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>446</td>\n",
       "      <td>1244581_A_both_320_176080</td>\n",
       "      <td>8139.0</td>\n",
       "      <td>8224.0</td>\n",
       "      <td>['ERFAHREN1A']</td>\n",
       "      <td>(ERFAHREN1A, WENIG2)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>449</td>\n",
       "      <td>1420216_B_both_313000_346940</td>\n",
       "      <td>1592.0</td>\n",
       "      <td>1690.0</td>\n",
       "      <td>['ICH1']</td>\n",
       "      <td>(ICH1,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>449</td>\n",
       "      <td>1420216_B_both_396920_419680</td>\n",
       "      <td>970.0</td>\n",
       "      <td>1060.0</td>\n",
       "      <td>['ICH1']</td>\n",
       "      <td>(SEHR-VIEL1, ICH1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>466</td>\n",
       "      <td>1431690_A_both_480_113460</td>\n",
       "      <td>4751.0</td>\n",
       "      <td>4842.0</td>\n",
       "      <td>['ICH2', 'ICH1']</td>\n",
       "      <td>(ICH1, SEHEN-AUF1, ICH2, KANN1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>466</td>\n",
       "      <td>1431690_A_both_480_113460</td>\n",
       "      <td>5224.0</td>\n",
       "      <td>5314.0</td>\n",
       "      <td>['ICH2', 'ICH1']</td>\n",
       "      <td>(ICH2, SEHEN-AUF3, ICH1)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>475</td>\n",
       "      <td>1432043_B_both_1281600_1305880</td>\n",
       "      <td>45.0</td>\n",
       "      <td>125.0</td>\n",
       "      <td>['GRENZE1B']</td>\n",
       "      <td>(GRENZE1B,)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>475</td>\n",
       "      <td>1432043_B_both_1281600_1305880</td>\n",
       "      <td>94.0</td>\n",
       "      <td>174.0</td>\n",
       "      <td>['GRENZE1B']</td>\n",
       "      <td>(GRENZE1B,)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  cluster_id                            file   start     end  \\\n",
       "0         75      1177918_B_both_32720_40900     1.0    84.0   \n",
       "1         75      1177918_B_both_32720_40900    56.0   139.0   \n",
       "0        121    1177918_A_both_112820_172780     1.0    77.0   \n",
       "1        121    1177918_A_both_112820_172780    56.0   132.0   \n",
       "0        158    1184756_B_both_691540_694720    31.0   110.0   \n",
       "1        158    1184756_B_both_691540_694720    80.0   158.0   \n",
       "0        238    1431690_B_both_113760_272000  2787.0  2891.0   \n",
       "1        238    1431690_B_both_113760_272000  3224.0  3346.0   \n",
       "0        253    1431690_B_both_113760_272000  2784.0  2885.0   \n",
       "1        253    1431690_B_both_113760_272000  2917.0  3013.0   \n",
       "0        292    1431690_B_both_113760_272000  2923.0  3022.0   \n",
       "1        292    1431690_B_both_113760_272000  3233.0  3342.0   \n",
       "0        302        1178133_A_both_800_42600   617.0   690.0   \n",
       "1        302        1178133_A_both_800_42600   664.0   736.0   \n",
       "0        350        1432043_A_both_640_17800   741.0   822.0   \n",
       "1        350  1432043_A_both_1291620_1298960    14.0   117.0   \n",
       "0        356      1177918_B_both_32720_40900   196.0   279.0   \n",
       "1        356      1177918_B_both_32720_40900   252.0   334.0   \n",
       "0        375    1177918_B_both_266840_280140   202.0   287.0   \n",
       "1        375    1177918_B_both_266840_280140   596.0   664.0   \n",
       "0        411      1420216_B_both_76300_99480   600.0   686.0   \n",
       "1        411    1420216_B_both_313000_346940  1591.0  1688.0   \n",
       "0        424    1178133_A_both_112880_133640    68.0   146.0   \n",
       "1        424        1178133_A_both_800_42600    41.0   143.0   \n",
       "0        446    1244581_A_both_198900_220380   923.0  1028.0   \n",
       "1        446       1244581_A_both_320_176080  8139.0  8224.0   \n",
       "0        449    1420216_B_both_313000_346940  1592.0  1690.0   \n",
       "1        449    1420216_B_both_396920_419680   970.0  1060.0   \n",
       "0        466       1431690_A_both_480_113460  4751.0  4842.0   \n",
       "1        466       1431690_A_both_480_113460  5224.0  5314.0   \n",
       "0        475  1432043_B_both_1281600_1305880    45.0   125.0   \n",
       "1        475  1432043_B_both_1281600_1305880    94.0   174.0   \n",
       "\n",
       "              common_labels                                             labels  \n",
       "0                ['SAGEN1']                          (NEIN1A, MOMENT2, SAGEN1)  \n",
       "1                ['SAGEN1']                            (SAGEN1, AUCH3A, DAZU1)  \n",
       "0                   ['JA2']                                             (JA2,)  \n",
       "1                   ['JA2']                                             (JA2,)  \n",
       "0             ['SCHWEIZ1A']                                       (SCHWEIZ1A,)  \n",
       "1             ['SCHWEIZ1A']                                       (SCHWEIZ1A,)  \n",
       "0            ['SEHEN-AUF1']   (SEHEN1, FÜHREN1, ICH1, SEHEN-AUF1, AUF-PERSON1)  \n",
       "1            ['SEHEN-AUF1']  (SEHEN-AUF2, SEHR6, ARMER1, ICH2, SEHEN-AUF1, ...  \n",
       "0    ['ICH1', 'SEHEN-AUF1']  (ICH2, SEHEN1, FÜHREN1, ICH1, SEHEN-AUF1, AUF-...  \n",
       "1    ['ICH1', 'SEHEN-AUF1']             (ICH1, SEHEN-AUF2, ARMER1, SEHEN-AUF1)  \n",
       "0  ['ARMER1', 'SEHEN-AUF1']       (ICH1, SEHEN-AUF2, ARMER1, SEHEN-AUF1, ICH1)  \n",
       "1  ['ARMER1', 'SEHEN-AUF1']  (SEHR6, ARMER1, ICH2, SEHEN-AUF1, ICH2, TAUB-G...  \n",
       "0           ['LANG-ZEIT4A']                                     (LANG-ZEIT4A,)  \n",
       "1           ['LANG-ZEIT4A']                                     (LANG-ZEIT4A,)  \n",
       "0                   ['JA2']                                    (JA2, LETZTE1A)  \n",
       "1                   ['JA2']                                             (JA2,)  \n",
       "0                ['HAKEN1']                            (WOLLEN-NICHT2, HAKEN1)  \n",
       "1                ['HAKEN1']                                          (HAKEN1,)  \n",
       "0                  ['ICH1']          (FUSSBALL2, ICH1, SCHAUEN1, ICH1, NOCH4A)  \n",
       "1                  ['ICH1']                   (EINIGE1, ICH1, NIEMALS2B, ICH1)  \n",
       "0                  ['ICH1']                                (ICH1, SCHREIBEN1E)  \n",
       "1                  ['ICH1']                                            (ICH1,)  \n",
       "0                ['PAPST1']                                          (PAPST1,)  \n",
       "1                ['PAPST1']                                          (PAPST1,)  \n",
       "0            ['ERFAHREN1A']                                      (ERFAHREN1A,)  \n",
       "1            ['ERFAHREN1A']                               (ERFAHREN1A, WENIG2)  \n",
       "0                  ['ICH1']                                            (ICH1,)  \n",
       "1                  ['ICH1']                                 (SEHR-VIEL1, ICH1)  \n",
       "0          ['ICH2', 'ICH1']                    (ICH1, SEHEN-AUF1, ICH2, KANN1)  \n",
       "1          ['ICH2', 'ICH1']                           (ICH2, SEHEN-AUF3, ICH1)  \n",
       "0              ['GRENZE1B']                                        (GRENZE1B,)  \n",
       "1              ['GRENZE1B']                                        (GRENZE1B,)  "
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alldf = pd.concat(all_clus)\n",
    "alldf.loc[alldf.common_labels != 'none']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"['SEHR-VIEL1']\""
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(common_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### write read config file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'overlap_th': 5.0,\n",
       " 'excluded_units': ['SIL', '__ON__', '__OFF__', '__EMOTION__', 'SPN'],\n",
       " 'discoverable_th': 1}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf = {'overlap_th': 5. ,\n",
    "'excluded_units' : ['SIL','__ON__','__OFF__','__EMOTION__','SPN'] ,\n",
    "'discoverable_th' : 1}\n",
    "conf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "config_pth = 'config.json'\n",
    "\n",
    "with open(config_pth, 'w') as f:\n",
    "    json.dump(conf, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(config_pth, 'r') as f:\n",
    "    conf = json.load(f)\n",
    "    \n",
    "conf['overlap_th']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import editdistance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s1 = []\n",
    "s2 = ['s']\n",
    "editdistance.eval(s1, s2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "TDE",
   "language": "python",
   "name": "tde"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}