Tags Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T11:10:10.140387Z",
     "start_time": "2019-04-22T11:10:10.132396Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_tag_files(tag_path, suffix='csv'):\n",
    "    \n",
    "    tag_path = Path(tag_path)\n",
    "    all_files = [str(tag_file) for tag_file in tag_path.rglob('*.' + suffix)]\n",
    "    \n",
    "    return all_files\n",
    "\n",
    "tag_files = get_tag_files(tag_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T11:13:24.990562Z",
     "start_time": "2019-04-22T11:13:24.974824Z"
    },
    "collapsed": true,
    "editable": true,
    "run_control": {
     "marked": true
    }
   },
   "outputs": [],
   "source": [
    "def create_csv_paths(data_types, base='', exp='', full=True):\n",
    "    \n",
    "    csv_paths = {}\n",
    "    \n",
    "    for data_type in data_types:\n",
    "\n",
    "        if not full:\n",
    "            csv_dir = '../data/2016_2019/samples/'\n",
    "            e = '_' + str(base) + 'e' + str(exp)\n",
    "            csv_path = csv_dir + data_type + e + '.csv'\n",
    "        else:\n",
    "            csv_dir = '../data/2016_2019/full/'\n",
    "            e = ''\n",
    "            csv_path = csv_dir + data_type + e + '.csv'\n",
    "            \n",
    "        csv_paths[data_type] = csv_path\n",
    "    \n",
    "    return csv_paths\n",
    "\n",
    "data_types = ['answers', 'comments', 'notes', 'questions', 'users', 'wikis', 'tags']\n",
    "csv_paths = create_csv_paths(data_types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T15:09:37.453141Z",
     "start_time": "2019-04-22T15:09:37.417408Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# DATA\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "# SYSTEM\n",
    "import os, sys\n",
    "from pathlib2 import Path\n",
    "\n",
    "# MISC \n",
    "import re\n",
    "import csv\n",
    "from itertools import izip\n",
    "\n",
    "# PLOTTING\n",
    "%matplotlib inline\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T11:15:18.585265Z",
     "start_time": "2019-04-22T11:15:18.569994Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "data_type = 'users'\n",
    "\n",
    "sample = False\n",
    "\n",
    "m = 25\n",
    "exp = 2\n",
    "\n",
    "if sample:\n",
    "    csv_dir = '../data/2016_2019/samples/'\n",
    "    e = '_' + str(m) + 'e' + str(exp)\n",
    "    csv_path = csv_dir + data_type + e + '.csv'\n",
    "else:\n",
    "    csv_dir = '../data/2016_2019/full/'\n",
    "    e = ''\n",
    "    csv_path = csv_dir + data_type + e + '.csv'\n",
    "\n",
    "data_types = ['answers', 'comments', 'notes', 'questions', 'users', 'wikis', 'paths', 'tags', 'node_tags', 'terms']\n",
    "csv_paths = create_csv_paths(data_types)\n",
    "\n",
    "tag_dir =  '../data/2016_2019/full/tags'\n",
    "tag_files = get_tag_files(tag_dir)\n",
    "\n",
    "# print csv_paths"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LOAD TAG USAGE AND TERM DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T11:11:11.890956Z",
     "start_time": "2019-04-22T11:11:11.817565Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>tid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5389</th>\n",
       "      <td>523</td>\n",
       "      <td>1</td>\n",
       "      <td>14336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>584</th>\n",
       "      <td>13702</td>\n",
       "      <td>420056</td>\n",
       "      <td>1009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1825</th>\n",
       "      <td>12966</td>\n",
       "      <td>468990</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6313</th>\n",
       "      <td>16853</td>\n",
       "      <td>545097</td>\n",
       "      <td>17997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10522</th>\n",
       "      <td>18842</td>\n",
       "      <td>580718</td>\n",
       "      <td>19940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20525</th>\n",
       "      <td>11802</td>\n",
       "      <td>319070</td>\n",
       "      <td>11684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8546</th>\n",
       "      <td>12916</td>\n",
       "      <td>461120</td>\n",
       "      <td>1697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3655</th>\n",
       "      <td>19056</td>\n",
       "      <td>584260</td>\n",
       "      <td>13971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17235</th>\n",
       "      <td>12331</td>\n",
       "      <td>46902</td>\n",
       "      <td>8709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6255</th>\n",
       "      <td>12204</td>\n",
       "      <td>1</td>\n",
       "      <td>15113</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         nid     uid    tid\n",
       "5389     523       1  14336\n",
       "584    13702  420056   1009\n",
       "1825   12966  468990     14\n",
       "6313   16853  545097  17997\n",
       "10522  18842  580718  19940\n",
       "20525  11802  319070  11684\n",
       "8546   12916  461120   1697\n",
       "3655   19056  584260  13971\n",
       "17235  12331   46902   8709\n",
       "6255   12204       1  15113"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LOAD CSV'S INTO DATAFRAME DROP DATE (DATAPOINT IS NO LONGER AVAILABLE COULD BE MERGED IN A FUTURE WITH MORE TIME)\n",
    "use_cols = ['nid', 'uid', 'tid']\n",
    "tag_dfs = [pd.read_csv(tag_file, usecols=use_cols) for tag_file in tag_files]\n",
    "\n",
    "# REORDER TAG DF COLUMNS IN PREP WITH TERM DATA MERGE\n",
    "tag_dfs = [tag_df[use_cols] for tag_df in tag_dfs]\n",
    "\n",
    "# CONCAT THE FRAMES INTO ONE BIG ONE\n",
    "tag_usage = pd.concat(tag_dfs)\n",
    "\n",
    "# CHECK OUT THE RESULTS!\n",
    "tag_usage.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T20:48:48.904473Z",
     "start_time": "2019-04-22T20:48:48.896508Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# tag_usage.loc[tag_usage['uid'] == 461120]\n",
    "tag_usage = tag_usage.loc[tag_usage['uid'] != 461120]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T11:17:03.582640Z",
     "start_time": "2019-04-22T11:17:03.543616Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tid</th>\n",
       "      <th>name</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7121</th>\n",
       "      <td>14388</td>\n",
       "      <td>enterprise</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7165</th>\n",
       "      <td>14432</td>\n",
       "      <td>writer</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7183</th>\n",
       "      <td>14450</td>\n",
       "      <td>redirect:13450</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5404</th>\n",
       "      <td>12671</td>\n",
       "      <td>environmentaljustice</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8771</th>\n",
       "      <td>16039</td>\n",
       "      <td>raspberry-pi-spectrometer</td>\n",
       "      <td>17.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2699</th>\n",
       "      <td>7026</td>\n",
       "      <td>monitoring</td>\n",
       "      <td>28.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6374</th>\n",
       "      <td>13641</td>\n",
       "      <td>date:12-28-2016</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3691</th>\n",
       "      <td>10074</td>\n",
       "      <td>lon:-74.14529577537604</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12219</th>\n",
       "      <td>19487</td>\n",
       "      <td>main</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10770</th>\n",
       "      <td>18038</td>\n",
       "      <td>lead-testing</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         tid                       name  count\n",
       "7121   14388                 enterprise    4.0\n",
       "7165   14432                     writer    2.0\n",
       "7183   14450             redirect:13450    1.0\n",
       "5404   12671       environmentaljustice    8.0\n",
       "8771   16039  raspberry-pi-spectrometer   17.0\n",
       "2699    7026                 monitoring   28.0\n",
       "6374   13641            date:12-28-2016    1.0\n",
       "3691   10074     lon:-74.14529577537604    1.0\n",
       "12219  19487                       main    1.0\n",
       "10770  18038               lead-testing   10.0"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LOAD TERM DATA\n",
    "use_cols = ['tid', 'name', 'count']\n",
    "terms = pd.read_csv(csv_paths['terms'], usecols=use_cols)\n",
    "terms.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PARSE TERM DATA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T11:22:53.658301Z",
     "start_time": "2019-04-22T11:22:53.602913Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tid</th>\n",
       "      <th>name</th>\n",
       "      <th>count</th>\n",
       "      <th>power</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>754</th>\n",
       "      <td>2349</td>\n",
       "      <td>events:boston</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>755</th>\n",
       "      <td>2350</td>\n",
       "      <td>list:plots-boston</td>\n",
       "      <td>3.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>756</th>\n",
       "      <td>2360</td>\n",
       "      <td>events:new-york-city</td>\n",
       "      <td>12.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>757</th>\n",
       "      <td>2361</td>\n",
       "      <td>list:plots-nyc</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>763</th>\n",
       "      <td>2369</td>\n",
       "      <td>list:plots-gulfcoast</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      tid                  name  count power\n",
       "754  2349         events:boston    5.0  True\n",
       "755  2350     list:plots-boston    3.0  True\n",
       "756  2360  events:new-york-city   12.0  True\n",
       "757  2361        list:plots-nyc    1.0  True\n",
       "763  2369  list:plots-gulfcoast    1.0  True"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MARK POWER TAGS\n",
    "terms['power'] = terms['name'].str.contains(':')\n",
    "terms.loc[terms['power'] == True].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T13:49:48.384626Z",
     "start_time": "2019-04-22T13:49:48.370728Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROWS WITH NAN'S: \n",
      "\n",
      "TID: 0 \n",
      "NAME: 2\n",
      "COUNT: 214 \n",
      "POWER: 2\n"
     ]
    }
   ],
   "source": [
    "# https://stackoverflow.com/a/41320210/6023530\n",
    "print 'ROWS WITH NAN\\'S:', '\\n'\n",
    "print 'TID:', terms['tid'].isnull().sum(), '\\n', 'NAME:', terms['name'].isnull().sum()\n",
    "print 'COUNT:', terms['count'].isnull().sum(), '\\n', 'POWER:', terms['power'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T13:52:00.731215Z",
     "start_time": "2019-04-22T13:52:00.718045Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# REMOVE ROWS W/O A NAME B/C THEN NO TAG TO APPLY FUNCTION TO\n",
    "terms.dropna(subset=['name', 'power'], inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T13:28:26.962953Z",
     "start_time": "2019-04-22T13:28:26.908341Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# https://www.geeksforgeeks.org/python-pandas-split-strings-into-two-list-columns-using-str-split/\n",
    "# https://stackoverflow.com/a/35373340/6023530\n",
    "\n",
    "def split_power_tags(row):\n",
    "    \n",
    "    split_row = row['name'].split(':')#, n=1, expand=True)\n",
    "    \n",
    "    if len(split_row) == 2: \n",
    "        row['name'] = split_row[1]\n",
    "        row['power_type'] = split_row[0]\n",
    "    else:\n",
    "        row['power_type'] = 'npt'\n",
    "    \n",
    "    return row['name'], row['power_type']\n",
    "\n",
    "power_test = terms.sample(20).copy()\n",
    "power_test['name'], power_test['power_type'] = izip(*power_test.apply(split_power_tags, axis=1))\n",
    "power_test = power_test[['tid', 'name', 'power_type', 'count', 'power']]\n",
    "power_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T13:28:28.586782Z",
     "start_time": "2019-04-22T13:28:28.564644Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tid</th>\n",
       "      <th>name</th>\n",
       "      <th>count</th>\n",
       "      <th>power</th>\n",
       "      <th>power_type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3769</th>\n",
       "      <td>10152</td>\n",
       "      <td>130.8757748325563</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>lon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12409</th>\n",
       "      <td>19677</td>\n",
       "      <td>8.496973</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "      <td>lon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11964</th>\n",
       "      <td>19232</td>\n",
       "      <td>image</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "      <td>npt</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8254</th>\n",
       "      <td>15522</td>\n",
       "      <td>software-outreach</td>\n",
       "      <td>2.0</td>\n",
       "      <td>True</td>\n",
       "      <td>activity</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5612</th>\n",
       "      <td>12879</td>\n",
       "      <td>barnraising-2015</td>\n",
       "      <td>20.0</td>\n",
       "      <td>False</td>\n",
       "      <td>npt</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         tid               name  count  power power_type\n",
       "3769   10152  130.8757748325563    1.0   True        lon\n",
       "12409  19677           8.496973    1.0   True        lon\n",
       "11964  19232              image    1.0  False        npt\n",
       "8254   15522  software-outreach    2.0   True   activity\n",
       "5612   12879   barnraising-2015   20.0  False        npt"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "power_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T13:39:54.310065Z",
     "start_time": "2019-04-22T13:39:54.280893Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tid</th>\n",
       "      <th>name</th>\n",
       "      <th>count</th>\n",
       "      <th>power</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13047</th>\n",
       "      <td>20315</td>\n",
       "      <td>laga</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13048</th>\n",
       "      <td>20316</td>\n",
       "      <td>redirect:pipeline-webinar</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13049</th>\n",
       "      <td>20317</td>\n",
       "      <td>psn</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13050</th>\n",
       "      <td>20318</td>\n",
       "      <td>activity:thermal-flashlight-assembly</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13051</th>\n",
       "      <td>20319</td>\n",
       "      <td>pirateship</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         tid                                  name  count  power\n",
       "13047  20315                                  laga    1.0  False\n",
       "13048  20316             redirect:pipeline-webinar    1.0   True\n",
       "13049  20317                                   psn    1.0  False\n",
       "13050  20318  activity:thermal-flashlight-assembly    5.0   True\n",
       "13051  20319                            pirateship    1.0  False"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "terms.iloc[-20:-15] # GOOD BUT REPEATABLE MIX OF POWER/NON-POWER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T13:53:33.411307Z",
     "start_time": "2019-04-22T13:53:12.403354Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "terms['name'], terms['power_type'] = izip(*terms.apply(split_power_tags, axis=1))\n",
    "terms = terms[['tid', 'name', 'power_type', 'count', 'power']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T13:53:44.836685Z",
     "start_time": "2019-04-22T13:53:44.810296Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tid</th>\n",
       "      <th>name</th>\n",
       "      <th>power_type</th>\n",
       "      <th>count</th>\n",
       "      <th>power</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13047</th>\n",
       "      <td>20315</td>\n",
       "      <td>laga</td>\n",
       "      <td>npt</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13048</th>\n",
       "      <td>20316</td>\n",
       "      <td>pipeline-webinar</td>\n",
       "      <td>redirect</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13049</th>\n",
       "      <td>20317</td>\n",
       "      <td>psn</td>\n",
       "      <td>npt</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13050</th>\n",
       "      <td>20318</td>\n",
       "      <td>thermal-flashlight-assembly</td>\n",
       "      <td>activity</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13051</th>\n",
       "      <td>20319</td>\n",
       "      <td>pirateship</td>\n",
       "      <td>npt</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         tid                         name power_type  count  power\n",
       "13047  20315                         laga        npt    1.0  False\n",
       "13048  20316             pipeline-webinar   redirect    1.0   True\n",
       "13049  20317                          psn        npt    1.0  False\n",
       "13050  20318  thermal-flashlight-assembly   activity    5.0   True\n",
       "13051  20319                   pirateship        npt    1.0  False"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# O. M. G. It wooorked!\n",
    "terms.iloc[-20:-15]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T13:49:14.010447Z",
     "start_time": "2019-04-22T13:49:13.995660Z"
    }
   },
   "source": [
    "## MERGE WITH TAG USAGE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T13:59:09.922760Z",
     "start_time": "2019-04-22T13:59:09.904937Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "54004\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>tid</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5073</th>\n",
       "      <td>14247</td>\n",
       "      <td>508251</td>\n",
       "      <td>15019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5074</th>\n",
       "      <td>14487</td>\n",
       "      <td>508247</td>\n",
       "      <td>15020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5075</th>\n",
       "      <td>14496</td>\n",
       "      <td>441262</td>\n",
       "      <td>15022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5076</th>\n",
       "      <td>14496</td>\n",
       "      <td>441262</td>\n",
       "      <td>15023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5077</th>\n",
       "      <td>14496</td>\n",
       "      <td>441262</td>\n",
       "      <td>15024</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        nid     uid    tid\n",
       "5073  14247  508251  15019\n",
       "5074  14487  508247  15020\n",
       "5075  14496  441262  15022\n",
       "5076  14496  441262  15023\n",
       "5077  14496  441262  15024"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print tag_usage.shape[0]\n",
    "tag_usage.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T14:44:25.485335Z",
     "start_time": "2019-04-22T14:44:25.473069Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROWS WITH NAN'S: \n",
      "\n",
      "NID: 0\n",
      "UID: 0\n",
      "TID: 0\n"
     ]
    }
   ],
   "source": [
    "print 'ROWS WITH NAN\\'S:', '\\n'\n",
    "print 'NID:', tag_usage['nid'].isnull().sum()\n",
    "print 'UID:', tag_usage['uid'].isnull().sum()\n",
    "print 'TID:', tag_usage['tid'].isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T13:59:34.320523Z",
     "start_time": "2019-04-22T13:59:34.295024Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13065\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tid</th>\n",
       "      <th>name</th>\n",
       "      <th>power_type</th>\n",
       "      <th>count</th>\n",
       "      <th>power</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13062</th>\n",
       "      <td>20330</td>\n",
       "      <td>phun</td>\n",
       "      <td>npt</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13063</th>\n",
       "      <td>20331</td>\n",
       "      <td>sprinkler</td>\n",
       "      <td>npt</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13064</th>\n",
       "      <td>20332</td>\n",
       "      <td>trung</td>\n",
       "      <td>npt</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13065</th>\n",
       "      <td>20333</td>\n",
       "      <td>10.823098</td>\n",
       "      <td>lat</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13066</th>\n",
       "      <td>20334</td>\n",
       "      <td>106.629663</td>\n",
       "      <td>lon</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         tid        name power_type  count  power\n",
       "13062  20330        phun        npt    1.0  False\n",
       "13063  20331   sprinkler        npt    1.0  False\n",
       "13064  20332       trung        npt    1.0  False\n",
       "13065  20333   10.823098        lat    1.0   True\n",
       "13066  20334  106.629663        lon    1.0   True"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print terms.shape[0]\n",
    "terms.tail(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T14:50:58.890504Z",
     "start_time": "2019-04-22T14:50:58.846110Z"
    },
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LEFT (MANY) \n",
      "         nid     uid    tid\n",
      "10989  19111  584933  20315\n",
      "10990  18421       1  20316\n",
      "10991  19113  584971  20317\n",
      "10992    625       1  20318\n",
      "10993   9922       1  20318\n",
      "10994  11581       1  20318\n",
      "10995    627       1  20319 \n",
      "\n",
      "RIGHT(ONE) \n",
      "         tid                         name power_type  count  power\n",
      "13047  20315                         laga        npt    1.0  False\n",
      "13048  20316             pipeline-webinar   redirect    1.0   True\n",
      "13049  20317                          psn        npt    1.0  False\n",
      "13050  20318  thermal-flashlight-assembly   activity    5.0   True\n",
      "13051  20319                   pirateship        npt    1.0  False \n",
      "\n",
      "MERGED\n",
      "         nid     uid    tid                         name power_type  count\n",
      "48908  19111  584933  20315                         laga        npt    1.0\n",
      "48909  18421       1  20316             pipeline-webinar   redirect    1.0\n",
      "48910  19113  584971  20317                          psn        npt    1.0\n",
      "48911    625       1  20318  thermal-flashlight-assembly   activity    5.0\n",
      "48912   9922       1  20318  thermal-flashlight-assembly   activity    5.0\n",
      "48913  11581       1  20318  thermal-flashlight-assembly   activity    5.0\n",
      "48914    627       1  20319                   pirateship        npt    1.0 \n",
      "\n",
      "TOTALS \n",
      "\n",
      "TAG USAGE: \t54004 3\n",
      "TERMS: \t\t13065 5\n",
      "MERGED: \t54004 7\n"
     ]
    }
   ],
   "source": [
    "# https://stackoverflow.com/a/12098586/6023530\n",
    "selection = list(terms.iloc[-20:-15]['tid'])\n",
    "tag_usage[tag_usage['tid'].isin(selection)]\n",
    "\n",
    "print 'LEFT (MANY) \\n', tag_usage[tag_usage['tid'].isin(selection)], '\\n'\n",
    "print 'RIGHT(ONE) \\n', terms.iloc[-20:-15], '\\n'\n",
    "\n",
    "# https://pandas.pydata.org/pandas-docs/stable/user_guide/merging.html#brief-primer-on-merge-methods-relational-algebra\n",
    "# merged = pd.merge(tag_usage, terms, on='tid', how='left')\n",
    "\n",
    "print 'MERGED'\n",
    "print merged[merged['tid'].isin(selection)].drop('power', axis=1), '\\n'\n",
    "\n",
    "print 'TOTALS \\n'\n",
    "print 'TAG USAGE:', '\\t', tag_usage.shape[0], tag_usage.shape[1]\n",
    "print 'TERMS:', '\\t\\t', terms.shape[0], terms.shape[1]\n",
    "print 'MERGED:', '\\t', merged.shape[0], merged.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T15:49:07.556033Z",
     "start_time": "2019-04-22T15:49:07.522210Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nid</th>\n",
       "      <th>uid</th>\n",
       "      <th>tid</th>\n",
       "      <th>name</th>\n",
       "      <th>power_type</th>\n",
       "      <th>count</th>\n",
       "      <th>power</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>48908</th>\n",
       "      <td>19111</td>\n",
       "      <td>584933</td>\n",
       "      <td>20315</td>\n",
       "      <td>laga</td>\n",
       "      <td>npt</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48909</th>\n",
       "      <td>18421</td>\n",
       "      <td>1</td>\n",
       "      <td>20316</td>\n",
       "      <td>pipeline-webinar</td>\n",
       "      <td>redirect</td>\n",
       "      <td>1.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48910</th>\n",
       "      <td>19113</td>\n",
       "      <td>584971</td>\n",
       "      <td>20317</td>\n",
       "      <td>psn</td>\n",
       "      <td>npt</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48911</th>\n",
       "      <td>625</td>\n",
       "      <td>1</td>\n",
       "      <td>20318</td>\n",
       "      <td>thermal-flashlight-assembly</td>\n",
       "      <td>activity</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48912</th>\n",
       "      <td>9922</td>\n",
       "      <td>1</td>\n",
       "      <td>20318</td>\n",
       "      <td>thermal-flashlight-assembly</td>\n",
       "      <td>activity</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48913</th>\n",
       "      <td>11581</td>\n",
       "      <td>1</td>\n",
       "      <td>20318</td>\n",
       "      <td>thermal-flashlight-assembly</td>\n",
       "      <td>activity</td>\n",
       "      <td>5.0</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48914</th>\n",
       "      <td>627</td>\n",
       "      <td>1</td>\n",
       "      <td>20319</td>\n",
       "      <td>pirateship</td>\n",
       "      <td>npt</td>\n",
       "      <td>1.0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         nid     uid    tid                         name power_type  count  \\\n",
       "48908  19111  584933  20315                         laga        npt    1.0   \n",
       "48909  18421       1  20316             pipeline-webinar   redirect    1.0   \n",
       "48910  19113  584971  20317                          psn        npt    1.0   \n",
       "48911    625       1  20318  thermal-flashlight-assembly   activity    5.0   \n",
       "48912   9922       1  20318  thermal-flashlight-assembly   activity    5.0   \n",
       "48913  11581       1  20318  thermal-flashlight-assembly   activity    5.0   \n",
       "48914    627       1  20319                   pirateship        npt    1.0   \n",
       "\n",
       "       power  \n",
       "48908  False  \n",
       "48909   True  \n",
       "48910  False  \n",
       "48911   True  \n",
       "48912   True  \n",
       "48913   True  \n",
       "48914  False  "
      ]
     },
     "execution_count": 321,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged.to_csv(csv_paths['tags'], index=False)\n",
    "tags = pd.read_csv(csv_paths['tags'])\n",
    "tags[tags['tid'].isin(selection)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SEPPERATE POWER TAG TYPES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T15:59:05.882249Z",
     "start_time": "2019-04-22T15:59:05.845006Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>power_type</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>npt</td>\n",
       "      <td>44866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>question</td>\n",
       "      <td>2389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>activity</td>\n",
       "      <td>1126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>lat</td>\n",
       "      <td>988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>lon</td>\n",
       "      <td>979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>response</td>\n",
       "      <td>810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>parent</td>\n",
       "      <td>361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>with</td>\n",
       "      <td>350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>barnstar</td>\n",
       "      <td>240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>event</td>\n",
       "      <td>211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>date</td>\n",
       "      <td>203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>replication</td>\n",
       "      <td>197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>place</td>\n",
       "      <td>153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>hidden</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>rsvp</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>list</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>location</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>category</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>redirect</td>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>style</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     power_type  count\n",
       "0           npt  44866\n",
       "1      question   2389\n",
       "2      activity   1126\n",
       "3           lat    988\n",
       "4           lon    979\n",
       "5      response    810\n",
       "6        parent    361\n",
       "7          with    350\n",
       "8      barnstar    240\n",
       "9         event    211\n",
       "10         date    203\n",
       "11  replication    197\n",
       "12        place    153\n",
       "13       hidden    105\n",
       "14         rsvp    102\n",
       "15         list     87\n",
       "16     location     84\n",
       "17     category     82\n",
       "18     redirect     74\n",
       "19        style     64"
      ]
     },
     "execution_count": 351,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "power_tag_counts = tags['power_type'].value_counts().reset_index()\n",
    "power_tag_counts.columns = ['power_type', 'count']\n",
    "# power_tag_counts = power_tag_counts[['power_type', 'count']]\n",
    "power_tag_counts.head(20)\n",
    "# power_tag_counts[20:40]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 344,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T15:56:08.812267Z",
     "start_time": "2019-04-22T15:56:08.773817Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>power_type</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>npt</td>\n",
       "      <td>44866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>question</td>\n",
       "      <td>2389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>activity</td>\n",
       "      <td>1126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>lat</td>\n",
       "      <td>988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>lon</td>\n",
       "      <td>979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>response</td>\n",
       "      <td>810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>parent</td>\n",
       "      <td>361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>with</td>\n",
       "      <td>350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>barnstar</td>\n",
       "      <td>240</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>event</td>\n",
       "      <td>211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>date</td>\n",
       "      <td>203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>replication</td>\n",
       "      <td>197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>hidden</td>\n",
       "      <td>105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>rsvp</td>\n",
       "      <td>102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>place</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>list</td>\n",
       "      <td>87</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>location</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>category</td>\n",
       "      <td>82</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>redirect</td>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>style</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     power_type  count\n",
       "0           npt  44866\n",
       "1      question   2389\n",
       "2      activity   1126\n",
       "3           lat    988\n",
       "4           lon    979\n",
       "5      response    810\n",
       "6        parent    361\n",
       "7          with    350\n",
       "8      barnstar    240\n",
       "9         event    211\n",
       "10         date    203\n",
       "11  replication    197\n",
       "12       hidden    105\n",
       "13         rsvp    102\n",
       "14        place    100\n",
       "15         list     87\n",
       "16     location     84\n",
       "17     category     82\n",
       "18     redirect     74\n",
       "19        style     64"
      ]
     },
     "execution_count": 344,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#\n",
    "\n",
    "# ['question', 'activity', 'with', 'barnstar' 'event', 'replication', 'place',  ]\n",
    "\n",
    "# LIST: GOOGLE GROUP TAGS, list:plots-spectrometry\n",
    "# WITH: THE USERS WHO ALSO WROTE THE NOTE, with:bsugar\n",
    "# BARNSTAR: CONTENT AWARDED A BARNSTAR\n",
    "\n",
    "\n",
    "# https://stackoverflow.com/a/38309823/6023530\n",
    "# tags.loc[tags['power_type'] == 'barnstar'].groupby('name')['nid'].nunique()#value_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 329,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T15:50:22.987218Z",
     "start_time": "2019-04-22T15:50:22.702303Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f63f2478e90>"
      ]
     },
     "execution_count": 329,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAD8CAYAAAB3u9PLAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAEZVJREFUeJzt3X2QXXV9x/H3Nw8SArGQZLWRkG6w\nDg86eeo2Yw0FQyzy0ErtpK0KIYhl+6AGp+3UKLbyDx2tFWOmTmusoYAog1HA+owl6MBQMBGExEBB\nDbiGSkzaEh9CEvj2j/tbWGOye7O7v3v34f2a2dlzz557ft/z49x8OOd37jmRmUiSNKHdBUiSRgYD\nQZIEGAiSpMJAkCQBBoIkqTAQJEmAgSBJKgwESRJgIEiSikntLqCvmTNnZmdnZ7vLkKRRY/PmzT/O\nzI7hWNeICoTOzk42bdrU7jIkadSIiMeGa12eMpIkAQaCJKkwECRJwAgbQ5Ck/uzfv5+enh727t3b\n7lJabsqUKcyePZvJkydXa8NAkDRq9PT0MG3aNDo7O4mIdpfTMpnJrl276OnpYe7cudXaqXrKKCKO\ni4gNEfFQRGyLiN+q2Z6ksW3v3r3MmDFjXIUBQEQwY8aM6kdGtY8QPgx8OTOXR8QLgKmV25M0xo23\nMOjViu2uFggR8ULgDOASgMzcB+yr1Z4kaWhqHiGcBOwEromI+cBm4PLM/GnFNiWNI52rvzCs69v+\nvvOHdX2DsWbNGrq7u5k6tfUnVGoGwiRgEfD2zLwnIj4MrAb+tu9CEdENdAPMmTNn0I2NxR1D0viz\nZs0aLrroorYEQs1B5R6gJzPvKa830AiIX5CZ6zKzKzO7OjqG5XYcklTVddddx7x585g/fz4rVqzg\nscceY9myZcybN49ly5bx+OOPA3DJJZewYcOG59537LHHAnDHHXfw6le/muXLl3PKKadw4YUXkpms\nXbuWHTt2sHTpUpYuXdry7ap2hJCZ/x0RP4iIkzPzYWAZ8J1a7UlSK2zdupWrrrqKu+66i5kzZ7J7\n925WrlzJxRdfzMqVK1m/fj2rVq3illtu6Xc99913H1u3buUlL3kJS5Ys4a677mLVqlVcffXVbNy4\nkZkzZ7Zoi55X+5vKbwduiIgHgAXA31duT5Kquv3221m+fPlz/2BPnz6du+++mze96U0ArFixgjvv\nvHPA9SxevJjZs2czYcIEFixYwPbt22uW3ZSql51m5v1AV802JKmVMnPAS0B7/z5p0iSeffbZ5963\nb9/zF1oeddRRz01PnDiRAwcOVKj2yHgvI0k6AsuWLeOmm25i165dAOzevZtXvepV3HjjjQDccMMN\nnH766UDjlv6bN28G4NZbb2X//v0Drn/atGns2bOnUvX989YVkkatdlwN+PKXv5wrrriCM888k4kT\nJ7Jw4ULWrl3LpZdeygc+8AE6Ojq45pprALjsssu44IILWLx4McuWLeOYY44ZcP3d3d2ce+65zJo1\ni40bN9benF8QmdnSBvvT1dWVg31AjpedSmPftm3bOPXUU9tdRtscavsjYnNmDsupeU8ZSZIAA0GS\nVBgIkkaVkXSau5Vasd0GgqRRY8qUKezatWvchULv8xCmTJlStR2vMpI0asyePZuenh527tzZ7lJa\nrveJaTUZCJJGjcmTJ1d9Yth45ykjSRJgIEiSCgNBkgQYCJKkwkCQJAEGgiSpMBAkSYCBIEkqDARJ\nEmAgSJIKA0GSBBgIkqTCQJAkAQaCJKkwECRJgIEgSSoMBEkSUPmJaRGxHdgDPAMcyMyumu1Jkgav\nFY/QXJqZP25BO5KkIfCUkSQJqB8ICXw1IjZHRPehFoiI7ojYFBGbdu7cWbkcSdLh1A6EJZm5CDgX\neGtEnHHwApm5LjO7MrOro6OjcjmSpMOpGgiZuaP8fhK4GVhcsz1J0uBVC4SIOCYipvVOA2cDW2q1\nJ0kamppXGb0YuDkietv5ZGZ+uWJ7kqQhqBYImfk9YH6t9UuShpeXnUqSAANBklQYCJIkwECQJBUG\ngiQJMBAkSYWBIEkCDARJUmEgSJIAA0GSVBgIkiTAQJAkFQaCJAkwECRJhYEgSQIMBElSYSBIkgAD\nQZJUGAiSJMBAkCQVBoIkCTAQJEmFgSBJAgwESVJhIEiSgBYEQkRMjIj7IuLztduSJA1eK44QLge2\ntaAdSdIQVA2EiJgNnA/8a812JElDV/sIYQ3wN8CzlduRJA1RtUCIiN8FnszMzQMs1x0RmyJi086d\nO2uVI0kaQM0jhCXA6yJiO3AjcFZEfOLghTJzXWZ2ZWZXR0dHxXIkSf2pFgiZ+a7MnJ2ZncAbgNsz\n86Ja7UmShsbvIUiSAJjUikYy8w7gjla0JUkanKaOECLiFbULkSS1V7OnjP4lIu6NiL+IiOOqViRJ\naoumAiEzTwcuBE4ENkXEJyPid6pWJklqqaYHlTPzEeA9wDuBM4G1EfFQRPxBreIkSa3T7BjCvIj4\nEI17Ep0F/F5mnlqmP1SxPklSizR7ldE/AR8D3p2ZP++dmZk7IuI9VSqTJLVUs4FwHvDzzHwGICIm\nAFMy82eZeX216iRJLdPsGMLXgKP7vJ5a5kmSxohmA2FKZv6k90WZnlqnJElSOzQbCD+NiEW9LyLi\nN4Cf97O8JGmUaXYM4R3ApyNiR3k9C/jjOiVJktqhqUDIzG9GxCnAyUAAD2Xm/qqVSZJa6khubveb\nQGd5z8KIIDOvq1KVJKnlmgqEiLgeeClwP/BMmZ2AgSBJY0SzRwhdwGmZmTWLkSS1T7NXGW0BfrVm\nIZKk9mr2CGEm8J2IuBd4undmZr6uSlWSpJZrNhCurFmEJKn9mr3s9OsR8WvAyzLzaxExFZhYtzRJ\nUis1e/vry4ANwEfLrBOAW2oVJUlqvWYHld8KLAGegucelvOiWkVJklqv2UB4OjP39b6IiEk0vocg\nSRojmg2Er0fEu4Gjy7OUPw38e72yJEmt1mwgrAZ2Ag8Cfwp8kcbzlSVJY0SzVxk9S+MRmh+rW44k\nqV2avZfR9znEmEFmnjTsFUmS2uJI7mXUawrwh8D0/t4QEVOAbwBHlXY2ZOZ7B1OkJKm+psYQMnNX\nn58fZuYa4KwB3vY0cFZmzgcWAOdExCuHWK8kqZJmTxkt6vNyAo0jhmn9vafcGbX3OcyTy4+XqkrS\nCNXsKaMP9pk+AGwH/migN0XERGAz8OvARzLznkMs0w10A8yZM6fJciRJw63Zq4yWDmblmfkMsCAi\njgNujohXZOaWg5ZZB6wD6Orq8ghCktqk2VNGf9nf3zPz6gH+/r8RcQdwDo1nK0iSRphmv5jWBfw5\njZvanQD8GXAajXGEQ44lRERHOTIgIo4GXgM8NNSCJUl1HMkDchZl5h6AiLgS+HRm/kk/75kFXFvG\nESYAN2Xm54dSrCSpnmYDYQ6wr8/rfUBnf2/IzAeAhYMrS5LUas0GwvXAvRFxM41LR18PXFetKklS\nyzV7ldFVEfEl4LfLrDdn5n31ypIktVqzg8oAU4GnMvPDQE9EzK1UkySpDZp9hOZ7gXcC7yqzJgOf\nqFWUJKn1mj1CeD3wOuCnAJm5gwFuXSFJGl2aDYR95d5ECRARx9QrSZLUDs0Gwk0R8VHguIi4DPga\nPixHksaUZq8y+sfyLOWngJOBv8vM26pWJklqqQEDoXzT+CuZ+RrAEJCkMWrAU0bljqU/i4hfaUE9\nkqQ2afabynuBByPiNsqVRgCZuapKVZKklms2EL5QfiRJY1S/gRARczLz8cy8tlUFSZLaY6AxhFt6\nJyLiM5VrkSS10UCBEH2mT6pZiCSpvQYKhDzMtCRpjBloUHl+RDxF40jh6DJNeZ2Z+cKq1UmSWqbf\nQMjMia0qRJLUXkfyPARJ0hhmIEiSAANBklQYCJIkwECQJBUGgiQJMBAkSUW1QIiIEyNiY0Rsi4it\nEXF5rbYkSUPX7O2vB+MA8FeZ+a2ImAZsjojbMvM7FduUJA1StSOEzHwiM79VpvcA24ATarUnSRqa\nlowhREQnsBC4pxXtSZKOXM1TRgBExLHAZ4B3ZOZTh/h7N9ANMGfOnNrlNK1z9fA+IG77+84f1vWN\nBsPdhzA++1FqlapHCBExmUYY3JCZnz3UMpm5LjO7MrOro6OjZjmSpH7UvMoogI8D2zLz6lrtSJKG\nR80jhCXACuCsiLi//JxXsT1J0hBUG0PIzDv5xUdwSpJGML+pLEkCDARJUmEgSJIAA0GSVBgIkiTA\nQJAkFQaCJAkwECRJhYEgSQIMBElSYSBIkgADQZJUGAiSJMBAkCQVBoIkCTAQJEmFgSBJAgwESVJh\nIEiSAANBklQYCJIkwECQJBUGgiQJMBAkSYWBIEkCDARJUlEtECJifUQ8GRFbarUhSRo+NY8Q/g04\np+L6JUnDqFogZOY3gN211i9JGl6T2l1ARHQD3QBz5sxpczXjW+fqL7S7hAENd43b33f+sK5P48NY\n3Q/bPqicmesysyszuzo6OtpdjiSNW20PBEnSyGAgSJKAupedfgq4Gzg5Inoi4i212pIkDV21QeXM\nfGOtdUuShp+njCRJgIEgSSoMBEkSYCBIkgoDQZIEGAiSpMJAkCQBBoIkqTAQJEmAgSBJKgwESRJg\nIEiSCgNBkgQYCJKkwkCQJAEGgiSpMBAkSYCBIEkqDARJEmAgSJIKA0GSBBgIkqTCQJAkAQaCJKkw\nECRJQOVAiIhzIuLhiHg0IlbXbEuSNDTVAiEiJgIfAc4FTgPeGBGn1WpPkjQ0NY8QFgOPZub3MnMf\ncCNwQcX2JElDUDMQTgB+0Od1T5knSRqBJlVcdxxiXv7SQhHdQHd5+ZOI2AX8uGJdbRHvH9TbZjIG\n+2IQqvXDIP+7tJP7RMOY6och7ocnD1MZVQOhBzixz+vZwI6DF8rMdcC63tcRsSkzuyrWNWrYFw32\nw/Psiwb74XkRsWm41lXzlNE3gZdFxNyIeAHwBuBzFduTJA1BtSOEzDwQEW8DvgJMBNZn5tZa7UmS\nhqbmKSMy84vAF4/wbesGXmTcsC8a7Ifn2RcN9sPzhq0vIvOXxnklSeOQt66QJAEjLBDG260uImJ7\nRDwYEff3XikQEdMj4raIeKT8Pr7Mj4hYW/rmgYhY1N7qhyYi1kfEkxGxpc+8I972iFhZln8kIla2\nY1uG4jD9cGVE/LDsF/dHxHl9/vau0g8PR8Rr+8wf9Z+diDgxIjZGxLaI2BoRl5f542q/6Kcf6u8X\nmTkifmgMPH8XOAl4AfBt4LR211V5m7cDMw+a9w/A6jK9Gnh/mT4P+BKN73e8Erin3fUPcdvPABYB\nWwa77cB04Hvl9/Fl+vh2b9sw9MOVwF8fYtnTyufiKGBu+bxMHCufHWAWsKhMTwP+q2zzuNov+umH\n6vvFSDpC8FYXDRcA15bpa4Hf7zP/umz4T+C4iJjVjgKHQ2Z+A9h90Owj3fbXArdl5u7M/B/gNuCc\n+tUPn8P0w+FcANyYmU9n5veBR2l8bsbEZyczn8jMb5XpPcA2Gnc3GFf7RT/9cDjDtl+MpEAYj7e6\nSOCrEbG5fGMb4MWZ+QQ0dgzgRWX+eOifI932sdwnbyunQdb3niJhHPVDRHQCC4F7GMf7xUH9AJX3\ni5EUCE3d6mKMWZKZi2jcEfatEXFGP8uOx/7pdbhtH6t98s/AS4EFwBPAB8v8cdEPEXEs8BngHZn5\nVH+LHmLemOmPQ/RD9f1iJAVCU7e6GEsyc0f5/SRwM41DvB/1ngoqv58si4+H/jnSbR+TfZKZP8rM\nZzLzWeBjNPYLGAf9EBGTafwjeENmfrbMHnf7xaH6oRX7xUgKhHF1q4uIOCYipvVOA2cDW2hsc+9V\nESuBW8v054CLy5UVrwT+r/cwegw50m3/CnB2RBxfDp/PLvNGtYPGhl5PY7+ARj+8ISKOioi5wMuA\nexkjn52ICODjwLbMvLrPn8bVfnG4fmjJftHuEfWDRsvPozGi/l3ginbXU3lbT6Ix6v9tYGvv9gIz\ngP8AHim/p5f5QeOBQ98FHgS62r0NQ9z+T9E47N1P4/9k3jKYbQcupTGI9ijw5nZv1zD1w/VlOx8o\nH+BZfZa/ovTDw8C5feaP+s8OcDqNUxoPAPeXn/PG237RTz9U3y/8prIkCRhZp4wkSW1kIEiSAANB\nklQYCJIkwECQJBUGgiQJMBAkSYWBIEkC4P8BwcM4rIX8UsEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f6400432150>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "count_gt_one = (power_tag_counts['count'] > 100)\n",
    "not_npt = (power_tag_counts['power_type'] != 'npt')\n",
    "filtered = power_tag_counts.loc[count_gt_one & not_npt]\n",
    "filtered.plot(kind='hist', bins=15)\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T17:12:57.794843Z",
     "start_time": "2019-04-22T17:12:57.786725Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "power_categories = ['question', 'activity', 'with', 'barnstar' 'event', 'replication', 'place']\n",
    "\n",
    "# LIST: GOOGLE GROUP TAGS, list:plots-spectrometry\n",
    "# WITH: THE USERS WHO ALSO WROTE THE NOTE, with:bsugar\n",
    "# BARNSTAR: CONTENT AWARDED A BARNSTAR\n",
    "\n",
    "# COMBINE TWO CATEGORIES\n",
    "# tags.loc[tags['power_type'] == 'activities', 'power_type'] = 'activity'\n",
    "# tags.loc[tags['power_type'] == 'events', 'power_type'] = 'place'\n",
    "\n",
    "# https://stackoverflow.com/a/38309823/6023530\n",
    "tags.loc[tags['power_type'] == 'events'].groupby('name')['nid'].nunique()#value_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-22T17:16:04.908732Z",
     "start_time": "2019-04-22T17:16:04.886811Z"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>power_type</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>question</td>\n",
       "      <td>2389</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>activity</td>\n",
       "      <td>1126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>with</td>\n",
       "      <td>350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>replication</td>\n",
       "      <td>197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>place</td>\n",
       "      <td>153</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     power_type  count\n",
       "1      question   2389\n",
       "2      activity   1126\n",
       "7          with    350\n",
       "11  replication    197\n",
       "12        place    153"
      ]
     },
     "execution_count": 353,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "power_categories = ['question', 'activity', 'with', 'barnstar' 'event', 'replication', 'place']\n",
    "\n",
    "power_tag_counts[power_tag_counts['power_type'].isin(power_categories)]\n",
    "\n",
    "# power_tag_counts = tags['power_type'].value_counts().reset_index()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  },
  "toc": {
   "colors": {
    "hover_highlight": "#DAA520",
    "navigate_num": "#000000",
    "navigate_text": "#333333",
    "running_highlight": "#FF0000",
    "selected_highlight": "#FFD700",
    "sidebar_border": "#EEEEEE",
    "wrapper_background": "#FFFFFF"
   },
   "moveMenuLeft": true,
   "nav_menu": {
    "height": "12px",
    "width": "252px"
   },
   "navigate_menu": true,
   "number_sections": true,
   "sideBar": true,
   "threshold": 4,
   "toc_cell": false,
   "toc_section_display": "block",
   "toc_window_display": false,
   "widenNotebook": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}