gitauth.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import requests\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load in api key\n",
    "credentials = json.load(open('./apikey.json', 'r'), encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GHE docs:\n",
    "# curl -H \"Authorization: token OAUTH-TOKEN\" http(s)://[hostname]/api/v3  <-- endpoint\n",
    "''' \n",
    "Declare global variables\n",
    "'''\n",
    "baseurl = 'git.generalassemb.ly' # of course may be different for you\n",
    "header = {'Authorization': 'token {}'.format(credentials['token'])}\n",
    "source_org = 'Data-science-immersive' # change this as you see fit\n",
    "target_org = 'DSI-ME-1' # this one too\n",
    "repo_names = []\n",
    "data = { 'organization': target_org }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get a list of all the orgs the API key has access to"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "dsi-unit-3: id = 261\n",
    "DSI-ATX-3: id = 6931\n",
    "data-part-time: id = 6944\n",
    "DSI-EAST-1: id = 7793\n",
    "DSI-DC-6: id = 7895\n",
    "DSI-EAST-2: id = 9038\n",
    "Data-science-immersive: id = 9634\n",
    "DAT-ME-1: id = 11079\n",
    "DSI-ME-1: id = 11622"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dsi-unit-3: id = 261\n",
      "DSI-ATX-3: id = 6931\n",
      "data-part-time: id = 6944\n",
      "DSI-EAST-1: id = 7793\n",
      "DSI-DC-6: id = 7895\n",
      "DSI-EAST-2: id = 9038\n",
      "Data-science-immersive: id = 9634\n",
      "DAT-ME-1: id = 11079\n",
      "DSI-ME-1: id = 11622\n",
      "DAT-COG-1: id = 16028\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "Example calls below\n",
    "'''\n",
    "# call the base endpoint\n",
    "'''\n",
    "r = requests.get('https://{}/api/v3'.format(baseurl), headers=header)\n",
    "'''\n",
    "\n",
    "# get a listing of orgs the user's api key has access to\n",
    "\n",
    "r = requests.get('https://{}/api/v3/user/orgs'.format(baseurl), headers=header)\n",
    "orgs = json.loads(r.content)\n",
    "for org in orgs:\n",
    "    print('{}: id = {}'.format(org['login'], org['id']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here's an example of a single repo's attributes in `data-science-immersive`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = requests.get('https://{}/api/v3/orgs/{}/repos'.format(baseurl, source_org), \\\n",
    "         headers=header, params={ 'type': 'all' , 'page': 4 })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(r.json())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get a dataframe with all the repos from that org"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_repos(baseurl, source_org, startpage=1):\n",
    "    # set initial page to fetch\n",
    "    page = startpage\n",
    "    # create blank dataframe to put results in\n",
    "    df_dict = {\n",
    "        'id': [], \n",
    "        'name': [],\n",
    "        'full_name': [],\n",
    "        'ssh_url': [],\n",
    "        'created_at': [],\n",
    "        'updated_at': [],\n",
    "        'pushed_at': [],\n",
    "        'forks_url': [],\n",
    "        'forks_count': [],\n",
    "        'ssh_url': [],\n",
    "        'page': [] # this doesn't come from the api\n",
    "    }\n",
    "    repo_df = pd.DataFrame.from_dict(df_dict).set_index('id')\n",
    "    # begin fetchin those shits\n",
    "    while True:\n",
    "        r = requests.get('https://{}/api/v3/orgs/{}/repos'.format(baseurl, source_org), \\\n",
    "                 headers=header, params={ 'type': 'all' , 'page': page })\n",
    "        try:\n",
    "            # see if there's data in the response\n",
    "            repos = r.json()\n",
    "            repos[0]\n",
    "            print('Successfully fetched page {}'.format(page))\n",
    "\n",
    "            for repo in repos:\n",
    "                for k in df_dict.keys():\n",
    "                    try:\n",
    "                        df_dict[k].append(repo[k])\n",
    "                    # this is kinda a hack to insert the page num we're on\n",
    "                    except KeyError:\n",
    "                        df_dict['page'].append(page)\n",
    "            # put the values from the dict into res_df\n",
    "            res_df = pd.DataFrame.from_dict(df_dict).set_index('id')\n",
    "            # clear dict for next page from api\n",
    "            for k in df_dict.keys():\n",
    "                df_dict[k] = []\n",
    "            # append res_df to initial blank dataframe\n",
    "            repo_df = repo_df.copy().append(res_df)\n",
    "\n",
    "            page += 1\n",
    "        except IndexError:\n",
    "            print('Page {} returned no data, exiting...'.format(page))\n",
    "            \n",
    "            # change the index from float to int\n",
    "            repo_df.set_index(repo_df.index.astype(int), inplace=True)\n",
    "\n",
    "            # make the date/time cols into ....datetime\n",
    "            for col in ['created_at', 'updated_at', 'pushed_at']:\n",
    "                repo_df[col] = pd.to_datetime(repo_df.copy()[col])\n",
    "\n",
    "            # change page and forks_count numbers from floats to ints\n",
    "            for col in ['page', 'forks_count']:\n",
    "                repo_df[col] = repo_df.copy()[col].apply(lambda x: int(x))   \n",
    "            \n",
    "            return repo_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's get the `DSI-ME-1` org repos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully fetched page 1\n",
      "Successfully fetched page 2\n",
      "Successfully fetched page 3\n",
      "Successfully fetched page 4\n",
      "Page 5 returned no data, exiting...\n"
     ]
    }
   ],
   "source": [
    "misk_df = get_all_repos(baseurl, 'DSI-ME-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>full_name</th>\n",
       "      <th>ssh_url</th>\n",
       "      <th>created_at</th>\n",
       "      <th>updated_at</th>\n",
       "      <th>pushed_at</th>\n",
       "      <th>forks_url</th>\n",
       "      <th>forks_count</th>\n",
       "      <th>page</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134818</th>\n",
       "      <td>templates</td>\n",
       "      <td>DSI-ME-1/templates</td>\n",
       "      <td>git@git.generalassemb.ly:DSI-ME-1/templates.git</td>\n",
       "      <td>2018-05-28 22:55:55</td>\n",
       "      <td>2018-05-28 22:55:57</td>\n",
       "      <td>2018-04-25 20:28:20</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/DSI-...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134819</th>\n",
       "      <td>python-101-lesson</td>\n",
       "      <td>DSI-ME-1/python-101-lesson</td>\n",
       "      <td>git@git.generalassemb.ly:DSI-ME-1/python-101-l...</td>\n",
       "      <td>2018-05-28 22:55:56</td>\n",
       "      <td>2018-05-28 22:55:57</td>\n",
       "      <td>2018-08-01 06:17:41</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/DSI-...</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134820</th>\n",
       "      <td>python-comprehensions-lab</td>\n",
       "      <td>DSI-ME-1/python-comprehensions-lab</td>\n",
       "      <td>git@git.generalassemb.ly:DSI-ME-1/python-compr...</td>\n",
       "      <td>2018-05-28 22:55:56</td>\n",
       "      <td>2018-05-28 22:55:58</td>\n",
       "      <td>2018-04-30 18:09:19</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/DSI-...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             name                           full_name  \\\n",
       "id                                                                      \n",
       "134818                  templates                  DSI-ME-1/templates   \n",
       "134819          python-101-lesson          DSI-ME-1/python-101-lesson   \n",
       "134820  python-comprehensions-lab  DSI-ME-1/python-comprehensions-lab   \n",
       "\n",
       "                                                  ssh_url          created_at  \\\n",
       "id                                                                              \n",
       "134818    git@git.generalassemb.ly:DSI-ME-1/templates.git 2018-05-28 22:55:55   \n",
       "134819  git@git.generalassemb.ly:DSI-ME-1/python-101-l... 2018-05-28 22:55:56   \n",
       "134820  git@git.generalassemb.ly:DSI-ME-1/python-compr... 2018-05-28 22:55:56   \n",
       "\n",
       "                updated_at           pushed_at  \\\n",
       "id                                               \n",
       "134818 2018-05-28 22:55:57 2018-04-25 20:28:20   \n",
       "134819 2018-05-28 22:55:57 2018-08-01 06:17:41   \n",
       "134820 2018-05-28 22:55:58 2018-04-30 18:09:19   \n",
       "\n",
       "                                                forks_url  forks_count  page  \n",
       "id                                                                            \n",
       "134818  https://git.generalassemb.ly/api/v3/repos/DSI-...            0     1  \n",
       "134819  https://git.generalassemb.ly/api/v3/repos/DSI-...           26     1  \n",
       "134820  https://git.generalassemb.ly/api/v3/repos/DSI-...            1     1  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "misk_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "misk_df.to_csv('./misk.csv', index_label='id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter just the new repos\n",
    "clones = list(misk_df[(misk_df['created_at'] > '8/1/2018 12:00:00')]['ssh_url'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's get the `DSI-ME-1` org repos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Successfully fetched page 1\n",
      "Successfully fetched page 2\n",
      "Page 3 returned no data, exiting...\n"
     ]
    }
   ],
   "source": [
    "dsi_me_1_df = get_all_repos(baseurl, 'DSI-ME-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>full_name</th>\n",
       "      <th>ssh_url</th>\n",
       "      <th>created_at</th>\n",
       "      <th>updated_at</th>\n",
       "      <th>pushed_at</th>\n",
       "      <th>forks_url</th>\n",
       "      <th>forks_count</th>\n",
       "      <th>page</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>134818</th>\n",
       "      <td>templates</td>\n",
       "      <td>DSI-ME-1/templates</td>\n",
       "      <td>git@git.generalassemb.ly:DSI-ME-1/templates.git</td>\n",
       "      <td>2018-05-28 22:55:55</td>\n",
       "      <td>2018-05-28 22:55:57</td>\n",
       "      <td>2018-04-25 20:28:20</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/DSI-...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134819</th>\n",
       "      <td>python-101-lesson</td>\n",
       "      <td>DSI-ME-1/python-101-lesson</td>\n",
       "      <td>git@git.generalassemb.ly:DSI-ME-1/python-101-l...</td>\n",
       "      <td>2018-05-28 22:55:56</td>\n",
       "      <td>2018-05-28 22:55:57</td>\n",
       "      <td>2018-08-01 06:17:41</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/DSI-...</td>\n",
       "      <td>26</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134820</th>\n",
       "      <td>python-comprehensions-lab</td>\n",
       "      <td>DSI-ME-1/python-comprehensions-lab</td>\n",
       "      <td>git@git.generalassemb.ly:DSI-ME-1/python-compr...</td>\n",
       "      <td>2018-05-28 22:55:56</td>\n",
       "      <td>2018-05-28 22:55:58</td>\n",
       "      <td>2018-04-30 18:09:19</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/DSI-...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             name                           full_name  \\\n",
       "id                                                                      \n",
       "134818                  templates                  DSI-ME-1/templates   \n",
       "134819          python-101-lesson          DSI-ME-1/python-101-lesson   \n",
       "134820  python-comprehensions-lab  DSI-ME-1/python-comprehensions-lab   \n",
       "\n",
       "                                                  ssh_url          created_at  \\\n",
       "id                                                                              \n",
       "134818    git@git.generalassemb.ly:DSI-ME-1/templates.git 2018-05-28 22:55:55   \n",
       "134819  git@git.generalassemb.ly:DSI-ME-1/python-101-l... 2018-05-28 22:55:56   \n",
       "134820  git@git.generalassemb.ly:DSI-ME-1/python-compr... 2018-05-28 22:55:56   \n",
       "\n",
       "                updated_at           pushed_at  \\\n",
       "id                                               \n",
       "134818 2018-05-28 22:55:57 2018-04-25 20:28:20   \n",
       "134819 2018-05-28 22:55:57 2018-08-01 06:17:41   \n",
       "134820 2018-05-28 22:55:58 2018-04-30 18:09:19   \n",
       "\n",
       "                                                forks_url  forks_count  page  \n",
       "id                                                                            \n",
       "134818  https://git.generalassemb.ly/api/v3/repos/DSI-...            0     1  \n",
       "134819  https://git.generalassemb.ly/api/v3/repos/DSI-...           26     1  \n",
       "134820  https://git.generalassemb.ly/api/v3/repos/DSI-...            1     1  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dsi_me_1_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "dsi_me_1_df.to_csv('./dsi_me_1.csv', index_label='id')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And now let's get a list of repos that are in `data-science-immersive` but _not_ in `DSI-ME-1`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_in_misk = data_science_immersive_df.merge(dsi_me_1_df, how='left', on='name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_in_misk = not_in_misk.copy()[not_in_misk.copy()['full_name_y'].isnull()][['name', 'forks_url_x', 'forks_count_x', 'created_at_x']] \\\n",
    "    .rename(columns={'forks_url_x': 'forks_url', 'forks_count_x': 'forks_count', 'created_at_x': 'created_at'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>forks_url</th>\n",
       "      <th>forks_count</th>\n",
       "      <th>created_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.02-lesson-python-control-flow</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/Data...</td>\n",
       "      <td>2</td>\n",
       "      <td>2018-04-30 18:12:39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2.03-lesson-eda</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/Data...</td>\n",
       "      <td>2</td>\n",
       "      <td>2018-05-10 18:36:37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>3.06-lesson-cross-validation-train-test-split</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/Data...</td>\n",
       "      <td>2</td>\n",
       "      <td>2018-05-10 18:47:34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             name  \\\n",
       "6                 1.02-lesson-python-control-flow   \n",
       "14                                2.03-lesson-eda   \n",
       "22  3.06-lesson-cross-validation-train-test-split   \n",
       "\n",
       "                                            forks_url  forks_count  \\\n",
       "6   https://git.generalassemb.ly/api/v3/repos/Data...            2   \n",
       "14  https://git.generalassemb.ly/api/v3/repos/Data...            2   \n",
       "22  https://git.generalassemb.ly/api/v3/repos/Data...            2   \n",
       "\n",
       "            created_at  \n",
       "6  2018-04-30 18:12:39  \n",
       "14 2018-05-10 18:36:37  \n",
       "22 2018-05-10 18:47:34  "
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_in_misk.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And a list of repos that are in `DSI-ME-1` but _not_ in `data-science-immersive`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_in_dsi = dsi_me_1_df.merge(data_science_immersive_df, how='left', on='name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_in_dsi = not_in_dsi.copy()[not_in_dsi.copy()['full_name_y'].isnull()][['name', 'forks_url_x', 'forks_count_x', 'created_at_x']] \\\n",
    "    .rename(columns={'forks_url_x': 'forks_url', 'forks_count_x': 'forks_count', 'created_at_x': 'created_at'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>forks_url</th>\n",
       "      <th>forks_count</th>\n",
       "      <th>created_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>python-iteration_control_functions-lesson</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/DSI-...</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-05-28 22:55:58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2.03-basic_eda_walkthrough-lesson-eda</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/DSI-...</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-05-28 22:56:02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>3.06-evaluation-cross_validation_train_test-le...</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/DSI-...</td>\n",
       "      <td>1</td>\n",
       "      <td>2018-05-28 22:56:05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 name  \\\n",
       "6           python-iteration_control_functions-lesson   \n",
       "14              2.03-basic_eda_walkthrough-lesson-eda   \n",
       "22  3.06-evaluation-cross_validation_train_test-le...   \n",
       "\n",
       "                                            forks_url  forks_count  \\\n",
       "6   https://git.generalassemb.ly/api/v3/repos/DSI-...            1   \n",
       "14  https://git.generalassemb.ly/api/v3/repos/DSI-...            1   \n",
       "22  https://git.generalassemb.ly/api/v3/repos/DSI-...            1   \n",
       "\n",
       "            created_at  \n",
       "6  2018-05-28 22:55:58  \n",
       "14 2018-05-28 22:56:02  \n",
       "22 2018-05-28 22:56:05  "
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_in_dsi.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It looks like some asshole renamed the repos along the way so a join by name isn't perfectly accurate and I can't find an ID field that would trace it upstream. I'll have to go manuelle here. Let's make a short list."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_in_misk = not_in_misk.copy().sort_values('created_at')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>forks_url</th>\n",
       "      <th>forks_count</th>\n",
       "      <th>created_at</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.02-lesson-python-control-flow</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/Data...</td>\n",
       "      <td>2</td>\n",
       "      <td>2018-04-30 18:12:39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>2.03-lesson-eda</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/Data...</td>\n",
       "      <td>2</td>\n",
       "      <td>2018-05-10 18:36:37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>3.06-lesson-cross-validation-train-test-split</td>\n",
       "      <td>https://git.generalassemb.ly/api/v3/repos/Data...</td>\n",
       "      <td>2</td>\n",
       "      <td>2018-05-10 18:47:34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             name  \\\n",
       "6                 1.02-lesson-python-control-flow   \n",
       "14                                2.03-lesson-eda   \n",
       "22  3.06-lesson-cross-validation-train-test-split   \n",
       "\n",
       "                                            forks_url  forks_count  \\\n",
       "6   https://git.generalassemb.ly/api/v3/repos/Data...            2   \n",
       "14  https://git.generalassemb.ly/api/v3/repos/Data...            2   \n",
       "22  https://git.generalassemb.ly/api/v3/repos/Data...            2   \n",
       "\n",
       "            created_at  \n",
       "6  2018-04-30 18:12:39  \n",
       "14 2018-05-10 18:36:37  \n",
       "22 2018-05-10 18:47:34  "
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "not_in_misk.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "not_in_misk.to_csv('./not_in_misk.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's fork."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "fork_urls = []\n",
    "with open('./finalfork.csv','r') as f:        \n",
    "     for row in f:\n",
    "         data = row.split()\n",
    "         try:\n",
    "            fork_urls.append(data[0])\n",
    "         except IndexError:\n",
    "            print('You have an empty row')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://git.generalassemb.ly/api/v3/repos/Data-science-immersive/6.02-lesson-bagging-rfs/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/Data-science-immersive/6.05-lesson-gradient_descent/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/Data-science-immersive/6.01-trees-CARTs-lesson/forks']"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fork_urls = fork_urls[1:]\n",
    "fork_urls[:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data-science-immersive/6.02-lesson-bagging-rfs successfully forked to DSI-ME-1/6.02-lesson-bagging-rfs\n",
      "Data-science-immersive/6.05-lesson-gradient_descent successfully forked to DSI-ME-1/6.05-lesson-gradient_descent\n",
      "Data-science-immersive/6.01-trees-CARTs-lesson successfully forked to DSI-ME-1/6.01-trees-CARTs-lesson\n",
      "Data-science-immersive/6.04-lesson-support_vector_machines-svm successfully forked to DSI-ME-1/6.04-lesson-support_vector_machines-svm\n",
      "Data-science-immersive/7.03-lesson-regularizing-optimizing-nn successfully forked to DSI-ME-1/7.03-lesson-regularizing-optimizing-nn\n",
      "Data-science-immersive/7.06-tensorflow successfully forked to DSI-ME-1/7.06-tensorflow\n",
      "Data-science-immersive/7.05-lesson-cnns_ii successfully forked to DSI-ME-1/7.05-lesson-cnns_ii\n",
      "Data-science-immersive/project-3 successfully forked to DSI-ME-1/project-3\n",
      "Data-science-immersive/7.01-lesson-keras successfully forked to DSI-ME-1/7.01-lesson-keras\n",
      "Data-science-immersive/7.02-lab-neural_network_applied successfully forked to DSI-ME-1/7.02-lab-neural_network_applied\n",
      "Data-science-immersive/7.01-lesson-intro_to_neural_networks-nn successfully forked to DSI-ME-1/7.01-lesson-intro_to_neural_networks-nn\n",
      "Data-science-immersive/7.01-lab-neural_network_conceptual successfully forked to DSI-ME-1/7.01-lab-neural_network_conceptual\n",
      "Data-science-immersive/7.04-lesson-cnn-i successfully forked to DSI-ME-1/7.04-lesson-cnn-i\n",
      "Data-science-immersive/8.05-pca-intro-lab successfully forked to DSI-ME-1/8.05-pca-intro-lab\n",
      "Data-science-immersive/8.06-lesson-recommender-systems successfully forked to DSI-ME-1/8.06-lesson-recommender-systems\n",
      "Data-science-immersive/8.02-clustering-intro_to_clustering_kmeans-lesson successfully forked to DSI-ME-1/8.02-clustering-intro_to_clustering_kmeans-lesson\n",
      "Data-science-immersive/project-4 successfully forked to DSI-ME-1/project-4\n",
      "Data-science-immersive/8.05-lesson-pca-intro-pca successfully forked to DSI-ME-1/8.05-lesson-pca-intro-pca\n",
      "Data-science-immersive/8.04-clustering-battle_of_clusterers-lab successfully forked to DSI-ME-1/8.04-clustering-battle_of_clusterers-lab\n",
      "Data-science-immersive/8.01-lesson-intro_to_unsupervised_learning_lda successfully forked to DSI-ME-1/8.01-lesson-intro_to_unsupervised_learning_lda\n",
      "Data-science-immersive/8.03-clustering-hierarchical_clustering-lesson successfully forked to DSI-ME-1/8.03-clustering-hierarchical_clustering-lesson\n",
      "Data-science-immersive/8.04-lesson-dbscan_hdbscan successfully forked to DSI-ME-1/8.04-lesson-dbscan_hdbscan\n",
      "Data-science-immersive/9.03-lesson-bayesian_inference successfully forked to DSI-ME-1/9.03-lesson-bayesian_inference\n",
      "Data-science-immersive/9.07-lesson-pymc3-bayes successfully forked to DSI-ME-1/9.07-lesson-pymc3-bayes\n",
      "Data-science-immersive/9.05-analysis-with-missing-data successfully forked to DSI-ME-1/9.05-analysis-with-missing-data\n",
      "Data-science-immersive/9.03-lab-bayes_data successfully forked to DSI-ME-1/9.03-lab-bayes_data\n",
      "Data-science-immersive/9.02-lab-prior_posterior_simulations successfully forked to DSI-ME-1/9.02-lab-prior_posterior_simulations\n",
      "Data-science-immersive/9.06-lesson-mcmc successfully forked to DSI-ME-1/9.06-lesson-mcmc\n",
      "Data-science-immersive/9.02-intro_to_bayes-lesson successfully forked to DSI-ME-1/9.02-intro_to_bayes-lesson\n",
      "Data-science-immersive/9.01-recommender-systems-ii successfully forked to DSI-ME-1/9.01-recommender-systems-ii\n",
      "Data-science-immersive/9.04-bayes-conjugacy-mle successfully forked to DSI-ME-1/9.04-bayes-conjugacy-mle\n",
      "Data-science-immersive/9.01-lab-bayes_practice successfully forked to DSI-ME-1/9.01-lab-bayes_practice\n",
      "Data-science-immersive/11.05-lesson-spark-ml successfully forked to DSI-ME-1/11.05-lesson-spark-ml\n",
      "Data-science-immersive/11.01-lab-spark-model successfully forked to DSI-ME-1/11.01-lab-spark-model\n",
      "Data-science-immersive/10.01-lesson-intro_to_correlated_data successfully forked to DSI-ME-1/10.01-lesson-intro_to_correlated_data\n",
      "Data-science-immersive/10.04-lesson-arima_aic_forecasting-timeseries successfully forked to DSI-ME-1/10.04-lesson-arima_aic_forecasting-timeseries\n",
      "Data-science-immersive/10.04-timeseries-arima-lab successfully forked to DSI-ME-1/10.04-timeseries-arima-lab\n",
      "Data-science-immersive/10.06-lesson-benfords-law successfully forked to DSI-ME-1/10.06-lesson-benfords-law\n",
      "Data-science-immersive/10.05-lesson-timeseries_forecasting_lstms successfully forked to DSI-ME-1/10.05-lesson-timeseries_forecasting_lstms\n",
      "Data-science-immersive/10.07-lesson-spatial_data_analysis successfully forked to DSI-ME-1/10.07-lesson-spatial_data_analysis\n",
      "Data-science-immersive/11.06-lesson-spark-pipelines successfully forked to DSI-ME-1/11.06-lesson-spark-pipelines\n",
      "Data-science-immersive/10.07-spatial-lab successfully forked to DSI-ME-1/10.07-spatial-lab\n",
      "Data-science-immersive/11.02-lesson-scala successfully forked to DSI-ME-1/11.02-lesson-scala\n",
      "Data-science-immersive/11.01-docker-on-aws successfully forked to DSI-ME-1/11.01-docker-on-aws\n",
      "Data-science-immersive/11.07-outcomes-interview_prep successfully forked to DSI-ME-1/11.07-outcomes-interview_prep\n",
      "Data-science-immersive/11.04-lesson-explainable_models-ethics successfully forked to DSI-ME-1/11.04-lesson-explainable_models-ethics\n",
      "Data-science-immersive/10.03-lesson-arma-timeseries successfully forked to DSI-ME-1/10.03-lesson-arma-timeseries\n",
      "Data-science-immersive/11.03-lesson-dataframes successfully forked to DSI-ME-1/11.03-lesson-dataframes\n",
      "Data-science-immersive/capstone successfully forked to DSI-ME-1/capstone\n"
     ]
    }
   ],
   "source": [
    "# set the target org parameter, where the forks will go\n",
    "data = { 'organization': 'DSI-ME-1' }\n",
    "\n",
    "with requests.Session() as s:\n",
    "    # this get request authenticates the user\n",
    "    r1 = s.get('https://{}/api/v3'.format(baseurl), headers=header)\n",
    "    # ...and then this post request creates the fork\n",
    "    for fork_url in fork_urls:\n",
    "        r2 = s.post(fork_url, headers=header, data=json.dumps(data))\n",
    "        print('{} successfully forked to {}'.format( \\\n",
    "            fork_url.split('/')[-3] + '/' + fork_url.split('/')[-2], \\\n",
    "            json.loads(r2.text)['full_name'])\n",
    "        )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Code below will clone an entire org's repos to a local folder; use with parseme.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "templates, git@git.generalassemb.ly:DSI-ME-1/templates.git\n",
      "python-101-lesson, git@git.generalassemb.ly:DSI-ME-1/python-101-lesson.git\n",
      "python-comprehensions-lab, git@git.generalassemb.ly:DSI-ME-1/python-comprehensions-lab.git\n",
      "python-comprehensions-lesson, git@git.generalassemb.ly:DSI-ME-1/python-comprehensions-lesson.git\n",
      "python-functions-lesson, git@git.generalassemb.ly:DSI-ME-1/python-functions-lesson.git\n"
     ]
    }
   ],
   "source": [
    "'''\n",
    "NEEDS CODE WRITTEN TO TRAVERSE PAGES (start at page 1, go until r.status_code = 202)\n",
    "see docs here:\n",
    "https://developer.github.com/v3/repos/#parameters-2\n",
    "\n",
    "Note: the 'page' parameter is not documented with the github API, I found it by trial/error\n",
    "'''\n",
    "\n",
    "# get a listing of all repos in the target org, 'Data-science-immersive'\n",
    "# from GHE docs: GET /orgs/:org/repos\n",
    "\n",
    "r = requests.get('https://{}/api/v3/orgs/{}/repos'.format(baseurl, 'DSI-ME-1'), \\\n",
    "                 headers=header, params={ 'type': 'all' , 'page': 1 })\n",
    "repos = json.loads(r.content)\n",
    "\n",
    "clones = []\n",
    "for repo in repos:\n",
    "    clones.append(repo['ssh_url'])\n",
    "    \n",
    "# show first 5\n",
    "for repo in repos[:5]:\n",
    "    print('{}, {}'.format(repo['name'], repo['ssh_url']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'id': 134818,\n",
       " 'name': 'templates',\n",
       " 'full_name': 'DSI-ME-1/templates',\n",
       " 'owner': {'login': 'DSI-ME-1',\n",
       "  'id': 11622,\n",
       "  'avatar_url': 'https://avatars.git.generalassemb.ly/u/11622?',\n",
       "  'gravatar_id': '',\n",
       "  'url': 'https://git.generalassemb.ly/api/v3/users/DSI-ME-1',\n",
       "  'html_url': 'https://git.generalassemb.ly/DSI-ME-1',\n",
       "  'followers_url': 'https://git.generalassemb.ly/api/v3/users/DSI-ME-1/followers',\n",
       "  'following_url': 'https://git.generalassemb.ly/api/v3/users/DSI-ME-1/following{/other_user}',\n",
       "  'gists_url': 'https://git.generalassemb.ly/api/v3/users/DSI-ME-1/gists{/gist_id}',\n",
       "  'starred_url': 'https://git.generalassemb.ly/api/v3/users/DSI-ME-1/starred{/owner}{/repo}',\n",
       "  'subscriptions_url': 'https://git.generalassemb.ly/api/v3/users/DSI-ME-1/subscriptions',\n",
       "  'organizations_url': 'https://git.generalassemb.ly/api/v3/users/DSI-ME-1/orgs',\n",
       "  'repos_url': 'https://git.generalassemb.ly/api/v3/users/DSI-ME-1/repos',\n",
       "  'events_url': 'https://git.generalassemb.ly/api/v3/users/DSI-ME-1/events{/privacy}',\n",
       "  'received_events_url': 'https://git.generalassemb.ly/api/v3/users/DSI-ME-1/received_events',\n",
       "  'type': 'Organization',\n",
       "  'site_admin': False},\n",
       " 'private': True,\n",
       " 'html_url': 'https://git.generalassemb.ly/DSI-ME-1/templates',\n",
       " 'description': 'Templates for readme files, notebooks, and other materials used in course lessons, labs, projects, and homework assignments.',\n",
       " 'fork': True,\n",
       " 'url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates',\n",
       " 'forks_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/forks',\n",
       " 'keys_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/keys{/key_id}',\n",
       " 'collaborators_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/collaborators{/collaborator}',\n",
       " 'teams_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/teams',\n",
       " 'hooks_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/hooks',\n",
       " 'issue_events_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/issues/events{/number}',\n",
       " 'events_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/events',\n",
       " 'assignees_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/assignees{/user}',\n",
       " 'branches_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/branches{/branch}',\n",
       " 'tags_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/tags',\n",
       " 'blobs_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/git/blobs{/sha}',\n",
       " 'git_tags_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/git/tags{/sha}',\n",
       " 'git_refs_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/git/refs{/sha}',\n",
       " 'trees_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/git/trees{/sha}',\n",
       " 'statuses_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/statuses/{sha}',\n",
       " 'languages_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/languages',\n",
       " 'stargazers_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/stargazers',\n",
       " 'contributors_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/contributors',\n",
       " 'subscribers_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/subscribers',\n",
       " 'subscription_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/subscription',\n",
       " 'commits_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/commits{/sha}',\n",
       " 'git_commits_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/git/commits{/sha}',\n",
       " 'comments_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/comments{/number}',\n",
       " 'issue_comment_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/issues/comments{/number}',\n",
       " 'contents_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/contents/{+path}',\n",
       " 'compare_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/compare/{base}...{head}',\n",
       " 'merges_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/merges',\n",
       " 'archive_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/{archive_format}{/ref}',\n",
       " 'downloads_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/downloads',\n",
       " 'issues_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/issues{/number}',\n",
       " 'pulls_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/pulls{/number}',\n",
       " 'milestones_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/milestones{/number}',\n",
       " 'notifications_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/notifications{?since,all,participating}',\n",
       " 'labels_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/labels{/name}',\n",
       " 'releases_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/releases{/id}',\n",
       " 'deployments_url': 'https://git.generalassemb.ly/api/v3/repos/DSI-ME-1/templates/deployments',\n",
       " 'created_at': '2018-05-28T22:55:55Z',\n",
       " 'updated_at': '2018-05-28T22:55:57Z',\n",
       " 'pushed_at': '2018-04-25T20:28:20Z',\n",
       " 'git_url': 'git://git.generalassemb.ly/DSI-ME-1/templates.git',\n",
       " 'ssh_url': 'git@git.generalassemb.ly:DSI-ME-1/templates.git',\n",
       " 'clone_url': 'https://git.generalassemb.ly/DSI-ME-1/templates.git',\n",
       " 'svn_url': 'https://git.generalassemb.ly/DSI-ME-1/templates',\n",
       " 'homepage': None,\n",
       " 'size': 55,\n",
       " 'stargazers_count': 0,\n",
       " 'watchers_count': 0,\n",
       " 'language': 'CSS',\n",
       " 'has_issues': False,\n",
       " 'has_projects': True,\n",
       " 'has_downloads': True,\n",
       " 'has_wiki': True,\n",
       " 'has_pages': False,\n",
       " 'forks_count': 0,\n",
       " 'mirror_url': None,\n",
       " 'archived': False,\n",
       " 'open_issues_count': 0,\n",
       " 'license': {'key': 'other', 'name': 'Other', 'spdx_id': None, 'url': None},\n",
       " 'forks': 0,\n",
       " 'open_issues': 0,\n",
       " 'watchers': 0,\n",
       " 'default_branch': 'master',\n",
       " 'permissions': {'admin': True, 'push': True, 'pull': True}}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "repos[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(clones)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a directory for the cloned repos to live"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nThis will create a place for all the cloned repos to live.\\nUse the parseme file, pointing to this directory, to\\nscan all clones repos for words\\n'"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "This will create a place for all the cloned repos to live.\n",
    "Use the parseme file, pointing to this directory, to\n",
    "scan all clones repos for words\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "created directory ./course_dump\n"
     ]
    }
   ],
   "source": [
    "%%bash\n",
    "if [ ! -d \"./course_dump\" ]; then\n",
    "    mkdir ./course_dump\n",
    "    echo \"created directory ./course_dump\"\n",
    "else\n",
    "    echo \"directory already exists: ./course_dump\"\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning git@git.generalassemb.ly:DSI-ME-1/10.02-lesson-timeseries.git into ./course_dump...\n",
      "Cloning into '10.02-lesson-timeseries'...\n",
      "remote: Counting objects: 43, done.\u001b[K\n",
      "remote: Total 43 (delta 0), reused 0 (delta 0), pack-reused 43\u001b[K\n",
      "Receiving objects: 100% (43/43), 924.38 KiB | 50.00 KiB/s, done.\n",
      "Resolving deltas: 100% (17/17), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/3.04-regression-metrics-lesson.git into ./course_dump...\n",
      "Cloning into '3.04-regression-metrics-lesson'...\n",
      "remote: Counting objects: 20, done.\u001b[K\n",
      "remote: Total 20 (delta 0), reused 0 (delta 0), pack-reused 20\u001b[K\n",
      "Receiving objects: 100% (20/20), 16.20 KiB | 281.00 KiB/s, done.\n",
      "Resolving deltas: 100% (6/6), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/6.02-lesson-bagging-rfs.git into ./course_dump...\n",
      "Cloning into '6.02-lesson-bagging-rfs'...\n",
      "remote: Counting objects: 36, done.\u001b[K\n",
      "remote: Compressing objects: 100% (22/22), done.\u001b[K\n",
      "remote: Total 36 (delta 11), reused 36 (delta 11), pack-reused 0\u001b[K\n",
      "Receiving objects: 100% (36/36), 401.54 KiB | 304.00 KiB/s, done.\n",
      "Resolving deltas: 100% (11/11), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/6.05-lesson-gradient_descent.git into ./course_dump...\n",
      "Cloning into '6.05-lesson-gradient_descent'...\n",
      "remote: Counting objects: 16, done.\u001b[K\n",
      "remote: Compressing objects: 100% (11/11), done.\u001b[K\n",
      "remote: Total 16 (delta 3), reused 16 (delta 3), pack-reused 0\u001b[K\n",
      "Receiving objects: 100% (16/16), 2.12 MiB | 404.00 KiB/s, done.\n",
      "Resolving deltas: 100% (3/3), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/6.01-trees-CARTs-lesson.git into ./course_dump...\n",
      "Cloning into '6.01-trees-CARTs-lesson'...\n",
      "remote: Counting objects: 45, done.\u001b[K\n",
      "remote: Compressing objects: 100% (30/30), done.\u001b[K\n",
      "remote: Total 45 (delta 14), reused 45 (delta 14), pack-reused 0\u001b[K\n",
      "Receiving objects: 100% (45/45), 3.11 MiB | 368.00 KiB/s, done.\n",
      "Resolving deltas: 100% (14/14), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/6.04-lesson-support_vector_machines-svm.git into ./course_dump...\n",
      "Cloning into '6.04-lesson-support_vector_machines-svm'...\n",
      "remote: Counting objects: 78, done.\u001b[K\n",
      "remote: Compressing objects: 100% (50/50), done.\u001b[K\n",
      "remote: Total 78 (delta 26), reused 78 (delta 26), pack-reused 0\u001b[K\n",
      "Receiving objects: 100% (78/78), 9.20 MiB | 422.00 KiB/s, done.\n",
      "Resolving deltas: 100% (26/26), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/7.03-lesson-regularizing-optimizing-nn.git into ./course_dump...\n",
      "Cloning into '7.03-lesson-regularizing-optimizing-nn'...\n",
      "remote: Counting objects: 39, done.\u001b[K\n",
      "remote: Total 39 (delta 0), reused 0 (delta 0), pack-reused 39\u001b[K\n",
      "Receiving objects: 100% (39/39), 634.77 KiB | 425.00 KiB/s, done.\n",
      "Resolving deltas: 100% (7/7), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/7.06-tensorflow.git into ./course_dump...\n",
      "Cloning into '7.06-tensorflow'...\n",
      "remote: Counting objects: 14, done.\u001b[K\n",
      "remote: Total 14 (delta 0), reused 0 (delta 0), pack-reused 14\u001b[K\n",
      "Receiving objects: 100% (14/14), 128.60 KiB | 562.00 KiB/s, done.\n",
      "Resolving deltas: 100% (2/2), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/7.05-lesson-cnns_ii.git into ./course_dump...\n",
      "Cloning into '7.05-lesson-cnns_ii'...\n",
      "remote: Counting objects: 9, done.\u001b[K\n",
      "remote: Total 9 (delta 0), reused 0 (delta 0), pack-reused 9\u001b[K\n",
      "Receiving objects: 100% (9/9), 55.33 KiB | 928.00 KiB/s, done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/project-3.git into ./course_dump...\n",
      "Cloning into 'project-3'...\n",
      "remote: Counting objects: 59, done.\u001b[K\n",
      "remote: Total 59 (delta 0), reused 0 (delta 0), pack-reused 59\u001b[K\n",
      "Receiving objects: 100% (59/59), 27.10 KiB | 486.00 KiB/s, done.\n",
      "Resolving deltas: 100% (20/20), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/7.01-lesson-keras.git into ./course_dump...\n",
      "Cloning into '7.01-lesson-keras'...\n",
      "remote: Counting objects: 8, done.\u001b[K\n",
      "remote: Compressing objects: 100% (6/6), done.\u001b[K\n",
      "remote: Total 8 (delta 2), reused 8 (delta 2), pack-reused 0\u001b[K\n",
      "Receiving objects: 100% (8/8), 42.42 KiB | 493.00 KiB/s, done.\n",
      "Resolving deltas: 100% (2/2), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/7.02-lab-neural_network_applied.git into ./course_dump...\n",
      "Cloning into '7.02-lab-neural_network_applied'...\n",
      "remote: Counting objects: 7, done.\u001b[K\n",
      "remote: Total 7 (delta 0), reused 0 (delta 0), pack-reused 7\u001b[K\n",
      "Receiving objects: 100% (7/7), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/7.01-lesson-intro_to_neural_networks-nn.git into ./course_dump...\n",
      "Cloning into '7.01-lesson-intro_to_neural_networks-nn'...\n",
      "remote: Counting objects: 32, done.\u001b[K\n",
      "remote: Compressing objects: 100% (22/22), done.\u001b[K\n",
      "remote: Total 32 (delta 8), reused 32 (delta 8), pack-reused 0\u001b[K\n",
      "Receiving objects: 100% (32/32), 1.35 MiB | 431.00 KiB/s, done.\n",
      "Resolving deltas: 100% (8/8), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/7.01-lab-neural_network_conceptual.git into ./course_dump...\n",
      "Cloning into '7.01-lab-neural_network_conceptual'...\n",
      "remote: Counting objects: 13, done.\u001b[K\n",
      "remote: Total 13 (delta 0), reused 0 (delta 0), pack-reused 13\u001b[K\n",
      "Receiving objects: 100% (13/13), done.\n",
      "Resolving deltas: 100% (1/1), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/7.04-lesson-cnn-i.git into ./course_dump...\n",
      "Cloning into '7.04-lesson-cnn-i'...\n",
      "remote: Counting objects: 25, done.\u001b[K\n",
      "remote: Total 25 (delta 0), reused 0 (delta 0), pack-reused 25\u001b[K\n",
      "Receiving objects: 100% (25/25), 39.24 MiB | 423.00 KiB/s, done.\n",
      "Resolving deltas: 100% (8/8), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/8.05-pca-intro-lab.git into ./course_dump...\n",
      "Cloning into '8.05-pca-intro-lab'...\n",
      "remote: Counting objects: 28, done.\u001b[K\n",
      "remote: Total 28 (delta 0), reused 0 (delta 0), pack-reused 28\u001b[K\n",
      "Receiving objects: 100% (28/28), 3.75 MiB | 427.00 KiB/s, done.\n",
      "Resolving deltas: 100% (10/10), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/8.06-lesson-recommender-systems.git into ./course_dump...\n",
      "Cloning into '8.06-lesson-recommender-systems'...\n",
      "remote: Counting objects: 30, done.\u001b[K\n",
      "remote: Total 30 (delta 0), reused 0 (delta 0), pack-reused 30\u001b[K\n",
      "Receiving objects: 100% (30/30), 30.59 MiB | 426.00 KiB/s, done.\n",
      "Resolving deltas: 100% (8/8), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/8.02-clustering-intro_to_clustering_kmeans-lesson.git into ./course_dump...\n",
      "Cloning into '8.02-clustering-intro_to_clustering_kmeans-lesson'...\n",
      "remote: Counting objects: 44, done.\u001b[K\n",
      "remote: Total 44 (delta 0), reused 0 (delta 0), pack-reused 44\u001b[K\n",
      "Receiving objects: 100% (44/44), 1.85 MiB | 435.00 KiB/s, done.\n",
      "Resolving deltas: 100% (19/19), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/project-4.git into ./course_dump...\n",
      "Cloning into 'project-4'...\n",
      "remote: Counting objects: 21, done.\u001b[K\n",
      "remote: Total 21 (delta 0), reused 0 (delta 0), pack-reused 21\u001b[K\n",
      "Receiving objects: 100% (21/21), 7.48 KiB | 3.74 MiB/s, done.\n",
      "Resolving deltas: 100% (5/5), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/8.05-lesson-pca-intro-pca.git into ./course_dump...\n",
      "Cloning into '8.05-lesson-pca-intro-pca'...\n",
      "remote: Counting objects: 51, done.\u001b[K\n",
      "remote: Total 51 (delta 0), reused 0 (delta 0), pack-reused 51\u001b[K\n",
      "Receiving objects: 100% (51/51), 1.77 MiB | 431.00 KiB/s, done.\n",
      "Resolving deltas: 100% (21/21), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/8.04-clustering-battle_of_clusterers-lab.git into ./course_dump...\n",
      "Cloning into '8.04-clustering-battle_of_clusterers-lab'...\n",
      "remote: Counting objects: 39, done.\u001b[K\n",
      "remote: Total 39 (delta 0), reused 0 (delta 0), pack-reused 39\u001b[K\n",
      "Receiving objects: 100% (39/39), 2.82 MiB | 428.00 KiB/s, done.\n",
      "Resolving deltas: 100% (8/8), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/8.01-lesson-intro_to_unsupervised_learning_lda.git into ./course_dump...\n",
      "Cloning into '8.01-lesson-intro_to_unsupervised_learning_lda'...\n",
      "remote: Counting objects: 27, done.\u001b[K\n",
      "remote: Total 27 (delta 0), reused 0 (delta 0), pack-reused 27\u001b[K\n",
      "Receiving objects: 100% (27/27), 88.19 MiB | 295.00 KiB/s, done.\n",
      "Resolving deltas: 100% (7/7), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/8.03-clustering-hierarchical_clustering-lesson.git into ./course_dump...\n",
      "Cloning into '8.03-clustering-hierarchical_clustering-lesson'...\n",
      "remote: Counting objects: 37, done.\u001b[K\n",
      "remote: Total 37 (delta 0), reused 0 (delta 0), pack-reused 37\u001b[K\n",
      "Receiving objects: 100% (37/37), 3.15 MiB | 253.00 KiB/s, done.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Resolving deltas: 100% (8/8), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/8.04-lesson-dbscan_hdbscan.git into ./course_dump...\n",
      "Cloning into '8.04-lesson-dbscan_hdbscan'...\n",
      "remote: Counting objects: 18, done.\u001b[K\n",
      "remote: Total 18 (delta 0), reused 0 (delta 0), pack-reused 18\u001b[K\n",
      "Receiving objects: 100% (18/18), 13.19 MiB | 378.00 KiB/s, done.\n",
      "Resolving deltas: 100% (4/4), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/9.03-lesson-bayesian_inference.git into ./course_dump...\n",
      "Cloning into '9.03-lesson-bayesian_inference'...\n",
      "remote: Counting objects: 10, done.\u001b[K\n",
      "remote: Total 10 (delta 0), reused 0 (delta 0), pack-reused 10\u001b[K\n",
      "Receiving objects: 100% (10/10), 19.56 MiB | 381.00 KiB/s, done.\n",
      "Resolving deltas: 100% (1/1), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/9.07-lesson-pymc3-bayes.git into ./course_dump...\n",
      "Cloning into '9.07-lesson-pymc3-bayes'...\n",
      "remote: Counting objects: 69, done.\u001b[K\n",
      "remote: Total 69 (delta 0), reused 0 (delta 0), pack-reused 69\u001b[K\n",
      "Receiving objects: 100% (69/69), 5.75 MiB | 379.00 KiB/s, done.\n",
      "Resolving deltas: 100% (24/24), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/9.05-analysis-with-missing-data.git into ./course_dump...\n",
      "Cloning into '9.05-analysis-with-missing-data'...\n",
      "remote: Counting objects: 11, done.\u001b[K\n",
      "remote: Total 11 (delta 0), reused 0 (delta 0), pack-reused 11\u001b[K\n",
      "Receiving objects: 100% (11/11), 7.20 MiB | 369.00 KiB/s, done.\n",
      "Resolving deltas: 100% (2/2), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/9.03-lab-bayes_data.git into ./course_dump...\n",
      "Cloning into '9.03-lab-bayes_data'...\n",
      "remote: Counting objects: 14, done.\u001b[K\n",
      "remote: Total 14 (delta 0), reused 0 (delta 0), pack-reused 14\u001b[K\n",
      "Receiving objects: 100% (14/14), 212.50 KiB | 366.00 KiB/s, done.\n",
      "Resolving deltas: 100% (2/2), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/9.02-lab-prior_posterior_simulations.git into ./course_dump...\n",
      "Cloning into '9.02-lab-prior_posterior_simulations'...\n",
      "remote: Counting objects: 10, done.\u001b[K\n",
      "remote: Total 10 (delta 0), reused 0 (delta 0), pack-reused 10\u001b[K\n",
      "Receiving objects: 100% (10/10), 228.55 KiB | 297.00 KiB/s, done.\n",
      "Resolving deltas: 100% (1/1), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/9.06-lesson-mcmc.git into ./course_dump...\n",
      "Cloning into '9.06-lesson-mcmc'...\n",
      "remote: Counting objects: 30, done.\u001b[K\n",
      "remote: Total 30 (delta 0), reused 0 (delta 0), pack-reused 30\u001b[K\n",
      "Receiving objects: 100% (30/30), 21.07 MiB | 247.00 KiB/s, done.\n",
      "Resolving deltas: 100% (9/9), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/9.02-intro_to_bayes-lesson.git into ./course_dump...\n",
      "Cloning into '9.02-intro_to_bayes-lesson'...\n",
      "remote: Counting objects: 70, done.\u001b[K\n",
      "remote: Total 70 (delta 0), reused 0 (delta 0), pack-reused 70\u001b[K\n",
      "Receiving objects: 100% (70/70), 544.13 KiB | 290.00 KiB/s, done.\n",
      "Resolving deltas: 100% (37/37), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/9.01-recommender-systems-ii.git into ./course_dump...\n",
      "Cloning into '9.01-recommender-systems-ii'...\n",
      "remote: Counting objects: 42, done.\u001b[K\n",
      "remote: Total 42 (delta 0), reused 0 (delta 0), pack-reused 42\u001b[K\n",
      "Receiving objects: 100% (42/42), 21.97 MiB | 292.00 KiB/s, done.\n",
      "Resolving deltas: 100% (17/17), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/9.04-bayes-conjugacy-mle.git into ./course_dump...\n",
      "Cloning into '9.04-bayes-conjugacy-mle'...\n",
      "remote: Counting objects: 18, done.\u001b[K\n",
      "remote: Total 18 (delta 0), reused 0 (delta 0), pack-reused 18\u001b[K\n",
      "Receiving objects: 100% (18/18), 325.74 KiB | 189.00 KiB/s, done.\n",
      "Resolving deltas: 100% (5/5), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/9.01-lab-bayes_practice.git into ./course_dump...\n",
      "Cloning into '9.01-lab-bayes_practice'...\n",
      "remote: Counting objects: 10, done.\u001b[K\n",
      "remote: Total 10 (delta 0), reused 0 (delta 0), pack-reused 10\u001b[K\n",
      "Receiving objects: 100% (10/10), 6.02 KiB | 6.02 MiB/s, done.\n",
      "Resolving deltas: 100% (2/2), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/11.05-lesson-spark-ml.git into ./course_dump...\n",
      "Cloning into '11.05-lesson-spark-ml'...\n",
      "remote: Counting objects: 26, done.\u001b[K\n",
      "remote: Total 26 (delta 0), reused 0 (delta 0), pack-reused 26\u001b[K\n",
      "Receiving objects: 100% (26/26), 365.43 KiB | 166.00 KiB/s, done.\n",
      "Resolving deltas: 100% (7/7), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/11.01-lab-spark-model.git into ./course_dump...\n",
      "Cloning into '11.01-lab-spark-model'...\n",
      "remote: Counting objects: 15, done.\u001b[K\n",
      "remote: Total 15 (delta 0), reused 0 (delta 0), pack-reused 15\u001b[K\n",
      "Receiving objects: 100% (15/15), done.\n",
      "Resolving deltas: 100% (3/3), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/10.01-lesson-intro_to_correlated_data.git into ./course_dump...\n",
      "Cloning into '10.01-lesson-intro_to_correlated_data'...\n",
      "remote: Counting objects: 12, done.\u001b[K\n",
      "remote: Total 12 (delta 0), reused 0 (delta 0), pack-reused 12\u001b[K\n",
      "Receiving objects: 100% (12/12), 24.45 MiB | 304.00 KiB/s, done.\n",
      "Resolving deltas: 100% (3/3), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/10.04-lesson-arima_aic_forecasting-timeseries.git into ./course_dump...\n",
      "Cloning into '10.04-lesson-arima_aic_forecasting-timeseries'...\n",
      "remote: Counting objects: 51, done.\u001b[K\n",
      "remote: Total 51 (delta 0), reused 0 (delta 0), pack-reused 51\u001b[K\n",
      "Receiving objects: 100% (51/51), 7.05 MiB | 262.00 KiB/s, done.\n",
      "Resolving deltas: 100% (16/16), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/10.04-timeseries-arima-lab.git into ./course_dump...\n",
      "Cloning into '10.04-timeseries-arima-lab'...\n",
      "remote: Counting objects: 26, done.\u001b[K\n",
      "remote: Total 26 (delta 0), reused 0 (delta 0), pack-reused 26\u001b[K\n",
      "Receiving objects: 100% (26/26), 3.28 MiB | 263.00 KiB/s, done.\n",
      "Resolving deltas: 100% (8/8), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/10.06-lesson-benfords-law.git into ./course_dump...\n",
      "Cloning into '10.06-lesson-benfords-law'...\n",
      "remote: Counting objects: 10, done.\u001b[K\n",
      "remote: Total 10 (delta 0), reused 0 (delta 0), pack-reused 10\u001b[K\n",
      "Receiving objects: 100% (10/10), 8.00 MiB | 272.00 KiB/s, done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/10.05-lesson-timeseries_forecasting_lstms.git into ./course_dump...\n",
      "Cloning into '10.05-lesson-timeseries_forecasting_lstms'...\n",
      "remote: Counting objects: 21, done.\u001b[K\n",
      "remote: Total 21 (delta 0), reused 0 (delta 0), pack-reused 21\u001b[K\n",
      "Receiving objects: 100% (21/21), 1.73 MiB | 367.00 KiB/s, done.\n",
      "Resolving deltas: 100% (5/5), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/10.07-lesson-spatial_data_analysis.git into ./course_dump...\n",
      "Cloning into '10.07-lesson-spatial_data_analysis'...\n",
      "remote: Counting objects: 30, done.\u001b[K\n",
      "remote: Total 30 (delta 0), reused 0 (delta 0), pack-reused 30\u001b[K\n",
      "Receiving objects: 100% (30/30), 26.12 MiB | 384.00 KiB/s, done.\n",
      "Resolving deltas: 100% (7/7), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/11.06-lesson-spark-pipelines.git into ./course_dump...\n",
      "Cloning into '11.06-lesson-spark-pipelines'...\n",
      "remote: Counting objects: 17, done.\u001b[K\n",
      "remote: Total 17 (delta 0), reused 0 (delta 0), pack-reused 17\u001b[K\n",
      "Receiving objects: 100% (17/17), 370.16 KiB | 442.00 KiB/s, done.\n",
      "Resolving deltas: 100% (3/3), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/10.07-spatial-lab.git into ./course_dump...\n",
      "Cloning into '10.07-spatial-lab'...\n",
      "remote: Counting objects: 22, done.\u001b[K\n",
      "remote: Total 22 (delta 0), reused 0 (delta 0), pack-reused 22\u001b[K\n",
      "Receiving objects: 100% (22/22), 1.32 MiB | 260.00 KiB/s, done.\n",
      "Resolving deltas: 100% (6/6), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/11.02-lesson-scala.git into ./course_dump...\n",
      "Cloning into '11.02-lesson-scala'...\n",
      "remote: Counting objects: 20, done.\u001b[K\n",
      "remote: Total 20 (delta 0), reused 0 (delta 0), pack-reused 20\u001b[K\n",
      "Receiving objects: 100% (20/20), 252.29 KiB | 279.00 KiB/s, done.\n",
      "Resolving deltas: 100% (6/6), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/11.01-docker-on-aws.git into ./course_dump...\n",
      "Cloning into '11.01-docker-on-aws'...\n",
      "remote: Counting objects: 6, done.\u001b[K\n",
      "remote: Total 6 (delta 0), reused 0 (delta 0), pack-reused 6\u001b[K\n",
      "Receiving objects: 100% (6/6), 5.56 KiB | 5.56 MiB/s, done.\n",
      "Resolving deltas: 100% (1/1), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/11.07-outcomes-interview_prep.git into ./course_dump...\n",
      "Cloning into '11.07-outcomes-interview_prep'...\n",
      "remote: Counting objects: 13, done.\u001b[K\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "remote: Total 13 (delta 0), reused 0 (delta 0), pack-reused 13\u001b[K\n",
      "Receiving objects: 100% (13/13), 5.35 MiB | 402.00 KiB/s, done.\n",
      "Resolving deltas: 100% (1/1), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/11.04-lesson-explainable_models-ethics.git into ./course_dump...\n",
      "Cloning into '11.04-lesson-explainable_models-ethics'...\n",
      "remote: Counting objects: 61, done.\u001b[K\n",
      "remote: Total 61 (delta 0), reused 0 (delta 0), pack-reused 61\u001b[K\n",
      "Receiving objects: 100% (61/61), 7.32 MiB | 382.00 KiB/s, done.\n",
      "Resolving deltas: 100% (22/22), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/10.03-lesson-arma-timeseries.git into ./course_dump...\n",
      "Cloning into '10.03-lesson-arma-timeseries'...\n",
      "remote: Counting objects: 31, done.\u001b[K\n",
      "remote: Total 31 (delta 0), reused 0 (delta 0), pack-reused 31\u001b[K\n",
      "Receiving objects: 100% (31/31), 2.72 MiB | 303.00 KiB/s, done.\n",
      "Resolving deltas: 100% (12/12), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/11.03-lesson-dataframes.git into ./course_dump...\n",
      "Cloning into '11.03-lesson-dataframes'...\n",
      "remote: Counting objects: 27, done.\u001b[K\n",
      "remote: Total 27 (delta 0), reused 0 (delta 0), pack-reused 27\u001b[K\n",
      "Receiving objects: 100% (27/27), 374.28 KiB | 440.00 KiB/s, done.\n",
      "Resolving deltas: 100% (7/7), done.\n",
      "Cloning git@git.generalassemb.ly:DSI-ME-1/capstone.git into ./course_dump...\n",
      "Cloning into 'capstone'...\n",
      "remote: Counting objects: 157, done.\u001b[K\n",
      "remote: Total 157 (delta 0), reused 0 (delta 0), pack-reused 157\u001b[K\n",
      "Receiving objects: 100% (157/157), 31.70 KiB | 5.28 MiB/s, done.\n",
      "Resolving deltas: 100% (66/66), done.\n",
      "Cloned 51/51 repositories sucksessfulli\n"
     ]
    }
   ],
   "source": [
    "# WARNING!!! ===\n",
    "# this will clone ALL repositories in the organization to ./course_dump!!\n",
    "# note - has bug; will not retry cloning of repository if network fails :'(\n",
    "failed_clones = []\n",
    "for repo in clones:\n",
    "    try:\n",
    "        print(\"Cloning {} into ./course_dump...\".format(repo))\n",
    "        # made it sleepy to be a good citizen\n",
    "        !cd ./course_dump && git clone $repo && sleep 1\n",
    "    except:\n",
    "        # outstanding error handling here\n",
    "        failed_clones.append(repo)\n",
    "        print(\"noooooo!!\")\n",
    "        continue\n",
    "print('Cloned {}/{} repositories sucksessfulli'.format((len(clones)-(len(failed_clones))), len(clones)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This will fork all repositories from a source org to a target org"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_org = 'DAT-COG-1'\n",
    "source_org = 'data-part-time'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = requests.get('https://{}/api/v3/orgs/{}/repos'.format(baseurl, source_org), \\\n",
    "                 headers=header)\n",
    "repos = json.loads(r.content)\n",
    "\n",
    "fork_urls = []\n",
    "for repo in repos:\n",
    "    fork_urls.append(repo['forks_url'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://git.generalassemb.ly/api/v3/repos/data-part-time/your-development-environment/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/what-is-data-science/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/python-foundations/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/unit-1_project/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/statistics-in-python/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/experiments-hypothesis-tests/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/eda-with-pandas/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/data-visualization-in-python/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/unit-2_project/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/linear-regression/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/train-test-split-and-bias-variance/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/knn-classification/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/logistic-regression/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/unit-3_project/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/getting-data-APIs/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/natural-language-processing/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/decision-trees/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/unit-4_project/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/instructor-materials/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/fundamentals-review/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/working-with-data-review/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/data-modeling-review/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/applications-review/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/flex_data-science-trends/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/flex_clustering/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/course-info/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/flex_time-series/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/flex_deploying-models/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/unit-project-solutions/forks',\n",
       " 'https://git.generalassemb.ly/api/v3/repos/data-part-time/flex_naive-bayes/forks']"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fork_urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data-part-time/your-development-environment successfully forked to DAT-COG-1/your-development-environment\n",
      "data-part-time/what-is-data-science successfully forked to DAT-COG-1/what-is-data-science\n",
      "data-part-time/python-foundations successfully forked to DAT-COG-1/python-foundations\n",
      "data-part-time/unit-1_project successfully forked to DAT-COG-1/unit-1_project\n",
      "data-part-time/statistics-in-python successfully forked to DAT-COG-1/statistics-in-python\n",
      "data-part-time/experiments-hypothesis-tests successfully forked to DAT-COG-1/experiments-hypothesis-tests\n",
      "data-part-time/eda-with-pandas successfully forked to DAT-COG-1/eda-with-pandas\n",
      "data-part-time/data-visualization-in-python successfully forked to DAT-COG-1/data-visualization-in-python\n",
      "data-part-time/unit-2_project successfully forked to DAT-COG-1/unit-2_project\n",
      "data-part-time/linear-regression successfully forked to DAT-COG-1/linear-regression\n",
      "data-part-time/train-test-split-and-bias-variance successfully forked to DAT-COG-1/train-test-split-and-bias-variance\n",
      "data-part-time/knn-classification successfully forked to DAT-COG-1/knn-classification\n",
      "data-part-time/logistic-regression successfully forked to DAT-COG-1/logistic-regression\n",
      "data-part-time/unit-3_project successfully forked to DAT-COG-1/unit-3_project\n",
      "data-part-time/getting-data-APIs successfully forked to DAT-COG-1/getting-data-APIs\n",
      "data-part-time/natural-language-processing successfully forked to DAT-COG-1/natural-language-processing\n",
      "data-part-time/decision-trees successfully forked to DAT-COG-1/decision-trees\n",
      "data-part-time/unit-4_project successfully forked to DAT-COG-1/unit-4_project\n",
      "data-part-time/instructor-materials successfully forked to DAT-COG-1/instructor-materials\n",
      "data-part-time/fundamentals-review successfully forked to DAT-COG-1/fundamentals-review\n",
      "data-part-time/working-with-data-review successfully forked to DAT-COG-1/working-with-data-review\n",
      "data-part-time/data-modeling-review successfully forked to DAT-COG-1/data-modeling-review\n",
      "data-part-time/applications-review successfully forked to DAT-COG-1/applications-review\n",
      "data-part-time/flex_data-science-trends successfully forked to DAT-COG-1/flex_data-science-trends\n",
      "data-part-time/flex_clustering successfully forked to DAT-COG-1/flex_clustering\n",
      "data-part-time/course-info successfully forked to DAT-COG-1/course-info\n",
      "data-part-time/flex_time-series successfully forked to DAT-COG-1/flex_time-series\n",
      "data-part-time/flex_deploying-models successfully forked to DAT-COG-1/flex_deploying-models\n",
      "data-part-time/unit-project-solutions successfully forked to DAT-COG-1/unit-project-solutions\n",
      "data-part-time/flex_naive-bayes successfully forked to DAT-COG-1/flex_naive-bayes\n"
     ]
    }
   ],
   "source": [
    "# set the target org parameter, where the forks will go\n",
    "data = { 'organization': target_org }\n",
    "\n",
    "'''\n",
    "This needs to be done as a session because the GET request does the\n",
    "auth, and the POST request does the forking. You CAN'T pass a cookie\n",
    "between the two sessions, I tried. Couldn't find any examples of anybody\n",
    "doing this on SO, etc. Was able to get it working with CURL but wanted\n",
    "to use python requests lib. Trick is passing the header with BOTH the \n",
    "GET *AND* the POST requests. You can't just send the header with the \n",
    "POST request, despite what the docs lead you to believe.\n",
    "\n",
    "School of hard knocks... damn it feels good to be a gangster\n",
    "'''\n",
    "\n",
    "with requests.Session() as s:\n",
    "    # this get request authenticates the user\n",
    "    r1 = s.get('https://{}/api/v3'.format(baseurl), headers=header)\n",
    "    # ...and then this post request creates the fork\n",
    "    for fork_url in fork_urls:\n",
    "        r2 = s.post(fork_url, headers=header, data=json.dumps(data))\n",
    "        print('{} successfully forked to {}'.format( \\\n",
    "            fork_url.split('/')[-3] + '/' + fork_url.split('/')[-2], \\\n",
    "            json.loads(r2.text)['full_name'])\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "failed_clones"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}