MeerCRAB-prediction-phase.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MeerCRAB prediction phase on new candidate files\n",
    "This script will be integrated in BlackBOX to make prediction on new candidate files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "//anaconda/envs/tflearn/lib/python3.5/importlib/_bootstrap.py:222: RuntimeWarning: compiletime version 3.6 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.5\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from meerCRAB_code.model import compile_model,model_save \n",
    "import matplotlib.pylab as plt\n",
    "from keras.utils import np_utils\n",
    "from time import gmtime, strftime\n",
    "from meerCRAB_code.util import makedirs, ensure_dir\n",
    "from meerCRAB_code.prediction_phase import load_new_candidate, realbogus_prediction\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load csv file\n",
    "\n",
    "Csv file having this format - Each row has 24 columns separated by semi-colons.\n",
    "\n",
    "In order they are:\n",
    "\n",
    "- transientid: ID of souce in DB\n",
    "- username:\n",
    "- vettingdate:\n",
    "- vetclas: can be either real, bogus, bogus_cosmicray, bogus_subtract, \n",
    "- bogus_spike or bogus_ghost\n",
    "- number: number of source in orig. FITS file\n",
    "- image: ID of image/FITS file in DB\n",
    "- date-obs:\n",
    "- filter:\n",
    "- object: the MeerLICHT/BlackGEM tile of observation\n",
    "- psf-fwhm:\n",
    "- s-seeing:\n",
    "- s-seestd:\n",
    "- x_peak: integer x position (no python index) of peak in Scorr image\n",
    "- y_peak: idem y\n",
    "- ra_peak: corresponding ra [degrees]\n",
    "- dec_peak: corresponding dec [degrees]\n",
    "- flux_peak: corresponding calibrated flux [microJy]\n",
    "- fluxerr_peak: flux uncertainty [microJy]\n",
    "- mag_peak: corresponding calibrated magnitude [AB magn.]\n",
    "- magerr_peak: magn. uncertainty [AB magn.]\n",
    "- thumbnail_red: 100x100 thumbnail\n",
    "- thumbnail_ref:\n",
    "- thumbnail_d:\n",
    "- thumbnail_scorr:\n",
    "\n",
    "Notice that the thumbnails are 2D numpy arrays of 32bit floats, and are \n",
    "written as binary large objects (BLOBs). \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>14</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9184</td>\n",
       "      <td>nblago</td>\n",
       "      <td>2019-11-05 20:23:46.518000</td>\n",
       "      <td>bogus</td>\n",
       "      <td>395</td>\n",
       "      <td>37</td>\n",
       "      <td>2017-09-01 18:30:00.000000</td>\n",
       "      <td>q</td>\n",
       "      <td>90005</td>\n",
       "      <td>3.056144</td>\n",
       "      <td>...</td>\n",
       "      <td>234.133234</td>\n",
       "      <td>-14.702009</td>\n",
       "      <td>46.362450</td>\n",
       "      <td>7.713804</td>\n",
       "      <td>19.734650</td>\n",
       "      <td>0.180645</td>\n",
       "      <td>A0E80BC240CF3AC200F8BD3F608BD441706D224200001D...</td>\n",
       "      <td>F06705C280BFE3C170B10F42403909C2401746C1A041DB...</td>\n",
       "      <td>575FDB406D7D9AC2D85514420177743F4F446140A113A9...</td>\n",
       "      <td>BBA2273F32826E3F899BA63FD24CE83FD54E1040BDFA0C...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9210</td>\n",
       "      <td>nblago</td>\n",
       "      <td>2019-11-05 20:24:02.033000</td>\n",
       "      <td>bogus</td>\n",
       "      <td>515</td>\n",
       "      <td>37</td>\n",
       "      <td>2017-09-01 18:30:00.000000</td>\n",
       "      <td>q</td>\n",
       "      <td>90005</td>\n",
       "      <td>3.056144</td>\n",
       "      <td>...</td>\n",
       "      <td>233.819094</td>\n",
       "      <td>-14.612320</td>\n",
       "      <td>50.849087</td>\n",
       "      <td>7.740949</td>\n",
       "      <td>19.634357</td>\n",
       "      <td>0.165286</td>\n",
       "      <td>6068E3C140616341C090AAC1005CE5C010410DC2300F0A...</td>\n",
       "      <td>00B2704060C559C280582641A04045C2803FBC4000ED23...</td>\n",
       "      <td>0484EB41AC3D82C154F6C541EFCA96407E925EC1281912...</td>\n",
       "      <td>FC739F3F65BA8C3F90CF333FF7710D3EBE6F68BE19C2CD...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>12110</td>\n",
       "      <td>nblago</td>\n",
       "      <td>2019-11-05 20:24:24.389000</td>\n",
       "      <td>real</td>\n",
       "      <td>1295</td>\n",
       "      <td>47</td>\n",
       "      <td>2017-09-01 19:17:50.000000</td>\n",
       "      <td>q</td>\n",
       "      <td>90004</td>\n",
       "      <td>3.398105</td>\n",
       "      <td>...</td>\n",
       "      <td>273.168550</td>\n",
       "      <td>-31.585850</td>\n",
       "      <td>177.599289</td>\n",
       "      <td>16.582558</td>\n",
       "      <td>18.276463</td>\n",
       "      <td>0.101376</td>\n",
       "      <td>7C9369440EDB2E44FC74DD43E8489743A85A8243402102...</td>\n",
       "      <td>18DC384454900B440CABD3438836B14350FF6543682C64...</td>\n",
       "      <td>82C3F3420D4806433E048AC2F618374297DEA7417BB908...</td>\n",
       "      <td>9FCE663E69D5B43D4D2ED2BECEB284BFB76CCEBFA2FFF3...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>13585</td>\n",
       "      <td>nblago</td>\n",
       "      <td>2019-11-05 20:24:38.576000</td>\n",
       "      <td>bogus_subtract</td>\n",
       "      <td>1767</td>\n",
       "      <td>49</td>\n",
       "      <td>2017-09-01 19:23:40.000000</td>\n",
       "      <td>q</td>\n",
       "      <td>90004</td>\n",
       "      <td>3.134960</td>\n",
       "      <td>...</td>\n",
       "      <td>273.659695</td>\n",
       "      <td>-31.597761</td>\n",
       "      <td>110.529106</td>\n",
       "      <td>16.169716</td>\n",
       "      <td>18.791374</td>\n",
       "      <td>0.158836</td>\n",
       "      <td>80ED6341C0E5FFC2C0700742F0299042603568429007AD...</td>\n",
       "      <td>C03886C2B0E2CCC20008823F80CA3E4140BC2F42C043D8...</td>\n",
       "      <td>078BC8C263E57242AC90BBC0ECB221C271570B4250A660...</td>\n",
       "      <td>EE37A53FA68E813F61464F3F4072263F5EE7283FC26D67...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>14590</td>\n",
       "      <td>nblago</td>\n",
       "      <td>2019-11-05 20:24:49.179000</td>\n",
       "      <td>bogus</td>\n",
       "      <td>775</td>\n",
       "      <td>50</td>\n",
       "      <td>2017-09-01 19:24:50.000000</td>\n",
       "      <td>q</td>\n",
       "      <td>90004</td>\n",
       "      <td>3.253015</td>\n",
       "      <td>...</td>\n",
       "      <td>273.958172</td>\n",
       "      <td>-31.143006</td>\n",
       "      <td>75.347374</td>\n",
       "      <td>10.799380</td>\n",
       "      <td>19.207396</td>\n",
       "      <td>0.155616</td>\n",
       "      <td>A0661DC240FCD7C1B09986C240C912C2603B49C2303BB9...</td>\n",
       "      <td>80F972C2E09EA1C2804E90C260D63AC280CE32410065D4...</td>\n",
       "      <td>A1E48A4248945042A64E64C132C71BC202BA5D4283EA03...</td>\n",
       "      <td>3E238E3F4C24573F77480F3FC10F023F80D22D3F23DE58...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0       1                           2               3     4   5   \\\n",
       "0    9184  nblago  2019-11-05 20:23:46.518000           bogus   395  37   \n",
       "3    9210  nblago  2019-11-05 20:24:02.033000           bogus   515  37   \n",
       "6   12110  nblago  2019-11-05 20:24:24.389000            real  1295  47   \n",
       "9   13585  nblago  2019-11-05 20:24:38.576000  bogus_subtract  1767  49   \n",
       "12  14590  nblago  2019-11-05 20:24:49.179000           bogus   775  50   \n",
       "\n",
       "                            6  7      8         9   ...          14  \\\n",
       "0   2017-09-01 18:30:00.000000  q  90005  3.056144  ...  234.133234   \n",
       "3   2017-09-01 18:30:00.000000  q  90005  3.056144  ...  233.819094   \n",
       "6   2017-09-01 19:17:50.000000  q  90004  3.398105  ...  273.168550   \n",
       "9   2017-09-01 19:23:40.000000  q  90004  3.134960  ...  273.659695   \n",
       "12  2017-09-01 19:24:50.000000  q  90004  3.253015  ...  273.958172   \n",
       "\n",
       "           15          16         17         18        19  \\\n",
       "0  -14.702009   46.362450   7.713804  19.734650  0.180645   \n",
       "3  -14.612320   50.849087   7.740949  19.634357  0.165286   \n",
       "6  -31.585850  177.599289  16.582558  18.276463  0.101376   \n",
       "9  -31.597761  110.529106  16.169716  18.791374  0.158836   \n",
       "12 -31.143006   75.347374  10.799380  19.207396  0.155616   \n",
       "\n",
       "                                                   20  \\\n",
       "0   A0E80BC240CF3AC200F8BD3F608BD441706D224200001D...   \n",
       "3   6068E3C140616341C090AAC1005CE5C010410DC2300F0A...   \n",
       "6   7C9369440EDB2E44FC74DD43E8489743A85A8243402102...   \n",
       "9   80ED6341C0E5FFC2C0700742F0299042603568429007AD...   \n",
       "12  A0661DC240FCD7C1B09986C240C912C2603B49C2303BB9...   \n",
       "\n",
       "                                                   21  \\\n",
       "0   F06705C280BFE3C170B10F42403909C2401746C1A041DB...   \n",
       "3   00B2704060C559C280582641A04045C2803FBC4000ED23...   \n",
       "6   18DC384454900B440CABD3438836B14350FF6543682C64...   \n",
       "9   C03886C2B0E2CCC20008823F80CA3E4140BC2F42C043D8...   \n",
       "12  80F972C2E09EA1C2804E90C260D63AC280CE32410065D4...   \n",
       "\n",
       "                                                   22  \\\n",
       "0   575FDB406D7D9AC2D85514420177743F4F446140A113A9...   \n",
       "3   0484EB41AC3D82C154F6C541EFCA96407E925EC1281912...   \n",
       "6   82C3F3420D4806433E048AC2F618374297DEA7417BB908...   \n",
       "9   078BC8C263E57242AC90BBC0ECB221C271570B4250A660...   \n",
       "12  A1E48A4248945042A64E64C132C71BC202BA5D4283EA03...   \n",
       "\n",
       "                                                   23  \n",
       "0   BBA2273F32826E3F899BA63FD24CE83FD54E1040BDFA0C...  \n",
       "3   FC739F3F65BA8C3F90CF333FF7710D3EBE6F68BE19C2CD...  \n",
       "6   9FCE663E69D5B43D4D2ED2BECEB284BFB76CCEBFA2FFF3...  \n",
       "9   EE37A53FA68E813F61464F3F4072263F5EE7283FC26D67...  \n",
       "12  3E238E3F4C24573F77480F3FC10F023F80D22D3F23DE58...  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('/Volumes/ZAFIIRAH/meerlicht-project/Bogus_against_interesting/data/dumpformachinelearning_20200114161507.csv',sep=';',header=None)\n",
    "data = data.drop_duplicates(subset=0, keep=\"first\")\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load the new candidates\n",
    "- ID: The first column of the csv file is 'transientid' --> data.iloc[:,0]\n",
    "- full_data : Here we will provide the whole csv file as above, the code will atomatically select the last 4 or 3 column to extract the images. Four last columns is the candidate file in the order of (20)New Image, (21)Ref Image, (22)Diff Image, (23)Scorr image\n",
    "\n",
    "- n_images: The number of images to consider 4, 3, 2. Note that here we should be careful. If             the network that we will select below has been trained on 3 images, therefore we               will need to use n_images=3\n",
    "- min_pix: value range from 0 to 100 (applied when cropped = True). Note that here we should be careful. If the network that we will select below has been trained on 30X30 images, therefore we will need to use min_pix=35.\n",
    "- max_pix: value range from 0 to 100 (applied when cropped = True). Note that here we should be careful. If the network that we will select below has been trained on 30X30 images, therefore we will need to use max_pix=65.\n",
    "- cropped: True - cropping is done from the centre. If we want 30X30 pixels image, then min_pix= 35, max_pix=65"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters to change\n",
    "nClasses   = 2  # The number of classes we are classifying: Real and Bogus\n",
    "minPix     = 35 # The minimum pixel to be used from the image \n",
    "maxPix     = 65 # The maximum pixel to be used from the image\n",
    "num_images = 'NRD'  # The number of images to used for training and testing either 'NRDS', 'NRD', 'NR', 'D', 'S'\n",
    "threshold = 'threshold_9' #This variable is used when loading the image- threshold atleast 9 people vetted a source as either real or bogus - threshold_9, can also use threshold_8\n",
    "model_cnn_name = 'NET3'+'_'+threshold+'_'+num_images  # The network name choose from: 'NET1','NET2','NET3', NET1_32_64','NET1_64_128','NET1_128_256'\n",
    "probability_threshold = 0.5\n",
    "model_path = \"./meerCRAB_model/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of training instances: 134\n",
      "The Shape of the test set is (134, 30, 30, 3)\n"
     ]
    }
   ],
   "source": [
    "test, ID_test = load_new_candidate(ID=data.iloc[:,0].values,full_data=data,n_images=num_images,minPix=minPix,maxPix=maxPix,cropped=True)\n",
    "print(\"Total number of training instances: {}\".format(str(len(ID_test))))\n",
    "print(\"The Shape of the test set is {}\".format(test.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prediction on new candidate files\n",
    "Here we will load the pre-existing train model using the parameter \n",
    "\n",
    "INPUTS:\n",
    "- model_name: model_cnn_name = 'NET1_32_64'\n",
    "- X_test : should have shape (Nimages,100,100,3), (Nimages,30,30,3), (Nimages,30,30,4). This will vary depending on the criteria one use for min_pix, max_pix and num_images.\n",
    "- ID: The transient ID extracted from the csv file ID=data.iloc[:,0]\n",
    "\n",
    "OUTPUTS:\n",
    "- overall_real_prob: An array of probability that each candidate is real\n",
    "- overall_dataframe: A table with column transientid and ML_PROB_REAL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loaded model:NET3_threshold_9_NRD from disk\n"
     ]
    }
   ],
   "source": [
    "overall_real_prob, overall_dataframe = realbogus_prediction(model_name=model_cnn_name, X_test=test,ID=ID_test, probability_threshold=probability_threshold,model_path=model_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  9184,   9210,  12110,  13585,  14590,  21103,  21355,  24134,\n",
       "        25362,  27118,  31383,  38607,  46836,  46970,  47752,  51364,\n",
       "        51594,  52013,  53320,  53465,  54064,  54202,  54423,  60038,\n",
       "        61252,  62892,  66172,  66249,  66558,  68367,  75021,  75152,\n",
       "        75621,  76630,  78401,  78888,  80804,  80875,  87566,  97416,\n",
       "        97494,  99695, 103858, 104758, 113393, 115324, 131115, 132902,\n",
       "       134030, 134334, 134850, 147178, 147296, 149201, 149297, 149488,\n",
       "       150770, 152647, 156544, 157838, 158087, 165939, 166129, 170915,\n",
       "       171648, 172410, 173430, 178944, 182469, 184310, 190724, 191986,\n",
       "       198859, 203779, 204589, 218048, 222098, 224815, 225928, 225986,\n",
       "       230616, 230800, 241567, 243336, 247661, 247797, 251461, 251743,\n",
       "       254029, 259438, 259793, 266891, 269943, 271943, 272473, 272652,\n",
       "       273131, 274010, 274616, 275065, 280030, 281340, 288492, 290158,\n",
       "       294475, 297652, 301444, 303095, 304415, 306058, 308004, 308992,\n",
       "       311660, 314650, 316119, 318124, 318282, 320924, 327518, 334769,\n",
       "       335411, 340876, 342833, 344264, 345213, 348467, 361936, 367885,\n",
       "       372102, 373673, 374696, 379978, 387543, 387548], dtype=int32)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The transient ID for each candidate\n",
    "ID_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.31654243e-08, 1.26158457e-05, 1.00000000e+00, 1.00000000e+00,\n",
       "       0.00000000e+00, 3.74807087e-07, 2.82803194e-06, 1.57054619e-05,\n",
       "       5.67756445e-07, 1.00000000e+00, 3.14599532e-36, 9.91135370e-03,\n",
       "       7.69833207e-07, 3.27276561e-36, 6.06705116e-13, 2.57796913e-16,\n",
       "       1.00000000e+00, 3.25098608e-06, 4.40713984e-06, 1.89745464e-04,\n",
       "       1.00000000e+00, 2.42841881e-17, 2.75323458e-28, 3.97761779e-09,\n",
       "       1.27492435e-20, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00,\n",
       "       4.93719792e-15, 1.00000000e+00, 5.68406016e-04, 1.00000000e+00,\n",
       "       1.00000000e+00, 1.00000000e+00, 1.00000000e+00, 1.00000000e+00,\n",
       "       4.67047556e-09, 1.28934010e-24, 1.00000000e+00, 1.00000000e+00,\n",
       "       1.00000000e+00, 1.00000000e+00, 3.37149389e-03, 1.00000000e+00,\n",
       "       1.00000000e+00, 1.36183007e-05, 1.38836938e-07, 9.98423934e-01,\n",
       "       9.08938350e-16, 1.00000000e+00, 1.36255677e-14, 1.00000000e+00,\n",
       "       1.00000000e+00, 7.47383165e-05, 8.18805006e-07, 3.90982132e-06,\n",
       "       1.15901157e-02, 2.81123701e-03, 4.17419523e-03, 2.58884230e-03,\n",
       "       7.88906217e-03, 4.43488134e-05, 3.45852698e-07, 1.03305822e-04,\n",
       "       1.36817071e-05, 7.75187470e-09, 2.37895965e-05, 1.00000000e+00,\n",
       "       1.00000000e+00, 1.44670662e-02, 1.33166600e-06, 3.97067238e-03,\n",
       "       1.24842172e-05, 1.62225611e-08, 4.63738060e-03, 4.31816625e-06,\n",
       "       9.57165100e-03, 2.62273796e-04, 8.97694938e-03, 7.37275602e-03,\n",
       "       9.70646679e-01, 3.67744178e-05, 1.00000000e+00, 2.66937539e-03,\n",
       "       4.21992852e-04, 2.61720947e-06, 1.04800129e-05, 6.14037053e-05,\n",
       "       7.02621927e-03, 8.98812024e-04, 9.11396754e-04, 8.88773892e-03,\n",
       "       2.93340531e-06, 5.43874779e-10, 1.00000000e+00, 1.00000000e+00,\n",
       "       5.13088073e-14, 1.23650636e-04, 5.49513707e-03, 1.38716970e-03,\n",
       "       6.29482965e-05, 2.11755395e-01, 1.00000000e+00, 1.00000000e+00,\n",
       "       0.00000000e+00, 1.78662827e-03, 5.27381338e-03, 2.64945536e-22,\n",
       "       1.00000000e+00, 2.18357545e-08, 2.88722967e-03, 3.79431294e-05,\n",
       "       1.40626857e-04, 4.09733242e-04, 1.00000000e+00, 0.00000000e+00,\n",
       "       1.00000000e+00, 1.44786703e-13, 4.22406383e-03, 1.00000000e+00,\n",
       "       1.00000000e+00, 3.46893468e-03, 2.14929343e-04, 1.79672719e-03,\n",
       "       7.70183804e-04, 9.93513167e-01, 9.78215758e-06, 1.19915123e-04,\n",
       "       1.32663501e-03, 2.04122649e-03, 6.53852755e-03, 1.15443310e-02,\n",
       "       6.55119587e-03, 1.78822564e-04], dtype=float32)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The probability that each source is a real source: It varies from 0 to 1\n",
    "overall_real_prob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>transientid</th>\n",
       "      <th>ML_PROB_REAL</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9184</td>\n",
       "      <td>1.316542e-08</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9210</td>\n",
       "      <td>1.261585e-05</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>12110</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13585</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14590</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>373673</td>\n",
       "      <td>2.041226e-03</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>374696</td>\n",
       "      <td>6.538528e-03</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>379978</td>\n",
       "      <td>1.154433e-02</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>387543</td>\n",
       "      <td>6.551196e-03</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>387548</td>\n",
       "      <td>1.788226e-04</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>134 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     transientid  ML_PROB_REAL  label\n",
       "0           9184  1.316542e-08    0.0\n",
       "1           9210  1.261585e-05    0.0\n",
       "2          12110  1.000000e+00    1.0\n",
       "3          13585  1.000000e+00    1.0\n",
       "4          14590  0.000000e+00    0.0\n",
       "..           ...           ...    ...\n",
       "129       373673  2.041226e-03    0.0\n",
       "130       374696  6.538528e-03    0.0\n",
       "131       379978  1.154433e-02    0.0\n",
       "132       387543  6.551196e-03    0.0\n",
       "133       387548  1.788226e-04    0.0\n",
       "\n",
       "[134 rows x 3 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# A dataframe that contains the transient ID and its probability that it is a Real source\n",
    "overall_dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}