pyleaves_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First, navigate to your chosen working directory."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %cd ~/..\n",
    "# %pwd\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Device mapping:\n",
      "/job:localhost/replica:0/task:0/device:XLA_CPU:0 -> device: XLA_CPU device\n",
      "/job:localhost/replica:0/task:0/device:GPU:0 -> device: 0, name: TITAN X (Pascal), pci bus id: 0000:88:00.0, compute capability: 6.1\n",
      "/job:localhost/replica:0/task:0/device:XLA_GPU:0 -> device: XLA_GPU device\n",
      "\n",
      "['/device:GPU:0']\n",
      "OpenCV is built with OpenMP support. This usually results in poor performance. For details, see https://github.com/tensorpack/benchmarks/blob/master/ImageNet/benchmark-opencv-resize.py\n",
      "1 Physical GPUs, 1 Logical GPU\n"
     ]
    }
   ],
   "source": [
    "random_seed = 34\n",
    "import random\n",
    "random.seed(random_seed)\n",
    "import dataset\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from stuf import stuf\n",
    "import time\n",
    "import os\n",
    "\n",
    "gpu=5\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"6\"#str(gpu)\n",
    "\n",
    "from tensorflow.keras import backend as K\n",
    "import tensorflow as tf;\n",
    "tf.enable_eager_execution()\n",
    "\n",
    "tf_config=tf.ConfigProto(log_device_placement=True)\n",
    "tf_config.gpu_options.allocator_type = 'BFC'\n",
    "tf_config.gpu_options.allow_growth = True\n",
    "tf_config.allow_soft_placement = True\n",
    "sess = tf.Session(config=tf_config)\n",
    "K.set_session(sess)\n",
    "\n",
    "from tensorflow.python.client import device_lib\n",
    "\n",
    "def get_available_gpus():\n",
    "    local_device_protos = device_lib.list_local_devices()\n",
    "    return [x.name for x in local_device_protos if x.device_type == 'GPU']\n",
    "\n",
    "\n",
    "print(get_available_gpus())\n",
    "\n",
    "import pyleaves\n",
    "\n",
    "from pyleaves.analysis.img_utils import convert_to_png\n",
    "from pyleaves import leavesdb\n",
    "from pyleaves.data_pipeline.tensorpack_loaders import get_multiprocess_dataflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET = \"Fossil\"\n",
    "# tf.test.gpu_device_name()\n",
    "# dir(tf.test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **1.** Initialize and connect to database in local filesystem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Proceeding with sql db at location /home/jacob/scripts/leavesdb.db\n",
      "/home/jacob/scripts/leavesdb.db\n"
     ]
    }
   ],
   "source": [
    "# local_db = os.path.join(os.getcwd(),'pyleaves','leavesdb','resources','leavesdb.db')\n",
    "\n",
    "local_db = leavesdb.init_local_db()\n",
    "\n",
    "print(local_db)\n",
    "\n",
    "db = dataset.connect(f'sqlite:///{local_db}', row_type=stuf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **2.** Print a summary of the database's contents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Database column keys': ['id', 'specie', 'genus', 'path', 'family', 'dataset'], 'distinct datasets': ['Fossil', 'Leaves', 'PNAS', 'plant_village'], 'Number of distinct families': [('Fossil', 27), ('Leaves', 376), ('PNAS', 19), ('plant_village', 3)], 'Number of rows in db': 119084}\n"
     ]
    }
   ],
   "source": [
    "db_summary = leavesdb.summarize_db(db)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **3.** Select a subset of datasets\n",
    "##### Here we select the Fossil dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data = leavesdb.db_query.load_Fossil_data(db)\n",
    "\n",
    "# data = leavesdb.db_query.load_data(db, dataset=DATASET)\n",
    "\n",
    "# data = leavesdb.db_query.load_Leaves_data(db)\n",
    "\n",
    "\n",
    "data = leavesdb.db_query.load_all_data(db)\n",
    "\n",
    "data_by_dataset = data.groupby(by='dataset')\n",
    "data_by_dataset_dict = {k:v for k,v in data_by_dataset}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                path   family        dataset\n",
      "0  /media/data_cifs/jacob/data/plantvillage/Apple...  Rosacea  plant_village\n",
      "1  /media/data_cifs/jacob/data/plantvillage/Apple...  Rosacea  plant_village\n",
      "2  /media/data_cifs/jacob/data/plantvillage/Apple...  Rosacea  plant_village\n",
      "3  /media/data_cifs/jacob/data/plantvillage/Apple...  Rosacea  plant_village\n",
      "4  /media/data_cifs/jacob/data/plantvillage/Apple...  Rosacea  plant_village\n"
     ]
    }
   ],
   "source": [
    "print(data.head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting  PNAS  with  5314  files\n",
      "Finished copying 5314 from PNAS in 439.638 seconds at a rate of 12.087 images/secae/Sapindaceae_Ungnadia_speciosa_356 {AxelrodUCMP} [1.00x].png].pnggged image 5247 and saved at /media/data/jacob/Fossil_Project/PNAS/Sapindaceae/Sapindaceae_Allophylus_granulatus_12574 {WolfeUSGS} [1.96x].pngConverted image 5254 and saved at /media/data/jacob/Fossil_Project/PNAS/Sapindaceae/Sapindaceae_Cupania_belizensis_5838 {WolfeUSGS} [1.59x].pngConverted image 5253 and saved at /media/data/jacob/Fossil_Project/PNAS/Sapindaceae/Sapindaceae_Cardiospermum_halicacabum_969 {AxelrodUCMP} [1.00x].pngn_assamicumi_08-094-03 {Klucking} [1.00x].pngata/jacob/Fossil_Project/PNAS/Celastraceae/Celastraceae_Microtropis_fokienensis_13592 {WolfeUSGS} [1.96x].pngConverted image 596 and saved at /media/data/jacob/Fossil_Project/PNAS/Ericaceae/Ericaceae_Arctostaphylos_bicolor_446 {AxelrodUCMP} [1.00x].pngConverted image 689 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Aganope_stuhlmannii_9917 {WolfeUSGS} [1.96x].pngConverted image 678 and saved at /media/data/jacob/Fossil_Project/PNAS/Ericaceae/Ericaceae_Vaccinium_stamineum_13144 {WolfeUSGS} [1.96x].pngConverted image 756 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Caesalpinia_pumila_6337 {WolfeUSGS} [1.96x].pngConverted image 745 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Brongniartia_revoluta_15662 {WolfeUSGS} [1.96x].pngConverted image 757 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Calliandra_guildingii_6368 {WolfeUSGS} [1.96x].pngConverted image 780 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Cassia_spectabilis_30377 {WolfeUSGS} [1.96x].pngConverted image 869 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Erythrina_fusca_16462 {WolfeUSGS} [1.96x].pngConverted image 873 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Eysenhardtia_platycarpa_16469 {WolfeUSGS} [1.96x].pngConverted image 875 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Galactia_belizensis_16473 {WolfeUSGS} [1.96x].pngConverted image 884 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Gastrolobium_grandiflorum_9857 {WolfeUSGS} [1.96x].pngConverted image 912 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Hymenaea_sp._4272 {WolfeUSGS} [1.96x].pngConverted image 901 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Harpalyce_pringlei_16488 {WolfeUSGS} [1.96x].pngConverted image 902 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Harpalyce_rupicola_7613 {WolfeUSGS} [1.96x].pngConverted image 921 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Inocarpus_edulis_9876 {WolfeUSGS} [1.96x].pngConverted image 932 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Lennea_modesta_30588 {WolfeUSGS} [1.96x].pngConverted image 946 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Lonchocarpus_latifolius_16519 {WolfeUSGS} [1.96x].pngConverted image 977 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Myroxylon_toluiferum_5206 {WolfeUSGS} [1.96x].pngConverted image 978 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Newtonia_zenkeri_5176 {WolfeUSGS} [1.96x].pngConverted image 1036 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Senna_pilifera_30358 {WolfeUSGS} [1.96x].pngConverted image 1112 and saved at /media/data/jacob/Fossil_Project/PNAS/Fagaceae/Fagaceae_Quercus_cordifolia_14727 {WolfeUSGS} [1.96x].pngConverted image 1111 and saved at /media/data/jacob/Fossil_Project/PNAS/Fagaceae/Fagaceae_Quercus_coccolobifolia_14723 {WolfeUSGS} [1.96x].pngConverted image 1110 and saved at /media/data/jacob/Fossil_Project/PNAS/Fagaceae/Fagaceae_Quercus_coccifera_10670 {WolfeUSGS} [1.96x].pngConverted image 1118 and saved at /media/data/jacob/Fossil_Project/PNAS/Fagaceae/Fagaceae_Quercus_dumosa_396 {AxelrodUCMP} [1.00x].pngConverted image 1179 and saved at /media/data/jacob/Fossil_Project/PNAS/Lauraceae/Lauraceae_Beilschmiedia_mexicana_14929 {WolfeUSGS} [1.96x].pngConverted image 1186 and saved at /media/data/jacob/Fossil_Project/PNAS/Lauraceae/Lauraceae_Cinnamomum_elegans_02-036-01 {Klucking} [1.00x].pngConverted image 1234 and saved at /media/data/jacob/Fossil_Project/PNAS/Lauraceae/Lauraceae_Lindera_obtusiloba_02-067-03 {Klucking} [1.00x].pngConverted image 1317 and saved at /media/data/jacob/Fossil_Project/PNAS/Malvaceae/Malvaceae_Abutilon_texense_10429 {WolfeUSGS} [1.96x].pngConverted image 1314 and saved at /media/data/jacob/Fossil_Project/PNAS/Lauraceae/Lauraceae_Sassafras_tzumu_776 {WolfeUSGS} [1.96x].pngConverted image 1320 and saved at /media/data/jacob/Fossil_Project/PNAS/Malvaceae/Malvaceae_Ayenia_tomentosa_11447 {WolfeUSGS} [1.96x].pngConverted image 1371 and saved at /media/data/jacob/Fossil_Project/PNAS/Malvaceae/Malvaceae_Leptonychia_caudata_07-047-06 {Klucking} [0.85x].pngConverted image 1381 and saved at /media/data/jacob/Fossil_Project/PNAS/Malvaceae/Malvaceae_Ochroma_lagopus_07-061-01 {Klucking} [0.85x].pngConverted image 1400 and saved at /media/data/jacob/Fossil_Project/PNAS/Malvaceae/Malvaceae_Tilia_cordata_120 {WolfeUSGS} [1.96x].pngConverted image 1407 and saved at /media/data/jacob/Fossil_Project/PNAS/Malvaceae/Malvaceae_Trichospermum_pleiostigma_8039 {WolfeUSGS} [1.96x].pngConverted image 1425 and saved at /media/data/jacob/Fossil_Project/PNAS/Melastomataceae/Melastomataceae_Clidemia_capitellata_04-023-05 {Klucking} [1.50x].pngConverted image 1429 and saved at /media/data/jacob/Fossil_Project/PNAS/Melastomataceae/Melastomataceae_Creochiton_bibracteata_04-030-01 {Klucking} [1.50x].pngConverted image 1423 and saved at /media/data/jacob/Fossil_Project/PNAS/Melastomataceae/Melastomataceae_Calycogonium_krugii_04-022-01 {Klucking} [1.00x].pngConverted image 1415 and saved at /media/data/jacob/Fossil_Project/PNAS/Melastomataceae/Melastomataceae_Axinaea_macrophylla_04-014-05 {Klucking} [1.50x].pngConverted image 1430 and saved at /media/data/jacob/Fossil_Project/PNAS/Melastomataceae/Melastomataceae_Dichaetanthera_cordifolia_04-030-03 {Klucking} [1.50x].pngConverted image 1571 and saved at /media/data/jacob/Fossil_Project/PNAS/Myrtaceae/Myrtaceae_Eucalyptus_haematoxylon_03-036-02 {Klucking} [1.50x].pngConverted image 1654 and saved at /media/data/jacob/Fossil_Project/PNAS/Myrtaceae/Myrtaceae_Myrcia_latior_03-090-04 {Klucking} [1.50x].pngConverted image 1671 and saved at /media/data/jacob/Fossil_Project/PNAS/Myrtaceae/Myrtaceae_Myrcia_uberavensis_03-099-01 {Klucking} [1.50x].pngConverted image 1777 and saved at /media/data/jacob/Fossil_Project/PNAS/Passifloraceae/Passifloraceae_Passiflora_tetrandra_3490 {WolfeUSGS} [1.96x].pngConverted image 1799 and saved at /media/data/jacob/Fossil_Project/PNAS/Phyllanthaceae/Phyllanthaceae_Antidesma_bunius_08-039-04 {Klucking} [1.00x].pngConverted image 1890 and saved at /media/data/jacob/Fossil_Project/PNAS/Phyllanthaceae/Phyllanthaceae_Glochidion_cyrtostylum_08-102-03 {Klucking} [1.00x].pngConverted image 1893 and saved at /media/data/jacob/Fossil_Project/PNAS/Phyllanthaceae/Phyllanthaceae_Glochidion_eriocarpum_08-092-05 {Klucking} [1.00x].pngConverted image 1946 and saved at /media/data/jacob/Fossil_Project/PNAS/Phyllanthaceae/Phyllanthaceae_Phyllanthus_iontrousieri_08-118-04 {Klucking} [1.00x].pngConverted image 1964 and saved at /media/data/jacob/Fossil_Project/PNAS/Phyllanthaceae/Phyllanthaceae_Phyllanthus_urceolatus_08-121-01 {Klucking} [1.00x].pngConverted image 1976 and saved at /media/data/jacob/Fossil_Project/PNAS/Phyllanthaceae/Phyllanthaceae_Uapaca_goetzei_08-138-01 {Klucking} [1.00x].pngConverted image 1997 and saved at /media/data/jacob/Fossil_Project/PNAS/Rosaceae/Rosaceae_Malus_formosana_8641 {WolfeUSGS} [1.96x].pngConverted image 2034 and saved at /media/data/jacob/Fossil_Project/PNAS/Rosaceae/Rosaceae_Prunus_venosa_8799 {WolfeUSGS} [1.96x].pngConverted image 2048 and saved at /media/data/jacob/Fossil_Project/PNAS/Rosaceae/Rosaceae_Rosa_californica_55 {AxelrodUCMP} [1.00x].pngConverted image 2067 and saved at /media/data/jacob/Fossil_Project/PNAS/Rosaceae/Rosaceae_Rubus_weberbaueri_12089 {WolfeUSGS} [1.96x].pngConverted image 2094 and saved at /media/data/jacob/Fossil_Project/PNAS/Rubiaceae/Rubiaceae_Bathysa_nicholsonii_7471 {WolfeUSGS} [1.96x].pngConverted image 2093 and saved at /media/data/jacob/Fossil_Project/PNAS/Rubiaceae/Rubiaceae_Aulacocalyx_jasminiflora_14397 {WolfeUSGS} [1.96x].pngConverted image 2132 and saved at /media/data/jacob/Fossil_Project/PNAS/Rubiaceae/Rubiaceae_Coffea_eugenioides_13857 {WolfeUSGS} [1.96x].pngConverted image 2138 and saved at /media/data/jacob/Fossil_Project/PNAS/Rubiaceae/Rubiaceae_Corynanthe_macroceras_5264 {WolfeUSGS} [1.96x].pngConverted image 2212 and saved at /media/data/jacob/Fossil_Project/PNAS/Rubiaceae/Rubiaceae_Manettia_divaricata_13967 {WolfeUSGS} [1.96x].pngConverted image 2218 and saved at /media/data/jacob/Fossil_Project/PNAS/Rubiaceae/Rubiaceae_Morinda_bucidifolia_13979 {WolfeUSGS} [1.96x].pngConverted image 2211 and saved at /media/data/jacob/Fossil_Project/PNAS/Rubiaceae/Rubiaceae_Manettia_cordifolia_13966 {WolfeUSGS} [1.96x].pngConverted image 2228 and saved at /media/data/jacob/Fossil_Project/PNAS/Rubiaceae/Rubiaceae_Pachystylus_guelcherianus_12777 {WolfeUSGS} [1.96x].pngConverted image 2250 and saved at /media/data/jacob/Fossil_Project/PNAS/Rubiaceae/Rubiaceae_Psilanthus_mannii_5352 {WolfeUSGS} [1.96x].pngConverted image 2252 and saved at /media/data/jacob/Fossil_Project/PNAS/Rubiaceae/Rubiaceae_Psychotria_alba_7344 {WolfeUSGS} [1.96x].pngConverted image 2323 and saved at /media/data/jacob/Fossil_Project/PNAS/Salicaceae/Salicaceae_Casearia_decandra_11237 {WolfeUSGS} [1.96x].pngConverted image 2335 and saved at /media/data/jacob/Fossil_Project/PNAS/Salicaceae/Salicaceae_Casearia_tremula_06-016-04 {Klucking} [1.50x].pngConverted image 2464 and saved at /media/data/jacob/Fossil_Project/PNAS/Salicaceae/Salicaceae_Salix_lucida_10291 {WolfeUSGS} [1.96x].pngConverted image 2475 and saved at /media/data/jacob/Fossil_Project/PNAS/Salicaceae/Salicaceae_Salix_paradoxa_18143 {WolfeUSGS} [1.96x].pngConverted image 2456 and saved at /media/data/jacob/Fossil_Project/PNAS/Salicaceae/Salicaceae_Salix_laevigata_14850 {WolfeUSGS} [1.96x].pngConverted image 2462 and saved at /media/data/jacob/Fossil_Project/PNAS/Salicaceae/Salicaceae_Salix_lemmonii_10294 {WolfeUSGS} [1.96x].pngConverted image 2474 and saved at /media/data/jacob/Fossil_Project/PNAS/Salicaceae/Salicaceae_Salix_padophylla_10379 {WolfeUSGS} [1.96x].pngConverted image 2735 and saved at /media/data/jacob/Fossil_Project/PNAS/Annonaceae/Annonaceae_Artabotrys_borneensis_01-113-02 {Klucking} [1.50x].pngConverted image 2774 and saved at /media/data/jacob/Fossil_Project/PNAS/Annonaceae/Annonaceae_Desmopsis_panamensis_9501 {WolfeUSGS} [0.41x].pngConverted image 2882 and saved at /media/data/jacob/Fossil_Project/PNAS/Annonaceae/Annonaceae_Unonopsis_rufescens_9533 {WolfeUSGS} [0.43x].pngConverted image 2895 and saved at /media/data/jacob/Fossil_Project/PNAS/Annonaceae/Annonaceae_Xylopia_discreta_11196 {WolfeUSGS} [1.96x].pngConverted image 2894 and saved at /media/data/jacob/Fossil_Project/PNAS/Annonaceae/Annonaceae_Xylopia_barbata_11190 {WolfeUSGS} [1.96x].pngConverted image 2896 and saved at /media/data/jacob/Fossil_Project/PNAS/Annonaceae/Annonaceae_Xylopia_platysperma_07-083-02 {Klucking} [0.85x].pngConverted image 3145 and saved at /media/data/jacob/Fossil_Project/PNAS/Celastraceae/Celastraceae_Salacia_belizensis_13614 {WolfeUSGS} [1.96x].pngConverted image 3163 and saved at /media/data/jacob/Fossil_Project/PNAS/Combretaceae/Combretaceae_Combretum_aculeatum_05-024-03 {Klucking} [1.50x].pngConverted image 3200 and saved at /media/data/jacob/Fossil_Project/PNAS/Combretaceae/Combretaceae_Combretum_punctatum_05-025-03 {Klucking} [1.50x].pngConverted image 3343 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Aeschynomene_purpusii_30159 {WolfeUSGS} [1.96x].pngConverted image 3478 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Cratylia_argentea_9447 {WolfeUSGS} [1.96x].pngConverted image 3530 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Diplotropis_purpurea_9450 {WolfeUSGS} [1.96x].pngConverted image 3536 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Dumasia_villosa_12693 {WolfeUSGS} [1.96x].pngConverted image 3545 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Erythrina_herbacea_30524 {WolfeUSGS} [1.96x].pngConverted image 3580 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Hymenaea_courbaril_4283 {WolfeUSGS} [1.96x].pngConverted image 3716 and saved at /media/data/jacob/Fossil_Project/PNAS/Fabaceae/Fabaceae_Tessmannia_anomala_9998 {WolfeUSGS} [1.96x].pngConverted image 3792 and saved at /media/data/jacob/Fossil_Project/PNAS/Fagaceae/Fagaceae_Quercus_intricata_14752 {WolfeUSGS} [1.96x].pngConverted image 3911 and saved at /media/data/jacob/Fossil_Project/PNAS/Lauraceae/Lauraceae_Mezilaurus_itauba_02-086-03 {Klucking} [1.00x].pngConverted image 3919 and saved at /media/data/jacob/Fossil_Project/PNAS/Lauraceae/Lauraceae_Neolitsea_cassiaefolia_07-015-02 {Klucking} [0.85x].pngConverted image 4048 and saved at /media/data/jacob/Fossil_Project/PNAS/Malvaceae/Malvaceae_Sterculia_ferruginea_11433 {WolfeUSGS} [1.96x].pngConverted image 4121 and saved at /media/data/jacob/Fossil_Project/PNAS/Melastomataceae/Melastomataceae_Miconia_aspergillaris_04-057-05 {Klucking} [1.50x].pngConverted image 4217 and saved at /media/data/jacob/Fossil_Project/PNAS/Myrtaceae/Myrtaceae_Eucalyptus_aspera_03-032-02 {Klucking} [1.50x].pngConverted image 4216 and saved at /media/data/jacob/Fossil_Project/PNAS/Myrtaceae/Myrtaceae_Decaspermum_parviflorum_03-019-06 {Klucking} [1.50x].pngConverted image 4220 and saved at /media/data/jacob/Fossil_Project/PNAS/Myrtaceae/Myrtaceae_Eucalyptus_calycogon_03-022-04 {Klucking} [1.50x].pngConverted image 4359 and saved at /media/data/jacob/Fossil_Project/PNAS/Myrtaceae/Myrtaceae_Syzygium_macranthum_03-125-02 {Klucking} [1.50x].pngConverted image 4366 and saved at /media/data/jacob/Fossil_Project/PNAS/Myrtaceae/Myrtaceae_Syzygium_samarangense_03-143-04 {Klucking} [1.50x].pngConverted image 4511 and saved at /media/data/jacob/Fossil_Project/PNAS/Phyllanthaceae/Phyllanthaceae_Bridelia_ovata_08-018-03 {Klucking} [1.00x].pngConverted image 4512 and saved at /media/data/jacob/Fossil_Project/PNAS/Phyllanthaceae/Phyllanthaceae_Bridelia_parvifolia_08-012-03 {Klucking} [1.00x].pngConverted image 4626 and saved at /media/data/jacob/Fossil_Project/PNAS/Phyllanthaceae/Phyllanthaceae_Securinega_ramiflora_3164 {WolfeUSGS} [1.96x].pngConverted image 4833 and saved at /media/data/jacob/Fossil_Project/PNAS/Rubiaceae/Rubiaceae_Ixora_vitiensis_13939 {WolfeUSGS} [1.96x].pngConverted image 4950 and saved at /media/data/jacob/Fossil_Project/PNAS/Rubiaceae/Rubiaceae_Timonius_wallichianus_14145 {WolfeUSGS} [1.96x].pngConverted image 4958 and saved at /media/data/jacob/Fossil_Project/PNAS/Rubiaceae/Rubiaceae_Vangueria_infausta_12773 {WolfeUSGS} [1.96x].pngConverted image 4977 and saved at /media/data/jacob/Fossil_Project/PNAS/Salicaceae/Salicaceae_Calantica_jalbertii_06-007-04 {Klucking} [1.50x].pngConverted image 4984 and saved at /media/data/jacob/Fossil_Project/PNAS/Salicaceae/Salicaceae_Casearia_glomerata_06-014-01 {Klucking} [1.50x].pngConverted image 5096 and saved at /media/data/jacob/Fossil_Project/PNAS/Salicaceae/Salicaceae_Salix_cinerea_10380 {WolfeUSGS} [1.96x].pngConverted image 5108 and saved at /media/data/jacob/Fossil_Project/PNAS/Salicaceae/Salicaceae_Salix_fragilis_10277 {WolfeUSGS} [1.96x].pngConverted image 5105 and saved at /media/data/jacob/Fossil_Project/PNAS/Salicaceae/Salicaceae_Salix_eastwoodiae_92 {AxelrodUCMP} [1.00x].pngConverted image 5106 and saved at /media/data/jacob/Fossil_Project/PNAS/Salicaceae/Salicaceae_Salix_eucalyptoides_3231 {WolfeUSGS} [1.96x].pngConverted image 5255 and saved at /media/data/jacob/Fossil_Project/PNAS/Sapindaceae/Sapindaceae_Cupania_glabra_5837 {WolfeUSGS} [1.96x].pngConverted image 5252 and saved at /media/data/jacob/Fossil_Project/PNAS/Sapindaceae/Sapindaceae_Bridgesia_incisifolia_5745 {WolfeUSGS} [1.96x].pngConverted image 5251 and saved at /media/data/jacob/Fossil_Project/PNAS/Sapindaceae/Sapindaceae_Blighia_unijugata_4922 {WolfeUSGS} [1.96x].png\n",
      "Starting  plant_village  with  6129  files\n",
      "Finished copying 6129 from plant_village in 203.200 seconds at a rate of 30.162 images/sece/ffd8aa68-138f-4114-96c7-21eef72e1e13___RS_HL 9881_final_masked.png.pngngted image 6093 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/fad75ca9-7035-4045-b3d3-475b558d49ba___RS_HL 0010_final_masked.pngmasked.pngConverted image 6084 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/f7069452-9405-45a6-bfb3-f21ea42b9b02___RS_HL 9966_final_masked.png6 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/715fa72a-0dc4-4bb1-bafe-46a50354cc73___RS_HL 0324_final_masked.pngConverted image 2275 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/713d5c9e-0b87-4e9f-9b60-5339f26ed924___RS_HL 5446_final_masked.pngConverted image 2279 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/71da69d8-7849-4a0e-be81-e94ba8b9379a___RS_HL 4995_final_masked.pngage 135 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/1398b04d-e259-418b-a5db-d4b20b7a3380___RS_HL 7536_final_masked.pngConverted image 251 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/21c42956-0894-43bf-a1b8-787ef33594c2___RS_HL 7686_final_masked.pngConverted image 527 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/4c368acb-0a47-4c7f-84bd-29dc74660db9___RS_HL 5800_final_masked.pngConverted image 536 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/4cd37f4e-557b-4317-8772-21abd1efec65___RS_HL 7420_final_masked.pngConverted image 534 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/4c7e1a5a-fc48-43a8-959d-c7c8ad6462b9___RS_HL 7561_final_masked.pngConverted image 528 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/4c3bd8fb-b507-4ef4-93b6-39f1b0d087d8___RS_HL 6252_final_masked.pngConverted image 532 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/4c6671cb-b873-4e04-9dd8-fafb839a3d28___RS_HL 7797_final_masked.pngConverted image 538 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/4cf06978-a0b5-43a0-8da3-fda45eee2e1c___RS_HL 5777_final_masked.pngConverted image 553 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/4f7c8fa5-f8cf-4b64-aaff-14d4a1c2cb27___RS_HL 7421_final_masked.pngConverted image 574 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/526782c6-8d24-4438-8255-37422811d33f___RS_HL 7601_final_masked.pngConverted image 695 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/65f9450e-cd32-465c-b2cd-e24111798fb1___RS_HL 7950_final_masked.pngConverted image 696 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/65fcf384-964b-41c5-9a54-aecb473283d2___RS_HL 7672_final_masked.pngConverted image 698 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/6695f23d-03f2-48ce-a3fe-b0fcd4f4972c___RS_HL 5884_final_masked.pngConverted image 725 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/6b11324f-6b70-40d7-9cd2-f7d36b0c69cd___RS_HL 7431_final_masked.pngConverted image 738 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/6fdc92a4-12e5-43f3-8766-96f49075b59d___RS_HL 8015_final_masked.pngConverted image 787 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/77004cfa-f435-49af-8435-7986927228ca___RS_HL 7609_final_masked.pngConverted image 842 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/80089e7a-4c0c-492b-ac23-753be3617cbf___RS_HL 6099_final_masked.pngConverted image 1001 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/990838f4-b976-45bb-859d-95de901acc8c___RS_HL 7365_final_masked.pngConverted image 1156 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/b31c759a-547a-46de-937f-56714107db63___RS_HL 5857_final_masked.pngConverted image 1222 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/be0f61c7-6307-4492-8a2c-50987ca6890e___RS_HL 7403_final_masked.pngConverted image 1230 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/bf26ad48-1ebb-4186-a52c-e249dc19317e___RS_HL 8195_final_masked.pngConverted image 1227 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/becd7e46-f87c-46d0-a973-f957e99ba4b7___RS_HL 7519_final_masked.pngConverted image 1224 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/be76ffa1-32a2-4d45-a5b4-740f0eb73156___RS_HL 5974_final_masked.pngConverted image 1238 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/bfff4d8f-8a42-435a-9793-07d7858d0e9a___RS_HL 6177_final_masked.pngConverted image 1241 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/c02b90e9-753e-44a7-a4b8-1e19f1de42f4___RS_HL 7971_final_masked.pngConverted image 1278 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/c69944f6-08da-4638-8ff6-7e97b5878b57___RS_HL 5816_final_masked.pngConverted image 1348 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/d1660af9-58e5-458f-b125-4940f06711c0___RS_HL 7522_final_masked.pngConverted image 1375 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/d44b9a3e-7cec-4e55-af4b-0df481af3f79___RS_HL 7831_final_masked.pngConverted image 1384 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/d5578393-868f-49fa-8fff-4f0484b4507e___RS_HL 5995_final_masked.pngConverted image 1389 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/d5f63752-d817-4b29-a773-f0e87544d61f___RS_HL 7247_final_masked.pngConverted image 1391 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/d6035c7f-f163-4c6b-ba51-d61d10ca1fed___RS_HL 8109_final_masked.pngConverted image 1395 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/d6590bcb-cb7e-47c9-beb0-80785efb4046___RS_HL 6234_final_masked.pngConverted image 1494 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/e439a002-e035-401e-a1aa-77f99fbd65ad___RS_HL 5880_final_masked.pngConverted image 1493 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/e3bbf9e4-4af5-4137-b996-440c5de827dd___RS_HL 5780_final_masked.pngConverted image 1495 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/e4bca84a-8bf3-477f-8c67-208f7484b0af___RS_HL 7287_final_masked.pngConverted image 1661 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/0353a7a5-75e0-4345-9264-0c2b72ea8af1___RS_HL 5239_final_masked.pngConverted image 1687 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/096010ca-2249-4787-b94d-8be6b27552c5___RS_HL 2383_final_masked.pngConverted image 1689 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/09692362-4a92-4a2c-a4a9-eac5f317e4cc___RS_HL 0638_final_masked.pngConverted image 1690 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/098a3191-5ad1-4890-af55-e9af2c446e48___RS_HL 5125_final_masked.pngConverted image 1766 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/1775f774-ea3a-45b5-9295-42a64d933600___RS_HL 5370_final_masked.pngConverted image 1774 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/192297e0-c7e4-4fc8-b283-0e1a87b316c9___RS_HL 2325_final_masked.pngConverted image 1826 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/202b0c37-fcfe-486f-ad9d-fd8569ce7087___RS_HL 5172_final_masked.pngConverted image 1828 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/204fc27c-f0b4-4ddf-bd76-4ae465b406f6___RS_HL 2585_final_masked.pngConverted image 1964 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/3a1a17f2-2d62-4594-9edb-2b19c038aa68___RS_HL 2370_final_masked.pngConverted image 1990 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/3eaddffc-ef51-4751-98d9-5ddb823d0a3f___RS_HL 4880_final_masked.pngConverted image 2096 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/4f71ab62-fa68-4288-8306-a26d30629ed7___RS_HL 2554_final_masked.pngConverted image 2104 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/50ee515b-c1cf-4340-9766-ab82881820a1___RS_HL 5209_final_masked.pngConverted image 2161 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/5be10ee8-d629-4b0a-bef4-5b12c67fe760___RS_HL 4976_final_masked.pngConverted image 2249 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/6cb8c035-59e3-4691-b88b-adc9c1d7f702___RS_HL 0419_final_masked.pngConverted image 2282 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/72cb878c-5714-4170-97ee-deee097d6de4___RS_HL 0743_final_masked.pngConverted image 2368 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/7ebded63-7349-4ed4-9e3c-488a79d6252e___RS_HL 5169_final_masked.pngConverted image 2396 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/83c4fb58-0c0b-4577-9378-864c686a5ab8___RS_HL 5322_final_masked.pngConverted image 2472 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/90f60cbf-91a1-4586-b30c-f1fff71d2ff3___RS_HL 5108_final_masked.pngConverted image 2479 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/9233752c-5a86-42d8-89a3-7aaec3c7af67___RS_HL 5339_final_masked.pngConverted image 2517 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/979711bc-7fad-43c3-a952-cbf8f4c9a796___RS_HL 2461_final_masked.pngConverted image 2525 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/98fe313e-c7cf-47b9-bc2d-dfe0ba4cd3ff___RS_HL 0515_final_masked.pngConverted image 2554 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/9e7cd61a-dc19-4e11-89a9-bde417b33227___RS_HL 5228_final_masked.pngConverted image 2558 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/9f2b853f-9e38-4867-a083-13222abdf778___RS_HL 5312_final_masked.pngConverted image 2638 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/aca56c01-7699-4b63-ad20-164ffc5af439___RS_HL 0562_final_masked.pngConverted image 2705 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/b84d3dac-ca17-4418-a702-72bf770bddb2___RS_HL 2596_final_masked.pngConverted image 2741 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/bdfe081a-cb47-42a9-a099-0aa24fd902d5___RS_HL 4956_final_masked.pngConverted image 2753 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/c04d0d90-d263-409e-b2c1-bd5f5ba0d539___RS_HL 4983_final_masked.pngConverted image 2847 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/d019c2fb-6d5f-47b9-abb8-bed0759e976c___RS_HL 5032_final_masked.pngConverted image 2885 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/d80889cc-1988-429e-a6d1-ab34eeaeabca___RS_HL 0422_final_masked.pngConverted image 2887 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/d854996e-44ef-4344-9896-f963cd355652___RS_HL 2302_final_masked.pngConverted image 2946 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/e1714c6b-2ebc-42db-81f1-b60c44448316___RS_HL 0375_final_masked.pngConverted image 3300 and saved at /media/data/jacob/Fossil_Project/plant_village/Vitaceae/59ecd16e-8136-4295-93e4-0dc8755f48eb___Mt.N.V_HL 6133_final_masked.pngConverted image 3303 and saved at /media/data/jacob/Fossil_Project/plant_village/Vitaceae/5b2c9b7e-4f3a-4850-bd70-35230df9c794___Mt.N.V_HL 8933_final_masked.pngConverted image 3314 and saved at /media/data/jacob/Fossil_Project/plant_village/Vitaceae/61c3750a-b90a-4d1b-b907-8683be4faa13___Mt.N.V_HL 9084_final_masked.pngConverted image 3354 and saved at /media/data/jacob/Fossil_Project/plant_village/Vitaceae/79ffc83e-aaf8-4706-9e30-da5e0f5b69f3___Mt.N.V_HL 6194_final_masked.pngConverted image 3446 and saved at /media/data/jacob/Fossil_Project/plant_village/Vitaceae/b297f494-eb0a-407f-8edf-25ecdbf278bf___Mt.N.V_HL 9022_final_masked.pngConverted image 3600 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/0d082b09-eca2-4ee5-b7b5-e840590bceac___Rutg._HL 3574_final_masked.pngConverted image 3616 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/1ac02917-7c49-4f14-8d31-9a4f497a9bd0___Rutg._HL 3695_final_masked.pngConverted image 3652 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/35d97a27-1462-4c2a-9e56-840ddb6a034b___Rutg._HL 3509_final_masked.pngConverted image 3665 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/3e2ac2c4-97c0-45bf-8770-9087bbe889b5___Rutg._HL 3745_final_masked.pngConverted image 3696 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/52545cfe-0344-4f21-b24f-834263a7525b___Rutg._HL 2456_final_masked.pngConverted image 3760 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/814d98ab-0c6e-493b-ad41-c7e108ddc851___Rutg._HL 2481_final_masked.pngConverted image 3827 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/b35019c6-ba6a-4d0b-a80f-fa07f9118db6___Rutg._HL 3660_final_masked.pngConverted image 3929 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/fe3bc54d-4164-4fea-a0bd-d96299ab3422___Rutg._HL 3578_final_masked.pngConverted image 3925 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/f9abf8f6-7783-473c-81d2-a84dba013d3e___Rutg._HL 2500_final_masked.pngConverted image 3920 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/f7bc6c3e-4ce1-43c1-95f6-ac8bd4af5b08___Rutg._HL 3610_final_masked.pngConverted image 3928 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/fd40d2b0-c139-4661-973f-7c5ee06a55a2___Rutg._HL 3597_final_masked.pngConverted image 3951 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/22322780-95b4-4b45-b626-26b22965d55e___RS_HL 1880_final_masked.pngConverted image 3952 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/23f1ab77-d664-49c4-92eb-41eb66c6d101___RS_HL 4169_final_masked.pngConverted image 4033 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/ad9b6cf4-e065-496d-91c7-5713ce14e19e___RS_HL 1926_final_masked.pngConverted image 4092 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/04cb725a-632d-44e0-9886-c84e2bbdd3da___RS_HL 2094_final_masked.pngConverted image 4120 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/15bd3182-f1d2-4d75-b54e-9aca616cec66___RS_HL 2077_final_masked.pngConverted image 4194 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/3fef560f-5660-4e47-bc07-7a847ceb6a3f___RS_HL 4740_final_masked.pngConverted image 4253 and saved at /media/data/jacob/Fossil_Project/plant_village/Rosacea/65759427-df44-4ec4-8598-c8bde2d431d7___RS_HL 4424_final_masked.pngConverted image 4574 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/06d90ad5-eb13-49aa-ae04-3a4a5454963d___RS_HL 0620_final_masked.pngConverted image 4778 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/271be296-4c0b-4817-a44b-cc9611d3f519___GH_HL Leaf 182.1_final_masked.pngConverted image 4779 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/273f3e74-dbdc-40bc-8507-3bdd3ce24134___RS_HL 0149_final_masked.pngConverted image 4868 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/338994aa-89ca-4fa5-80ea-9efc94093280___RS_HL 0194_final_masked.pngConverted image 4950 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/414359b9-153b-4985-bfe0-eaf22f7ff6ab___RS_HL 0062_final_masked.pngConverted image 5082 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/54059d2e-8903-4cf1-9856-676620a5be93___GH_HL Leaf 484_final_masked.pngConverted image 5151 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/6193b998-c9ab-46fc-9ca9-249efceb9220___RS_HL 0642_final_masked.pngConverted image 5166 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/62fa284f-8fc8-46b7-a74d-71bada562464___RS_HL 0303_final_masked.pngConverted image 5169 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/6323ad6c-28ea-4050-bc40-310d58715755___RS_HL 0294_final_masked.pngConverted image 5227 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/6be1ec13-6964-4706-b94a-069d4a8592cf___GH_HL Leaf 175.1_final_masked.pngConverted image 5221 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/6a9836cb-f987-4ccf-8841-f227989ff5ec___RS_HL 0045_final_masked.pngConverted image 5225 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/6b9b1ad6-3e75-4378-ad97-7dc9b8fa084e___GH_HL Leaf 179.1_final_masked.pngConverted image 5236 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/6ce07d35-afb2-4290-9239-ef1dcb92acd8___GH_HL Leaf 492.1_final_masked.pngConverted image 5353 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/80c566ba-6754-4abb-a61e-986b92cfc4fd___RS_HL 0436_final_masked.pngConverted image 5428 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/8ad09f74-a565-413a-bcf5-21cf58a4db3a___RS_HL 0173_final_masked.pngConverted image 5470 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/9268e1da-3fc4-49b1-8fcb-3f0d8957114a___GH_HL Leaf 277.1_final_masked.pngConverted image 5472 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/92c06519-cec4-4091-9ae1-5ea8b56c89cd___GH_HL Leaf 525_final_masked.pngConverted image 5482 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/93c9da62-19ca-4e90-b355-0dc71ff5cc4b___RS_HL 9752_final_masked.pngConverted image 5653 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/b427ec05-0b9a-45c6-8148-cbddad66e99d___GH_HL Leaf 516.3_final_masked.pngConverted image 5636 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/b1e628a8-b250-4ee7-9d3f-60f93a6a556b___RS_HL 0148_final_masked.pngConverted image 5647 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/b3813caa-82f2-4b9b-a5ef-a9f0db6f08bf___GH_HL Leaf 466_final_masked.pngConverted image 5642 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/b2652ef9-f0d2-4039-b40a-520073551b16___RS_HL 0009_final_masked.pngConverted image 5728 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/bd531960-92ec-4745-9e25-645decb4b2b0___RS_HL 0542_final_masked.pngConverted image 5769 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/c49577fa-06ce-40f1-ab69-ff21836b5061___GH_HL Leaf 277.2_final_masked.pngConverted image 5881 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/d6128bab-2cc4-4005-9184-2b7303773682___RS_HL 9828_final_masked.pngConverted image 5876 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/d53448cd-ddd7-4dd6-a657-c60bde629130___RS_HL 0190_final_masked.pngConverted image 5875 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/d4fc1abf-fd0d-4625-9882-36acd3755ede___RS_HL 0307_final_masked.pngConverted image 5889 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/d75ed8c7-610e-4abd-8a3a-a08993a92df3___GH_HL Leaf 323.2_final_masked.pngConverted image 5981 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/e59d735a-0312-46e4-943a-2262ea2cd502___RS_HL 0428_final_masked.pngConverted image 5987 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/e6584f1d-9338-4807-a2f2-8c0a70220c22___GH_HL Leaf 239_final_masked.pngConverted image 6010 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/e969c25d-8cc4-4730-a65b-330cd4d38c9a___GH_HL Leaf 427_final_masked.pngConverted image 6098 and saved at /media/data/jacob/Fossil_Project/plant_village/Solanaceae/fb2c7e00-39a7-4be3-b8d1-b4c42c65e20e___RS_HL 9636_final_masked.png\n"
     ]
    }
   ],
   "source": [
    "new_data_locations = {}\n",
    "for dataset_name, rows in data_by_dataset_dict.items():\n",
    "    if dataset_name in ['PNAS','plant_village']:\n",
    "        filepaths = list(rows['path'].values)\n",
    "        labels = list(rows['family'].values)\n",
    "        assert len(filepaths) == len(labels)\n",
    "        print('Starting ', dataset_name, ' with ', len(filepaths), ' files')\n",
    "    #     filepaths = filepaths[:10]\n",
    "    #     labels = labels[:10]\n",
    "\n",
    "        num_files = len(filepaths)\n",
    "        start_time = time.perf_counter()\n",
    "        new_dataset_paths = convert_to_png(filepaths, labels, dataset_name = dataset_name)\n",
    "        end_time = time.perf_counter()\n",
    "        total_time = end_time-start_time\n",
    "        print(f'Finished copying {num_files} from {dataset_name} in {total_time:.3f} seconds at a rate of {num_files/total_time:.3f} images/sec')\n",
    "        new_dataset_paths = list(new_dataset_paths)\n",
    "        new_data_locations.update({dataset_name:new_dataset_paths})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### **4.** Encode labels as integers for feeding into model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyleaves.data_pipeline.preprocessing import encode_labels\n",
    "\n",
    "data_df = encode_labels(data)\n",
    "\n",
    "data_df.sample(frac=1).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(data_df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyleaves.data_pipeline.preprocessing import filter_low_count_labels, one_hot_encode_labels, one_hot_decode_labels\n",
    "\n",
    "test_size = 0.25\n",
    "val_size = 0.25\n",
    "\n",
    "data_df = filter_low_count_labels(data_df, threshold=3, verbose = True)\n",
    "\n",
    "data_df = encode_labels(data_df) #Re-encode numeric labels after removing sub-threshold classes so that max(labels) == len(labels)\n",
    "\n",
    "image_paths = data_df['path'].values.reshape((-1,1))\n",
    "one_hot_labels = one_hot_encode_labels(data_df['label'].values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data, test_data = train_test_split(data_df, test_size=test_size, random_state=random_seed, shuffle=True, stratify=data_df['label'])\n",
    "train_paths, test_paths, train_labels, test_labels  = train_test_split(image_paths, one_hot_labels, test_size=test_size, random_state=random_seed, shuffle=True, stratify=data_df['label'])\n",
    "\n",
    "train_paths, val_paths, train_labels, val_labels = train_test_split(train_paths, train_labels, test_size=val_size, random_state=random_seed, shuffle=True, stratify=train_labels)\n",
    "\n",
    "\n",
    "train_data = {'path': train_paths, 'label': train_labels}\n",
    "val_data = {'path': val_paths, 'label': val_labels}\n",
    "test_data = {'path': test_paths, 'label': test_labels}\n",
    "\n",
    "data_splits = {'train': train_data,\n",
    "              'val': val_data,\n",
    "              'test': test_data}\n",
    "\n",
    "# train_gen = get_multiprocess_dataflow(train_data['path'], train_data['label'], size=(299,299), batch_size=32, num_prefetch=25, num_proc=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **Let's set up our model**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_class_frequencies = leavesdb.utils.plot_class_frequencies\n",
    "    \n",
    "plot_class_frequencies(labels=one_hot_decode_labels(train_data['label']).ravel().tolist());\n",
    "plot_class_frequencies(labels=one_hot_decode_labels(val_data['label']).ravel().tolist());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_classes = len(np.unique(data_df['label']))\n",
    "img_size = [299,299]\n",
    "channels = 3\n",
    "batch_size = 32\n",
    "learning_rate=0.01\n",
    "num_epochs = 1\n",
    "\n",
    "def parse_function(filename, label):\n",
    "    img = tf.io.read_file(filename)\n",
    "    img = tf.io.decode_jpeg(img, channels=channels)#, dtype=tf.float32)\n",
    "    img = tf.image.resize(img, img_size)\n",
    "    return img, label #{'image':img, 'label':label}\n",
    "\n",
    "# def train_preprocess(img, label):\n",
    "#     img = tf.image.resize(img, img_size)\n",
    "#     return {'image':img, 'label':label}\n",
    "    \n",
    "\n",
    "def get_tf_dataset(filenames, labels):\n",
    "    data = tf.data.Dataset.from_tensor_slices((filenames, labels))\n",
    "    data = data.shuffle(len(filenames))\n",
    "#     data = data.interleave((lambda x, y: tf.data.Dataset(x,y).map(parse_function, num_parallel_calls=1)), cycle_length=4, block_length=16)\n",
    "    data = data.map(parse_function, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
    "#     data = data.map(train_preprocess, num_parallel_calls=4)\n",
    "    data = data.batch(batch_size)\n",
    "    data = data.prefetch(tf.data.experimental.AUTOTUNE)\n",
    "#     data = data.apply(tf.data.experimental.prefetch_to_device('/device:GPU:0'))\n",
    "    return data\n",
    "\n",
    "##############################\n",
    "\n",
    "\n",
    "def debug_parse_function(filename, label):\n",
    "    img = tf.io.read_file(filename)\n",
    "    img = tf.io.decode_jpeg(img, channels=channels)#, dtype=tf.float32)\n",
    "    img = tf.image.resize(img, img_size)\n",
    "    return img, label, filename #{'image':img, 'label':label}\n",
    "\n",
    "\n",
    "\n",
    "def debug_get_tf_dataset(filenames, labels):\n",
    "    data = tf.data.Dataset.from_tensor_slices((filenames, labels))\n",
    "    data = data.shuffle(len(filenames))\n",
    "#     data = data.interleave((lambda x, y: tf.data.Dataset(x,y).map(parse_function, num_parallel_calls=1)), cycle_length=4, block_length=16)\n",
    "    data = data.map(debug_parse_function, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
    "#     data = data.map(train_preprocess, num_parallel_calls=4)\n",
    "    data = data.batch(batch_size)\n",
    "    data = data.prefetch(tf.data.experimental.AUTOTUNE)\n",
    "    data = data.cache()\n",
    "#     data = data.apply(tf.data.experimental.prefetch_to_device('/device:GPU:0'))\n",
    "    return data\n",
    "\n",
    "##############################\n",
    "\n",
    "debug = False#True\n",
    "\n",
    "if debug == True:\n",
    "    get_tf_dataset = debug_get_tf_dataset\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def decode_labels(data_df):\n",
    "    data_df=data_df.groupby('label', group_keys=False).apply(lambda df: df.sample(1).loc[:,['label','family']])\n",
    "    data_df.sort_values(by='label', inplace=True)\n",
    "    data_df.set_index(keys='label',drop=True, inplace=True)\n",
    "    data_df = data_df.to_dict()\n",
    "    \n",
    "    return data_df['family']\n",
    "\n",
    "\n",
    "# train_dataset = get_tf_dataset(filenames = train_data['path'].values, labels = train_data['label'].values)\n",
    "# val_dataset = get_tf_dataset(filenames = val_data['path'].values, labels = val_data['label'].values)\n",
    "\n",
    "train_dataset = get_tf_dataset(filenames = train_data['path'].ravel(), labels = train_data['label'])\n",
    "val_dataset = get_tf_dataset(filenames = val_data['path'].ravel(), labels = val_data['label'])\n",
    "\n",
    "label_map = decode_labels(data_df=data_df)\n",
    "\n",
    "num_samples_train = len(train_data['path'])\n",
    "num_samples_val = len(val_data['path'])\n",
    "num_samples_test = len(test_data['path'])\n",
    "print(num_samples_train)\n",
    "print(num_samples_val)\n",
    "print(num_samples_test)\n",
    "\n",
    "label_map[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for features in  train_dataset.take(1):\n",
    "#     image_batch = features[0].numpy().astype(np.int)\n",
    "#     label_batch = features[1].numpy().astype(np.int)\n",
    "\n",
    "# plot_image_grid = pyleaves.analysis.img_utils.plot_image_grid\n",
    "    \n",
    "    \n",
    "# plot_image_grid(image_batch, label_batch, x_plots = 4, y_plots = 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#############################################################################\n",
    "\n",
    "\n",
    "# datagen = tf.keras.preprocessing.image.ImageDataGenerator(rescale=1.0/255.0)\n",
    "\n",
    "# train_data['label'] = train_data['label'].astype(str)\n",
    "\n",
    "# datagen_flow = datagen.flow_from_dataframe(train_data.iloc[:100,:], x_col='path', y_col='label', class_mode='sparse', batch_size=batch_size)\n",
    "\n",
    "# a=next(datagen_flow)\n",
    "\n",
    "# print(a[0].shape, a[1].shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import time\n",
    "\n",
    "\n",
    "# start_time = time.time()\n",
    "# n=100\n",
    "# total_time = 0\n",
    "# try:\n",
    "#     for i, features in enumerate(train_dataset.take(n)):\n",
    "# #         print(i, features[0].shape, features[1].shape)\n",
    "#         run_time = time.time()-start_time\n",
    "#         total_time += run_time\n",
    "#         print(f'Took {run_time:.2f} seconds')\n",
    "#         start_time = time.time()\n",
    "# except Exception as e:\n",
    "#     print(e)\n",
    "#     print(f'finished {i} iterations')\n",
    "\n",
    "# avg_time = total_time / i+1\n",
    "\n",
    "# rate = (i+1)*batch_size/total_time\n",
    "\n",
    "# print(f'Avg time = {avg_time:.2f} | Ran {i+1} iterations using batch size = {batch_size} & {batch_size*n} samples')\n",
    "# print(f'rate = {rate}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = pyleaves.models.inception_v3.build_model(num_classes, learning_rate=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# history = pyleaves.models.inception_v3.train_model(model,\n",
    "# \t\t\t\ttrain_dataset,\n",
    "# \t\t\t\tvalidation_data=val_dataset, \n",
    "# \t\t\t\tsteps_per_epoch=int(num_samples_train//batch_size),\n",
    "# \t\t\t\tvalidation_steps=int(num_samples_val//batch_size),\n",
    "# \t\t\t\tmax_epochs=num_epochs,\n",
    "# # \t\t\t\tcallbacks=None,\n",
    "# \t\t\t\tworkers=5,\n",
    "# \t\t\t\tinitial_epoch=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_dataset = train_dataset.make_initializable_iterator().get_next()\n",
    "# val_dataset = val_dataset.make_initializable_iterator().get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# current_epoch = 0\n",
    "# while current_epoch < 20:\n",
    "#     try:\n",
    "#         history = model.fit(\n",
    "#                         train_dataset,\n",
    "#                         validation_data=val_dataset, \n",
    "#         #                 steps_per_epoch=int(num_samples_train//batch_size),\n",
    "#         #                 validation_steps=int(num_samples_val//batch_size),\n",
    "#                         epochs=num_epochs,\n",
    "#                         # \t\t\t\tcallbacks=None,\n",
    "#                         workers=10,\n",
    "#                         initial_epoch=current_epoch,\n",
    "#                         verbose=1)\n",
    "#     except KeyboardInterrupt:\n",
    "#         break\n",
    "#     except Exception as e:\n",
    "#         print(f'current epoch = {current_epoch}, error: {e}')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read_data = tf.get_default_session().run(train_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from tensorflow.python.framework.errors_impl import InvalidArgumentError\n",
    "\n",
    "\n",
    "filename_ids = []\n",
    "batch_log = []\n",
    "invalid_filenames = []\n",
    "\n",
    "start_time = time.time()\n",
    "time_log = []\n",
    "steps_per_epoch = num_samples_train//batch_size\n",
    "\n",
    "valid_filenames = []\n",
    "\n",
    "reset_iter = True\n",
    "current_epoch = 0\n",
    "while current_epoch < 20:\n",
    "    if reset_iter == True:\n",
    "        epoch_dataset = train_dataset.take(steps_per_epoch)\n",
    "        reset_iter = False\n",
    "    try:\n",
    "        for i, (imgs, labels, filenames) in enumerate(epoch_dataset):        \n",
    "            run_time = time.time()-start_time\n",
    "            time_log.append(run_time)\n",
    "            print(f'Took {run_time:.2f} seconds')\n",
    "            \n",
    "            valid_filenames.append([fname.numpy().decode('utf-8') for fname in filenames])\n",
    "            \n",
    "            start_time = time.time()\n",
    "    except InvalidArgumentError as e:\n",
    "        invalid_flag = 0\n",
    "        for j, fname in enumerate(filenames):\n",
    "            fname = fname.numpy().decode('utf-8')\n",
    "            if os.path.isfile(fname):\n",
    "                filename_ids.append(i*batch_size+j)\n",
    "                valid_filenames.append(fname)\n",
    "                continue\n",
    "            else:\n",
    "                filename_ids.append(i*batch_size+j)\n",
    "                invalid_filenames.append(fname)    \n",
    "                print(f'invalid filename = {fname}')\n",
    "                invalid_flag = 1\n",
    "        print(f'current epoch = {current_epoch}, error: {e}', type(e))\n",
    "        continue\n",
    "\n",
    "    except KeyboardInterrupt:\n",
    "        break\n",
    "    except Exception as e:\n",
    "        reset_iter = True\n",
    "        print(f'current epoch = {current_epoch}, error: {e}', type(e))\n",
    "            \n",
    "print(f'finished {i*batch_size} samples over {i} iterations in {np.sum(time_log):.2f} seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(invalid_filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "\n",
    "plot_image_grid = pyleaves.analysis.img_utils.plot_image_grid\n",
    "\n",
    "invalid_filenames = np.concatenate([fname.numpy().tolist() for fname in invalid_filenames]).tolist()\n",
    "\n",
    "for i, fname in enumerate(invalid_filenames):\n",
    "    if type(fname)==bytes:\n",
    "        invalid_filenames[i] = fname.decode('utf-8')    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "invalid_imgs = []\n",
    "for i, fname in enumerate(invalid_filenames):\n",
    "    img = cv2.imread(fname)\n",
    "    img = cv2.resize(img, tuple(img_size))\n",
    "    invalid_imgs.append(img)\n",
    "\n",
    "    \n",
    "invalid_images = np.stack(invalid_imgs)\n",
    "\n",
    "plot_image_grid(invalid_images, labels = np.ones(len(invalid_imgs)), x_plots = 4, y_plots = 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}