cw_tiler_Atlanta-V3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demo of CW-Tiler - Atlanta - V3\n",
    "\n",
    "This is a walk through for creation of the Atlantas Dataset.  \n",
    "\n",
    "This demo will walk through the use of the tiler to create utm tiles with the SpaceNet Data Repository.  We will be taking advantage of cloud optimized geotiffs.  \n",
    "\n",
    "For more information about SpaceNet visit https://spacenetchallenge.github.io/\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import base tools\n",
    "\n",
    "## Note, for mac osx compatability import something from shapely.geometry before importing fiona or geopandas\n",
    "## https://github.com/Toblerity/Shapely/issues/553  * Import shapely before rasterio or fioana\n",
    "from shapely import geometry\n",
    "import rasterio\n",
    "import random\n",
    "from cw_tiler import main\n",
    "from cw_tiler import utils\n",
    "from cw_tiler import vector_utils\n",
    "import numpy as np\n",
    "import geopandas as gpd\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "# Setting Certificate Location for Ubuntu/Mac OS locations (Rasterio looks for certs in centos locations)  This is important for s3 access  (s3 must be configured using aws cli / boto3)\n",
    "os.environ['CURL_CA_BUNDLE']='/etc/ssl/certs/ca-certificates.crt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "from stac_tools.stac_item import spacenetStacItem\n",
    "import glob\n",
    "import os\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create List of Images for Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from stac_tools.stac_item import spacenetStacItem\n",
    "import glob\n",
    "import os\n",
    "baseLocation = \"/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/srcData/rasterData/\"\n",
    "workorder_list = glob.glob(os.path.join(baseLocation, '**', \"*.tif\"), recursive=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "spacenetImageList = []\n",
    "imageDictList = []\n",
    "stac_list = []\n",
    "from shapely.geometry import shape \n",
    "\n",
    "for workorder_item in workorder_list:\n",
    "    item_basename = os.path.splitext(os.path.basename(workorder_item))[0]\n",
    "    item_folder = os.path.dirname(workorder_item)\n",
    "    raster_path = workorder_item\n",
    "    image_id = os.path.basename(os.path.dirname(item_folder))\n",
    "    ## Note adding extra insertion\n",
    "    image_id = image_id.replace('nadir', 'nadir_')\n",
    "    xml_path = os.path.join(item_folder, item_basename+\".XML\")\n",
    "    imd_path = os.path.join(item_folder, item_basename+\".IMD\")\n",
    "    \n",
    "\n",
    "    \n",
    "    item = spacenetStacItem(raster_path, \"DigitalGlobe\", \"CC 4.0 BY-SA\", 'ID1', assetDict=[], imdPath=xml_path)\n",
    "    #print(shape(item.geometry).area/(1000*1000))\n",
    "    imageDict = {'image_id': image_id, \n",
    "                 'image_path': workorder_item,\n",
    "                 'xml_path': xml_path, 'imd_path': imd_path,\n",
    "                'image_type': item_basename.rsplit('_',1)[1],\n",
    "                'image_area':shape(item.geometry).area/(1000*1000),\n",
    "                'geometry': shape(item.geometry)}\n",
    "    imageDictList.append(imageDict)\n",
    "    \n",
    "    \n",
    "    \n",
    "    stac_list.append(item)\n",
    "\n",
    "print(len(imageDictList))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import geopandas as gpd\n",
    "imageDF = gpd.GeoDataFrame(imageDictList)\n",
    "\n",
    "\n",
    "imageDF.head()\n",
    "imageDF.loc[imageDF['image_type']=='assembly','image_type']='Pan-Sharpen'\n",
    "imageDF[imageDF['image_id']=='rasterData']['image_path'].values\n",
    "\n",
    "imageDF.to_file(\"/raid/nfs/workingDir/cw-tiler/Atlanta_data.shp\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create Vector File Atlanta Dataset for tiling extent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# open s3 Location\n",
    "### This Shapefile will be fed to cw-tiler to create tiles.  \n",
    "tile_index_shp = '/raid/nfs/workingDir/cw-tiler/AOI_6_Atlanta_Cells_v1.shp'\n",
    "\n",
    "# Each grid starting point will be spaced 400m apart\n",
    "stride_size_meters = 450\n",
    "\n",
    "# Each grid cell will be 400m on a side\n",
    "cell_size_meters   = 450\n",
    "\n",
    "# Specify the number of pixels in a tile cell_size_meters/tile_size_pixels == Pixel_Size_Meters\n",
    "tile_size_pixels   = 900\n",
    "\n",
    "# Generate list of cells to read from utm_bounds \n",
    "cells_list = main.calculate_analysis_grid(aoi_geom.bounds, stride_size_meters=stride_size_meters, cell_size_meters=cell_size_meters)\n",
    "\n",
    "## \n",
    "cells_list = cells_list[0]\n",
    "    \n",
    "# Convert to Shapefile\n",
    "#shapely.geometry.box(minx, miny, maxx, maxy, ccw=True)\n",
    "from shapely import geometry\n",
    "geom_list = [geometry.box(*cell_selection) for cell_selection in cells_list]\n",
    "cell_gdf = gpd.GeoDataFrame(geometry=geom_list)\n",
    "cell_gdf.crs = utm_crs\n",
    "cell_gdf.to_file(tile_index_shp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "## Read List of Images into GeoDataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import geopandas as gpd\n",
    "\n",
    "imageDF = gpd.read_file(\"/raid/nfs/workingDir/cw-tiler/tmpGeoFiles/Atlanta_data.shp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['image_area', 'image_id', 'image_path', 'image_type', 'imd_path',\n",
       "       'xml_path', 'geometry'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imageDF.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Images = (81, 7)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_area</th>\n",
       "      <th>image_id</th>\n",
       "      <th>image_path</th>\n",
       "      <th>image_type</th>\n",
       "      <th>imd_path</th>\n",
       "      <th>xml_path</th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>529.597357</td>\n",
       "      <td>Atlanta_nadir14_catid_10300100039AB000</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>PAN</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>POLYGON ((750151.0046244338 3715733.306822867,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>529.577299</td>\n",
       "      <td>Atlanta_nadir14_catid_10300100039AB000</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>MS</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>POLYGON ((750148.3462557473 3715732.979846669,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>529.597357</td>\n",
       "      <td>Atlanta_nadir14_catid_10300100039AB000</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>Pan-Sharpen</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>POLYGON ((750151.0046244338 3715733.306822867,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>963.246923</td>\n",
       "      <td>Atlanta_nadir39_catid_1030010003832800</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>Pan-Sharpen</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>POLYGON ((753426.2862091018 3713963.090985105,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>963.246923</td>\n",
       "      <td>Atlanta_nadir39_catid_1030010003832800</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>PAN</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>/raid/nfs/data/Datasets/CosmiQ_General_Study/A...</td>\n",
       "      <td>POLYGON ((753426.2862091018 3713963.090985105,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   image_area                                image_id  \\\n",
       "0  529.597357  Atlanta_nadir14_catid_10300100039AB000   \n",
       "1  529.577299  Atlanta_nadir14_catid_10300100039AB000   \n",
       "2  529.597357  Atlanta_nadir14_catid_10300100039AB000   \n",
       "3  963.246923  Atlanta_nadir39_catid_1030010003832800   \n",
       "4  963.246923  Atlanta_nadir39_catid_1030010003832800   \n",
       "\n",
       "                                          image_path   image_type  \\\n",
       "0  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...          PAN   \n",
       "1  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...           MS   \n",
       "2  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...  Pan-Sharpen   \n",
       "3  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...  Pan-Sharpen   \n",
       "4  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...          PAN   \n",
       "\n",
       "                                            imd_path  \\\n",
       "0  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...   \n",
       "1  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...   \n",
       "2  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...   \n",
       "3  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...   \n",
       "4  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...   \n",
       "\n",
       "                                            xml_path  \\\n",
       "0  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...   \n",
       "1  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...   \n",
       "2  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...   \n",
       "3  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...   \n",
       "4  /raid/nfs/data/Datasets/CosmiQ_General_Study/A...   \n",
       "\n",
       "                                            geometry  \n",
       "0  POLYGON ((750151.0046244338 3715733.306822867,...  \n",
       "1  POLYGON ((750148.3462557473 3715732.979846669,...  \n",
       "2  POLYGON ((750151.0046244338 3715733.306822867,...  \n",
       "3  POLYGON ((753426.2862091018 3713963.090985105,...  \n",
       "4  POLYGON ((753426.2862091018 3713963.090985105,...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Total Images = {}\".format(imageDF.shape))\n",
    "imageDF.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create JSON Work order Files for Processing Raw Imagery into Tiles.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "### \n",
    "# Feed index file to chip all other images into.  The Shape File has a set of cells.  CW-Tiler will clip imagery to each cell extent.  \n",
    "tile_index_shp = '/raid/nfs/workingDir/cw-tiler/tmpGeoFiles/AOI_6_Atlanta_Cells_v1.shp'\n",
    "\n",
    "# Pick location for Cells\n",
    "results_data_base_location = '/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav4/'\n",
    "\n",
    "## Location to write Command Script to:\n",
    "command_script_location = '/raid/nfs/workingDir/cw-tiler/Atlanta_AOI_parallelScript_v1.sh'\n",
    "\n",
    "import json\n",
    "commandScript = []\n",
    "## Python promt calling cw_tiler_AOI_6_Atlanta.py to process\n",
    "pythonCommand = 'python cw_tiler_AOI_6_Atlanta.py {}'\n",
    "## Iterate through each image to create the work order command.  This includes the creation of a json file with the details of the work order.  \n",
    "for idx, image_row in imageDF.iterrows():\n",
    "        workorder = image_row.to_dict()\n",
    "        workorder['geometry']=workorder['geometry'].to_wkt()\n",
    "        if workorder['image_type'] == 'Pan-Sharpen':\n",
    "            ## Pan Sharpen tile should bee 900 x 900px\n",
    "            workorder['tile_pixel_size'] = 900\n",
    "        elif workorder['image_type'] == 'PAN':\n",
    "            ## Pan  tile should bee 900 x 900px\n",
    "            workorder['tile_pixel_size'] = 900\n",
    "        elif workorder['image_type'] == 'MS':\n",
    "            ## Pan  tile should bee 225 x 225px\n",
    "            workorder['tile_pixel_size'] = 225\n",
    "            \n",
    "        workorder['tile_index_shp'] = tile_index_shp\n",
    "        workorder['results_data_base_location'] = results_data_base_location\n",
    "        \n",
    "        ## Worker Orders should work on a _cog for quicker tiling.  This is not required\n",
    "        workorder['image_path'] = workorder['image_path'].replace('.tif', '_cog.tif')\n",
    "        \n",
    "        ## Save Work order\n",
    "        workorder_location = os.path.join(results_data_base_location, \"{}_{}_v1.json\".format(workorder['image_id'], workorder['image_type']))\n",
    "        \n",
    "        #print(workorder_location)\n",
    "        with open(workorder_location, 'w') as fp:\n",
    "            json.dump(workorder, fp, indent=4)\n",
    "            \n",
    "        commandScript.append(pythonCommand.format(workorder_location))\n",
    "\n",
    "with open(command_script_location, 'w') as fp:\n",
    "    \n",
    "    fp.writelines([\"{}\\n\".format(item)  for item in commandScript])\n",
    "        \n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Create Train /Test / Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import glob\n",
    "tile_index_shp = '/raid/nfs/workingDir/cw-tiler/AOI_6_Atlanta_Cells_v1.shp'\n",
    "results_data_base_location = '/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav4/'\n",
    "\n",
    "results = glob.glob(os.path.join(results_data_base_location, '**', '*.shp'), recursive=True)\n",
    "## Each Imagery Workfile generates a ShapeFile with results.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav2/Atlanta_nadir14_catid_10300100039AB000/Pan-Sharpenworkorder.shp'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Process Vector File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "import geopandas as gpd\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "baseIndexFile = gpd.read_file(tile_index_shp)\n",
    "baseIndexFile1 = baseIndexFile.copy()\n",
    "\n",
    "## Analyze Results to check all tiles are made.  \n",
    "for result in results:\n",
    "    if os.path.basename(result) == 'Pan-Sharpenworkorder.shp':\n",
    "        tmpFile = gpd.read_file(result)\n",
    "        idName = os.path.basename(os.path.dirname(result))\n",
    "        tmpFile[idName] = tmpFile['result']\n",
    "        tmpFile = tmpFile.drop(columns=['result', 'geometry'])\n",
    "        baseIndexFile = pd.merge(baseIndexFile, tmpFile, how='outer', on='FID')\n",
    "    if os.path.basename(result) == 'spacenet-buildingsworkorder.shp':\n",
    "        tmpFile = gpd.read_file(result)\n",
    "        #Index(['FID', 'object_cou', 'object_are', 'geometry'], dtype='object')\n",
    "        idName = 'spacenet-buildings'\n",
    "        tmpFile['spacenet-buildings_count'] = tmpFile['object_cou']\n",
    "        tmpFile['spacenet-buildings_area'] = tmpFile['object_are']\n",
    "        tmpFile = tmpFile.drop(columns=['object_cou', 'object_are','geometry'])\n",
    "        baseIndexFile = pd.merge(baseIndexFile, tmpFile, how='outer', on='FID')\n",
    "    \n",
    "    if os.path.basename(result) == 'spacenet-tile-idworkorder.shp':\n",
    "        tmpFile = gpd.read_file(result)\n",
    "        #Index(['FID', 'object_cou', 'object_are', 'geometry'], dtype='object')\n",
    "        tmpFile['label_suffix'] = tmpFile['tile_index']\n",
    "        tmpFile['image_suffix'] = tmpFile['tile_ind_1']\n",
    "        tmpFile = tmpFile.drop(columns=['tile_index', 'tile_ind_1','geometry'])\n",
    "        baseIndexFile = pd.merge(baseIndexFile, tmpFile, how='outer', on='FID')\n",
    "        \n",
    "\n",
    "baseIndexFile.crs = baseIndexFile1.crs\n",
    "#baseIndexFile.to_file('/raid/nfs/workingDir/cw-tiler/AOI_6_Atlanta_PAN_details.geojson', driver='GeoJSON')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "baseIndexFile.head()\n",
    "baseIndexFile['_TotalCount'] = baseIndexFile.drop(columns=['FID', 'geometry', 'spacenet-buildings_count', 'spacenet-buildings_area', 'label_suffix', 'image_suffix']).sum(numeric_only=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Tiles Spatially = 2318\n",
      "Total Tiles Total = 62170\n",
      "Train Tiles = 1159.0, 50.0% of Tiles\n",
      "Test Tiles = 927.2, 40.0% of Tiles\n",
      "Validate Tiles = 231.8, 10.0% of Tiles\n"
     ]
    }
   ],
   "source": [
    "#Split Train/Test/Validate\n",
    "TrainSplit = .50\n",
    "TestSplit  = .40\n",
    "ValidateSplit = .10\n",
    "totalTiles = np.sum(np.sum(baseIndexFile.drop(columns=['FID', 'geometry', \"_TotalCount\"])==0))\n",
    "totalSpatialTiles = baseIndexFile.shape[0]\n",
    "print(\"Total Tiles Spatially = {}\".format(totalSpatialTiles))\n",
    "print(\"Total Tiles Total = {}\".format(totalTiles))\n",
    "print(\"Train Tiles = {}, {}% of Tiles\".format(totalSpatialTiles*TrainSplit, TrainSplit*100))\n",
    "print(\"Test Tiles = {}, {}% of Tiles\".format(totalSpatialTiles*TestSplit*1, TestSplit*100))\n",
    "print(\"Validate Tiles = {}, {}% of Tiles\".format(totalSpatialTiles*ValidateSplit, ValidateSplit*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Train    2090\n",
       "Test      228\n",
       "Name: Category, dtype: int64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Assign all tiles to Train by default\n",
    "baseIndexFile['Category'] = 'Train'\n",
    "## Assign All Tiles that have a Total Count of missing Tiles >0 To Test  This is only neccessary because all imagery does not have the same extent.  \\\n",
    "## Therefore along the edges there are some nadir angles that don't have complete tiles.  These are passed to test because we only need one tile from each\n",
    "baseIndexFile.loc[baseIndexFile['_TotalCount']>0, 'Category']='Test'\n",
    "baseIndexFile['Category'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "699\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Train    1391\n",
       "Test      927\n",
       "Name: Category, dtype: int64"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Assign more tiles to Test based on the requires split\n",
    "TestIndex = np.random.choice(baseIndexFile[baseIndexFile['Category']=='Train']['FID'].values, np.int(np.round(totalSpatialTiles*TestSplit - float(baseIndexFile['Category'].value_counts()['Test']))), replace=False)\n",
    "print(len(TestIndex))    \n",
    "baseIndexFile.iloc[TestIndex, baseIndexFile.columns.get_loc(\"Category\")]='Test'\n",
    "baseIndexFile['Category'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "231\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Train       1160\n",
       "Test         927\n",
       "Validate     231\n",
       "Name: Category, dtype: int64"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Assign Tiles to Validate\n",
    "TestIndex = np.random.choice(baseIndexFile[baseIndexFile['Category']=='Train']['FID'].values, int(totalSpatialTiles*ValidateSplit), replace=False)\n",
    "print(len(TestIndex))    \n",
    "baseIndexFile.iloc[TestIndex, baseIndexFile.columns.get_loc(\"Category\")]='Validate'\n",
    "baseIndexFile['Category'].value_counts()\n",
    "#baseIndexFile.to_file(\"/raid/nfs/workingDir/cw-tiler/AOI_6_Atlanta_Train_Test_Validate_Split.geojson\", driver='GeoJSON')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "309\n",
      "309\n",
      "309\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Train                  1160\n",
       "Test_Very-Off-Nadir     309\n",
       "Test_Nadir              309\n",
       "Test_Off-Nadir          309\n",
       "Validate                231\n",
       "Name: Category, dtype: int64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Split Test into 3 categories based on Nadir Angle\n",
    "TestIndex = np.random.choice(baseIndexFile[baseIndexFile['Category']=='Test']['FID'].values, int(totalSpatialTiles*TestSplit/3), replace=False)\n",
    "print(len(TestIndex))    \n",
    "baseIndexFile.iloc[TestIndex, baseIndexFile.columns.get_loc(\"Category\")]='Test_Very-Off-Nadir'\n",
    "TestIndex = np.random.choice(baseIndexFile[baseIndexFile['Category']=='Test']['FID'].values, int(totalSpatialTiles*TestSplit/3), replace=False)\n",
    "print(len(TestIndex))    \n",
    "baseIndexFile.iloc[TestIndex, baseIndexFile.columns.get_loc(\"Category\")]='Test_Off-Nadir'\n",
    "print(len(TestIndex))    \n",
    "baseIndexFile.loc[baseIndexFile['Category']=='Test', \"Category\"]='Test_Nadir'\n",
    "baseIndexFile['Category'].value_counts()\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Write Resulting geojson File\n",
    "baseIndexFile.to_file(\"/raid/nfs/workingDir/cw-tiler/AOI_6_Atlanta_Train_Test_Validate_Split_V3.geojson\", driver='GeoJSON')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>Category</th>\n",
       "      <th colspan=\"8\" halign=\"left\">spacenet-buildings_count</th>\n",
       "      <th colspan=\"8\" halign=\"left\">spacenet-buildings_area</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Test_Nadir</td>\n",
       "      <td>309.0</td>\n",
       "      <td>51.912621</td>\n",
       "      <td>45.281251</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>205.0</td>\n",
       "      <td>309.0</td>\n",
       "      <td>16977.266444</td>\n",
       "      <td>14795.469159</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6408.938288</td>\n",
       "      <td>13790.524743</td>\n",
       "      <td>23214.448014</td>\n",
       "      <td>82848.486821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Test_Off-Nadir</td>\n",
       "      <td>309.0</td>\n",
       "      <td>57.877023</td>\n",
       "      <td>49.052428</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>86.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>309.0</td>\n",
       "      <td>18902.260714</td>\n",
       "      <td>17435.747904</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8464.256954</td>\n",
       "      <td>16179.726708</td>\n",
       "      <td>25075.588407</td>\n",
       "      <td>197721.695840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Test_Very-Off-Nadir</td>\n",
       "      <td>309.0</td>\n",
       "      <td>55.181230</td>\n",
       "      <td>47.517524</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>44.0</td>\n",
       "      <td>81.0</td>\n",
       "      <td>207.0</td>\n",
       "      <td>309.0</td>\n",
       "      <td>16331.746063</td>\n",
       "      <td>14090.254241</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7521.933113</td>\n",
       "      <td>13595.710858</td>\n",
       "      <td>21080.191473</td>\n",
       "      <td>95643.674119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Train</td>\n",
       "      <td>1160.0</td>\n",
       "      <td>59.581034</td>\n",
       "      <td>52.583886</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>297.0</td>\n",
       "      <td>1160.0</td>\n",
       "      <td>18145.884553</td>\n",
       "      <td>15339.270785</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7681.139083</td>\n",
       "      <td>15153.943630</td>\n",
       "      <td>24286.449631</td>\n",
       "      <td>154871.791252</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Validate</td>\n",
       "      <td>231.0</td>\n",
       "      <td>57.891775</td>\n",
       "      <td>49.816459</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>48.0</td>\n",
       "      <td>84.5</td>\n",
       "      <td>201.0</td>\n",
       "      <td>231.0</td>\n",
       "      <td>17714.313251</td>\n",
       "      <td>14370.438253</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8356.967146</td>\n",
       "      <td>15452.977571</td>\n",
       "      <td>23996.264816</td>\n",
       "      <td>106009.227127</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Category spacenet-buildings_count                             \\\n",
       "                                          count       mean        std  min   \n",
       "0           Test_Nadir                    309.0  51.912621  45.281251  0.0   \n",
       "1       Test_Off-Nadir                    309.0  57.877023  49.052428  0.0   \n",
       "2  Test_Very-Off-Nadir                    309.0  55.181230  47.517524  0.0   \n",
       "3                Train                   1160.0  59.581034  52.583886  0.0   \n",
       "4             Validate                    231.0  57.891775  49.816459  0.0   \n",
       "\n",
       "                           spacenet-buildings_area                \\\n",
       "    25%   50%   75%    max                   count          mean   \n",
       "0  15.0  42.0  75.0  205.0                   309.0  16977.266444   \n",
       "1  18.0  49.0  86.0  222.0                   309.0  18902.260714   \n",
       "2  15.0  44.0  81.0  207.0                   309.0  16331.746063   \n",
       "3  15.0  46.0  93.0  297.0                  1160.0  18145.884553   \n",
       "4  16.0  48.0  84.5  201.0                   231.0  17714.313251   \n",
       "\n",
       "                                                                              \n",
       "            std  min          25%           50%           75%            max  \n",
       "0  14795.469159  0.0  6408.938288  13790.524743  23214.448014   82848.486821  \n",
       "1  17435.747904  0.0  8464.256954  16179.726708  25075.588407  197721.695840  \n",
       "2  14090.254241  0.0  7521.933113  13595.710858  21080.191473   95643.674119  \n",
       "3  15339.270785  0.0  7681.139083  15153.943630  24286.449631  154871.791252  \n",
       "4  14370.438253  0.0  8356.967146  15452.977571  23996.264816  106009.227127  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "baseIndexFile.groupby(\"Category\")['spacenet-buildings_count', 'spacenet-buildings_area'].describe().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FID</th>\n",
       "      <th>geometry_x</th>\n",
       "      <th>result</th>\n",
       "      <th>geometry_y</th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>POLYGON ((732701 3719289, 732251 3719289, 7322...</td>\n",
       "      <td>1</td>\n",
       "      <td>POLYGON ((732701 3719289, 732251 3719289, 7322...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>POLYGON ((732701 3719739, 732251 3719739, 7322...</td>\n",
       "      <td>0</td>\n",
       "      <td>POLYGON ((732701 3719739, 732251 3719739, 7322...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>POLYGON ((732701 3720189, 732251 3720189, 7322...</td>\n",
       "      <td>0</td>\n",
       "      <td>POLYGON ((732701 3720189, 732251 3720189, 7322...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>POLYGON ((732701 3720639, 732251 3720639, 7322...</td>\n",
       "      <td>0</td>\n",
       "      <td>POLYGON ((732701 3720639, 732251 3720639, 7322...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>POLYGON ((732701 3721089, 732251 3721089, 7322...</td>\n",
       "      <td>0</td>\n",
       "      <td>POLYGON ((732701 3721089, 732251 3721089, 7322...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   FID                                         geometry_x  result  \\\n",
       "0    0  POLYGON ((732701 3719289, 732251 3719289, 7322...       1   \n",
       "1    1  POLYGON ((732701 3719739, 732251 3719739, 7322...       0   \n",
       "2    2  POLYGON ((732701 3720189, 732251 3720189, 7322...       0   \n",
       "3    3  POLYGON ((732701 3720639, 732251 3720639, 7322...       0   \n",
       "4    4  POLYGON ((732701 3721089, 732251 3721089, 7322...       0   \n",
       "\n",
       "                                          geometry_y  test  \n",
       "0  POLYGON ((732701 3719289, 732251 3719289, 7322...     1  \n",
       "1  POLYGON ((732701 3719739, 732251 3719739, 7322...     0  \n",
       "2  POLYGON ((732701 3720189, 732251 3720189, 7322...     0  \n",
       "3  POLYGON ((732701 3720639, 732251 3720639, 7322...     0  \n",
       "4  POLYGON ((732701 3721089, 732251 3721089, 7322...     0  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Read Final Tile Split details\n",
    "\n",
    "final_Tile_Split = gpd.read_file(\"/raid/nfs/workingDir/cw-tiler/AOI_6_Atlanta_Train_Test_Validate_Split_V4.geojson\", driver='GeoJSON')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train                  1160\n",
      "Test_Very-Off-Nadir     309\n",
      "Test_Nadir              309\n",
      "Test_Off-Nadir          309\n",
      "Validate                231\n",
      "Name: Category, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label_suffix</th>\n",
       "      <th>image_suffix</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>732251_3719739_label.geojson</td>\n",
       "      <td>732251_3719739_image.tif</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>732251_3720189_label.geojson</td>\n",
       "      <td>732251_3720189_image.tif</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>732251_3721539_label.geojson</td>\n",
       "      <td>732251_3721539_image.tif</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>732251_3721989_label.geojson</td>\n",
       "      <td>732251_3721989_image.tif</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>732251_3722439_label.geojson</td>\n",
       "      <td>732251_3722439_image.tif</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   label_suffix              image_suffix\n",
       "1  732251_3719739_label.geojson  732251_3719739_image.tif\n",
       "2  732251_3720189_label.geojson  732251_3720189_image.tif\n",
       "5  732251_3721539_label.geojson  732251_3721539_image.tif\n",
       "6  732251_3721989_label.geojson  732251_3721989_image.tif\n",
       "7  732251_3722439_label.geojson  732251_3722439_image.tif"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(final_Tile_Split['Category'].value_counts())\n",
    "final_Tile_Split[final_Tile_Split['Category']=='Train'][['label_suffix', 'image_suffix']].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Move files from initial processing into seperate Train/Test/Validate Split using mv command\n",
    "import shutil\n",
    "from tqdm import tqdm\n",
    "results_data_base_location = '/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav2/'\n",
    "processedData_path = '/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav3/'\n",
    "commandPathList = []\n",
    "for idx, row in tqdm(final_Tile_Split[final_Tile_Split['Category']=='Train'].iterrows()):\n",
    "    results = glob.glob(os.path.join(results_data_base_location, '**', '*{}'.format(row['image_suffix'])), recursive=True)\n",
    "    for result in results:\n",
    "        newPath = os.path.join(processedData_path, 'spacenet_4_Train', result.replace('/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav2/', ''))\n",
    "        if not os.path.exists(os.path.dirname(newPath)):\n",
    "            os.makedirs(os.path.dirname(newPath))\n",
    "        commandPath = \"mv {} {}\".format(result, newPath)\n",
    "        commandPathList.append(commandPath)\n",
    "        \n",
    "\n",
    "\n",
    "## Makes sure folder exists\n",
    "for pathName in tqdm(list(set([os.path.dirname(item.split(' ')[2]) for item in commandPathList]))):\n",
    "    if not os.path.exists(pathName):\n",
    "            os.makedirs(pathName)\n",
    "        \n",
    "        \n",
    "with open('/raid/nfs/workingDir/cw-tiler/Atlanta_AOI_mv_Train_Data.sh', 'w') as fp:\n",
    "    fp.writelines([\"{}\\n\".format(item)  for item in commandPathList])\n",
    "    \n",
    "# To Execute command use sh ./Atlanta_AOI_mv_Train_Data.sh\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Work with clipping Labels.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "from tqdm import tqdm\n",
    "results_data_base_location = '/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav2/'\n",
    "processedData_path = '/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav3/'\n",
    "commandPathList = []\n",
    "for idx, row in tqdm(final_Tile_Split[final_Tile_Split['Category']=='Train'].iterrows()):\n",
    "    results = glob.glob(os.path.join(results_data_base_location, '**', '*{}'.format(row['label_suffix'])), recursive=True)\n",
    "    for result in results:\n",
    "        newPath = os.path.join(processedData_path, 'spacenet_4_Train', result.replace('/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav2/', ''))\n",
    "        if not os.path.exists(os.path.dirname(newPath)):\n",
    "            os.makedirs(os.path.dirname(newPath))\n",
    "        commandPath = \"mv {} {}\".format(result, newPath)\n",
    "        commandPathList.append(commandPath)\n",
    "with open('/raid/nfs/workingDir/cw-tiler/Atlanta_AOI_mv_Train_Data_label.sh', 'w') as fp:\n",
    "         fp.writelines([\"{}\\n\".format(item)  for item in commandPathList]) \n",
    "for pathName in tqdm(list(set([os.path.dirname(item.split(' ')[2]) for item in commandPathList]))):\n",
    "    if not os.path.exists(pathName):\n",
    "            os.makedirs(pathName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav3/spacenet_4_Train/Atlanta_nadir14_catid_10300100039AB000/PAN/Atlanta_nadir14_catid_10300100039AB000_PAN_732251_3719739_image.tif\n"
     ]
    }
   ],
   "source": [
    "## Creation of Test Data Set\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "Off_nadir=25\n",
    "Very_Off_nadir=40\n",
    "\n",
    "columnList = [column for column in  final_Tile_Split.columns if column[0:3]=='Atl']\n",
    "AOI_List_Dict = []\n",
    "for column in columnList:\n",
    "    nameParts = column.split('_')\n",
    "    nadirAngle = int(nameParts[1].replace('nadir', \"\"))\n",
    "    if nadirAngle <= 25:\n",
    "        nadirClass = 'Nadir'\n",
    "    elif nadirAngle < 40:\n",
    "        nadirClass = 'Off-Nadir'\n",
    "    else:\n",
    "        nadirClass = 'Very-off-Nadir'\n",
    "    \n",
    "    AOI_List_Dict.append({'FolderName': column,\n",
    "     'nadir_angle': nadirAngle,\n",
    "    'nadir_class': nadirClass})\n",
    "    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Atl'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_Tile_Split.columns[3][0:3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Nadir             11\n",
       "Very-off-Nadir     9\n",
       "Off-Nadir          7\n",
       "Name: nadir_class, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aoi_Dict = pd.DataFrame(AOI_List_Dict)\n",
    "aoi_Dict.head()\n",
    "aoi_Dict['nadir_class'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "NadirList = aoi_Dict[aoi_Dict['nadir_class']=='Nadir']['FolderName'].values\n",
    "OffNadirList = aoi_Dict[aoi_Dict['nadir_class']=='Off-Nadir']['FolderName'].values\n",
    "VeryOffNadirList = aoi_Dict[aoi_Dict['nadir_class']=='Very-off-Nadir']['FolderName'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_Tile_Split = gpd.read_file(\"/raid/nfs/workingDir/cw-tiler/AOI_6_Atlanta_Train_Test_Validate_Split_V4.geojson\", driver='GeoJSON')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Train                  1160\n",
       "Test_Off-Nadir          309\n",
       "Test_Very-Off-Nadir     309\n",
       "Test_Nadir              309\n",
       "Validate                231\n",
       "Name: Category, dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_Tile_Split['Category'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "from tqdm import tqdm\n",
    "import glob\n",
    "results_data_base_location = '/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav2/'\n",
    "processedData_path = '/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav3/'\n",
    "commandPathList = []\n",
    "\n",
    "NadirList = aoi_Dict[aoi_Dict['nadir_class']=='Nadir']['FolderName'].values\n",
    "OffNadirList = aoi_Dict[aoi_Dict['nadir_class']=='Off-Nadir']['FolderName'].values\n",
    "VeryOffNadirList = aoi_Dict[aoi_Dict['nadir_class']=='Very-off-Nadir']['FolderName'].values\n",
    "\n",
    "testCategoryList = [\"Test_Nadir\",\"Test_Off-Nadir\", \"Test_Very-Off-Nadir\"]\n",
    "nadirCategoryListList = [NadirList, OffNadirList, VeryOffNadirList]\n",
    "rowList = []\n",
    "\n",
    "for testCategory, nadirCategoryList in zip(testCategoryList, nadirCategoryListList):\n",
    "    for idx, row in tqdm(final_Tile_Split[final_Tile_Split['Category']==testCategory].iterrows()):\n",
    "\n",
    "        \n",
    "        rowTmp = row[nadirCategoryList]\n",
    "        baseID = np.random.choice(rowTmp[rowTmp>=450].index,1)[0]\n",
    "        print(baseID)\n",
    "        row['TestimageID'] = baseID\n",
    "\n",
    "\n",
    "        results = glob.glob(os.path.join(results_data_base_location, baseID, '**', '*{}'.format(row['image_suffix'])), recursive=True)\n",
    "        for result in results:\n",
    "            newPath = os.path.join(processedData_path, 'spacenet_4_Test', result.replace('/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav2/', ''))\n",
    "            if not os.path.exists(os.path.dirname(newPath)):\n",
    "                os.makedirs(os.path.dirname(newPath))\n",
    "            commandPath = \"cp {} {}\".format(result, newPath)\n",
    "            commandPathList.append(commandPath)\n",
    "            \n",
    "\n",
    "        results = glob.glob(os.path.join(results_data_base_location, '**', '*{}'.format(row['label_suffix'])), recursive=True)\n",
    "        for result in results:\n",
    "            newPath = os.path.join(processedData_path, 'spacenet_4_Test', result.replace('/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav2/', ''))\n",
    "            if not os.path.exists(os.path.dirname(newPath)):\n",
    "                os.makedirs(os.path.dirname(newPath))\n",
    "            commandPath = \"cp {} {}\".format(result, newPath)\n",
    "            commandPathList.append(commandPath)\n",
    "\n",
    "        rowList.append(row)\n",
    "\n",
    "    \n",
    "final_test_train_split_update = gpd.GeoDataFrame(rowList)\n",
    "final_test_train_split_update.crs = final_Tile_Split.crs\n",
    "#final_test_train_split_update.to_file(\"/raid/nfs/workingDir/cw-tiler/AOI_6_Atlanta_Train_Test_Validate_Split_V3p3.geojson\", driver='GeoJSON')\n",
    "\n",
    "        \n",
    "with open('/raid/nfs/workingDir/cw-tiler/Atlanta_AOI_mv_Test_Data_v4.sh', 'w') as fp:\n",
    "         fp.writelines([\"{}\\n\".format(item)  for item in commandPathList]) \n",
    "        \n",
    "for pathName in tqdm(list(set([os.path.dirname(item.split(' ')[2]) for item in commandPathList]))):\n",
    "    if not os.path.exists(pathName):\n",
    "            os.makedirs(pathName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cp /raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav2/Atlanta_nadir23_catid_103001000352C200/PAN/Atlanta_nadir23_catid_103001000352C200_PAN_732251_3719289_image.tif /raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav3/spacenet_4_Test/Atlanta_nadir23_catid_103001000352C200/PAN/Atlanta_nadir23_catid_103001000352C200_PAN_732251_3719289_image.tif'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cp /raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav2/geojson/spacenet-buildings/spacenet-buildings_748901_3745389_label.geojson /raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav3/spacenet_4_Test/geojson/spacenet-buildings/spacenet-buildings_748901_3745389_label.geojson'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import shutil\n",
    "from tqdm import tqdm\n",
    "results_data_base_location = '/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav2/'\n",
    "processedData_path = '/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav3/'\n",
    "commandPathList = []\n",
    "\n",
    "\n",
    "rowList = []\n",
    "\n",
    "\n",
    "for idx, row in tqdm(final_Tile_Split[final_Tile_Split['Category']=='Validate'].iterrows()):\n",
    "    results = glob.glob(os.path.join(results_data_base_location, '**', '*{}'.format(row['image_suffix'])), recursive=True)\n",
    "    for result in results:\n",
    "        newPath = os.path.join(processedData_path, 'spacenet_4_Validate', result.replace('/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav2/', ''))\n",
    "        if not os.path.exists(os.path.dirname(newPath)):\n",
    "            os.makedirs(os.path.dirname(newPath))\n",
    "        commandPath = \"mv {} {}\".format(result, newPath)\n",
    "        commandPathList.append(commandPath)\n",
    "\n",
    "    results = glob.glob(os.path.join(results_data_base_location, '**', '*{}'.format(row['label_suffix'])), recursive=True)\n",
    "    for result in results:\n",
    "        newPath = os.path.join(processedData_path, 'spacenet_4_Validate', result.replace('/raid/nfs/data/Datasets/CosmiQ_General_Study/AOI_6_Atlanta/processedDatav2/', ''))\n",
    "        if not os.path.exists(os.path.dirname(newPath)):\n",
    "            os.makedirs(os.path.dirname(newPath))\n",
    "        commandPath = \"mv {} {}\".format(result, newPath)\n",
    "        commandPathList.append(commandPath)\n",
    "         \n",
    "\n",
    "        \n",
    "with open('/raid/nfs/workingDir/cw-tiler/Atlanta_AOI_mv_Validate_Data_v2.sh', 'w') as fp:\n",
    "         fp.writelines([\"{}\\n\".format(item)  for item in commandPathList]) \n",
    "        \n",
    "for pathName in tqdm(list(set([os.path.dirname(item.split(' ')[2]) for item in commandPathList]))):\n",
    "    if not os.path.exists(pathName):\n",
    "            os.makedirs(pathName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:cw-tiler]",
   "language": "python",
   "name": "conda-env-cw-tiler-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}