samples/sun/Read_big_Matlab_file.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import scipy.io\n",
    "import numpy as np\n",
    "import h5py\n",
    "import json\n",
    "\n",
    "file_name = \"/scratch/datasets/SUNRGBDtoolbox/Metadata/SUNRGBD2Dseg.mat\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dtypes = {}\n",
    "def string(seq):\n",
    "    \"\"\"Convert a sequence of integers into a single string.\n",
    "    \"\"\"\n",
    "    return ''.join([chr(a) for a in seq])\n",
    "\n",
    "def add_dtype_name(f, name):\n",
    "    \"\"\"Keep track of all dtypes and names in the HDF5 file using it.\n",
    "    \"\"\"\n",
    "    global dtypes\n",
    "    dtype = f.dtype            \n",
    "    if str(dtypes) in dtype.name:\n",
    "        dtypes[dtype.name].add(name)\n",
    "    else:\n",
    "        dtypes[dtype.name] = set([name])\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recursive_dict(f, root=None, name='root'):\n",
    "    if root is None: root = f\n",
    "    if hasattr(f, 'keys'):\n",
    "        a = dict(f)\n",
    "        if u'#refs#' in a.keys(): # we don't want to keep this\n",
    "            del(a[u'#refs#'])\n",
    "        for k in a.keys():\n",
    "            # print k\n",
    "            a[k] = recursive_dict(f[k], root, name = name + '->' + k)\n",
    "        return a\n",
    "    elif hasattr(f, 'shape'):\n",
    "        if f.dtype.name not in ['object', 'uint16']: # this is a numpy array\n",
    "            # Check shape to assess whether it can fit in memory\n",
    "            # or not. If not recast to a smaller dtype!\n",
    "            add_dtype_name(f, name)\n",
    "            dtype = f.dtype\n",
    "            if (np.prod(f.shape)*f.dtype.itemsize) > 2e9:\n",
    "                print(\"WARNING: The array\" + name + \"requires > 2Gb\")\n",
    "                if f.dtype.char=='d':\n",
    "                    print(\"\\t Recasting\" + dtype + \"to float32\")\n",
    "                    dtype = np.float32\n",
    "                else:\n",
    "                    raise MemoryError\n",
    "            return np.array(f, dtype=dtype).squeeze()\n",
    "        elif f.dtype.name in ['uint16']: # this may be a string for Matlab\n",
    "            add_dtype_name(f, name)\n",
    "            try:\n",
    "                return string(f)\n",
    "            except ValueError: # it wasn't...\n",
    "                print(\"WARNING:\"+ name, \":\")\n",
    "                print(\"\\t\" + f)\n",
    "                print(\"\\t CONVERSION TO STRING FAILED, USING ARRAY!\")\n",
    "                tmp = np.array(f).squeeze()\n",
    "                print(\"\\t\" + tmp)\n",
    "                return tmp\n",
    "            pass\n",
    "        elif f.dtype.name=='object': # this is a 2D array of HDF5 object references or just objects\n",
    "            add_dtype_name(f, name)\n",
    "            container = []\n",
    "            for i in range(f.shape[0]):\n",
    "                for j in range(f.shape[1]):\n",
    "                    if str(f[i][j])=='<HDF5 object reference>': # reference follow it:\n",
    "                        container.append(recursive_dict(root[f[i][j]], root, name=name))\n",
    "                    else:\n",
    "                        container.append(np.array(f[i][j]).squeeze())\n",
    "            try:\n",
    "                return np.array(container).squeeze()\n",
    "            except ValueError:\n",
    "                print(\"WARNING:\" + name + \":\")\n",
    "                print(\"\\t\" + container)\n",
    "                print(\"\\t CANNOT CONVERT INTO NON-OBJECT ARRAY\")\n",
    "                return np.array(container, dtype=np.object).squeeze()\n",
    "        else:\n",
    "            raise NotImplemented\n",
    "    else:\n",
    "        raise NotImplemented\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "f = h5py.File(file_name, mode='r')\n",
    "data = recursive_dict(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = data['SUNRGBD2Dseg']['seglabel']\n",
    "b = data['SUNRGBD2Dseg']['seglabelall']\n",
    "\n",
    "del(data)\n",
    "\n",
    "import gc\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "VALUES = {}\n",
    "\n",
    "for i in range(0, 10334): #10334\n",
    "    pic = a[i]\n",
    "    label = b[i]\n",
    "    for r in range(len(pic)):\n",
    "        for j in range(0, len(pic[r])):\n",
    "            print(label[r][j])\n",
    "            print(pic[r][j])\n",
    "            VALUES[label[r][j]] = pic[r][j] \n",
    "    print('Ended with ' + str(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('labels_matching_full.json', 'w') as fp:\n",
    "    json.dump(VALUES, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ideally after this point you need to restart the kernel and clean the memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('labels_matching_full.json') as handle:\n",
    "    mapping = json.loads(handle.read())\n",
    "    \n",
    "labels_full_names = ['wall', 'floor', 'cabinet', 'bed', 'chair', 'sofe', 'table', 'door', 'window', 'bookshelf', \n",
    "        'picture', 'counter', 'blinds', 'desk', 'shelves', 'curtain', 'dresser', 'pillow', 'mirrow', 'floor_mat',\n",
    "       'clothes', 'ceiling', 'books', 'fridge', 'tv', 'paper', 'towel', 'shower_curtain', 'box', 'whiteboard', \n",
    "        'person', 'night_stand', 'toilet', 'sink', 'lamp', 'bathtub', 'bag']\n",
    "Labels_37 = list(range(1,38))\n",
    "import pandas as pd\n",
    "labels_37 = pd.DataFrame({\n",
    "     'Label_37': Labels_37,\n",
    "     'Name_37': labels_full_names})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique = []\n",
    "parsing = []\n",
    "for key in mapping.items():\n",
    "    if key[1]!= 0.:\n",
    "        unique.append(int(float(key[1])))\n",
    "        parsing.append(int(float(key[0])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Name_6585 = []\n",
    "for i in range(0, len(parsing)):\n",
    "    Name_6585.append(labels_full_names[parsing[i]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels_full = pd.DataFrame({\n",
    "     'Label_6585': parsing,\n",
    "     'Label_37': unique,\n",
    "     'Name_6585': Name_6585})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dataset = pd.merge(labels_full, labels_37)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_dataset.to_csv('name_mapping_from_toolbox', sep=',')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}