8.1-Handling-large-scale-datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!pip install -r https://raw.githubusercontent.com/datamllab/automl-in-action-notebooks/master/requirements.txt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## 8.1.1 Loading image classification dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!!wget https://github.com/datamllab/automl-in-action-notebooks/raw/master/data/mnist.tar.gz\n",
    "!!tar xzf mnist.tar.gz\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "```\n",
    "train/\n",
    "  0/\n",
    "    1.png\n",
    "    21.png\n",
    "    ...\n",
    "  1/\n",
    "  2/\n",
    "  3/\n",
    "  ...\n",
    "\n",
    "test/\n",
    "  0/\n",
    "  1/\n",
    "  ...\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import autokeras as ak\n",
    "\n",
    "batch_size = 32\n",
    "img_height = 28\n",
    "img_width = 28\n",
    "\n",
    "parent_dir = \"data\"\n",
    "\n",
    "test_data = ak.image_dataset_from_directory(\n",
    "    os.path.join(parent_dir, \"test\"),\n",
    "    seed=123,\n",
    "    color_mode=\"grayscale\",\n",
    "    image_size=(img_height, img_width),\n",
    "    batch_size=batch_size,\n",
    ")\n",
    "for images, labels in test_data.take(1):\n",
    "    print(images.shape, images.dtype)\n",
    "    print(labels.shape, labels.dtype)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## 8.1.2 Splitting the loaded dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "all_train_data = ak.image_dataset_from_directory(\n",
    "    os.path.join(parent_dir, \"train\"),\n",
    "    seed=123,\n",
    "    color_mode=\"grayscale\",\n",
    "    image_size=(img_height, img_width),\n",
    "    batch_size=batch_size,\n",
    ")\n",
    "train_data = all_train_data.take(int(60000 / batch_size * 0.8))\n",
    "validation_data = all_train_data.skip(int(60000 / batch_size * 0.8))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "train_data = ak.image_dataset_from_directory(\n",
    "    os.path.join(parent_dir, \"train\"),\n",
    "    validation_split=0.2,\n",
    "    subset=\"training\",\n",
    "    seed=123,\n",
    "    color_mode=\"grayscale\",\n",
    "    image_size=(img_height, img_width),\n",
    "    batch_size=batch_size,\n",
    ")\n",
    "\n",
    "validation_data = ak.image_dataset_from_directory(\n",
    "    os.path.join(parent_dir, \"train\"),\n",
    "    validation_split=0.2,\n",
    "    subset=\"validation\",\n",
    "    seed=123,\n",
    "    color_mode=\"grayscale\",\n",
    "    image_size=(img_height, img_width),\n",
    "    batch_size=batch_size,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "train_data = train_data.prefetch(5)\n",
    "validation_data = validation_data.prefetch(5)\n",
    "test_data = test_data.prefetch(tf.data.AUTOTUNE)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "Then we just do one quick demo of AutoKeras to make sure the dataset works.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "clf = ak.ImageClassifier(overwrite=True, max_trials=1)\n",
    "clf.fit(train_data, epochs=1, validation_data=validation_data)\n",
    "print(clf.evaluate(test_data))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## 8.1.3 Loading text classification dataset\n",
    "You can also load text datasets in the same way.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!!wget https://github.com/datamllab/automl-in-action-notebooks/raw/master/data/imdb.tar.gz\n",
    "!!tar xzf imdb.tar.gz\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "For this dataset, the data is already split into train and test.\n",
    "We just load them separately.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import autokeras as ak\n",
    "import tensorflow as tf\n",
    "\n",
    "train_data = ak.text_dataset_from_directory(\n",
    "    \"imdb/train\",\n",
    "    validation_split=0.2,\n",
    "    subset=\"training\",\n",
    "    seed=123,\n",
    "    max_length=1000,\n",
    "    batch_size=32,\n",
    ").prefetch(1000)\n",
    "\n",
    "validation_data = ak.text_dataset_from_directory(\n",
    "    \"imdb/train\",\n",
    "    validation_split=0.2,\n",
    "    subset=\"validation\",\n",
    "    seed=123,\n",
    "    max_length=1000,\n",
    "    batch_size=32,\n",
    ").prefetch(1000)\n",
    "\n",
    "test_data = ak.text_dataset_from_directory(\n",
    "    \"imdb/test\",\n",
    "    max_length=1000,\n",
    ").prefetch(1000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "clf = ak.TextClassifier(overwrite=True, max_trials=1)\n",
    "clf.fit(train_data, epochs=2, validation_data=validation_data)\n",
    "print(clf.evaluate(test_data))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## 8.1.4 Handling large dataset in general format\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "data = [5, 8, 9, 3, 6]\n",
    "\n",
    "\n",
    "def generator():\n",
    "    for i in data:\n",
    "        yield i\n",
    "\n",
    "\n",
    "for x in generator():\n",
    "    print(x)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "dataset = tf.data.Dataset.from_generator(generator, output_types=tf.int32)\n",
    "for x in dataset:\n",
    "    print(x.numpy())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "parent_dir = \"imdb\"\n",
    "\n",
    "\n",
    "def load_data(path):\n",
    "    data = []\n",
    "    for class_label in [\"pos\", \"neg\"]:\n",
    "        for file_name in os.listdir(os.path.join(path, class_label)):\n",
    "            data.append((os.path.join(path, class_label, file_name), class_label))\n",
    "\n",
    "    data = np.array(data)\n",
    "    np.random.shuffle(data)\n",
    "    return data\n",
    "\n",
    "\n",
    "def get_generator(data):\n",
    "    def data_generator():\n",
    "        for file_path, class_label in data:\n",
    "            text_file = open(file_path, \"r\")\n",
    "            text = text_file.read()\n",
    "            text_file.close()\n",
    "            yield text, class_label\n",
    "\n",
    "    return data_generator\n",
    "\n",
    "\n",
    "all_train_np = load_data(os.path.join(parent_dir, \"train\"))\n",
    "\n",
    "\n",
    "def np_to_dataset(data_np):\n",
    "    return (\n",
    "        tf.data.Dataset.from_generator(\n",
    "            get_generator(data_np),\n",
    "            output_types=tf.string,\n",
    "            output_shapes=tf.TensorShape([2]),\n",
    "        )\n",
    "        .map(lambda x: (x[0], x[1]))\n",
    "        .batch(32)\n",
    "        .prefetch(5)\n",
    "    )\n",
    "\n",
    "\n",
    "train_data = np_to_dataset(all_train_np[:20000])\n",
    "validation_data = np_to_dataset(all_train_np[20000:])\n",
    "test_np = load_data(os.path.join(parent_dir, \"test\"))\n",
    "test_data = np_to_dataset(test_np)\n",
    "\n",
    "for texts, labels in train_data.take(1):\n",
    "    print(texts.shape)\n",
    "    print(labels.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "clf = ak.TextClassifier(overwrite=True, max_trials=1)\n",
    "clf.fit(train_data, epochs=2, validation_data=validation_data)\n",
    "print(clf.evaluate(test_data))\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "8.1-Handling-large-scale-datasets",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}