exercise-1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "71486d3b-327f-4e81-b812-a7b83930d4fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "9fbf89be-58be-48b4-abdc-7cdcf2a20fda",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torchvision import datasets\n",
    "import webdataset as wds\n",
    "from PIL import Image, ImageDraw, ImageFont"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e32e3785-6b31-4c1d-b7a8-cf50269a265c",
   "metadata": {},
   "source": [
    "# Writing MNIST as Shards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d86bcf64-6716-4d64-a077-6642549e474a",
   "metadata": {},
   "outputs": [],
   "source": [
    "mnist = datasets.MNIST(root=\"./data\", download=True, train=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "43a61f00-f6a2-482d-9ee2-f76945fb5c24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# writing mnist-000000.tar 0 0.0 GB 0\n"
     ]
    }
   ],
   "source": [
    "sink = wds.ShardWriter(pattern=\"mnist-%06d.tar\")\n",
    "for i, (image, cls) in enumerate(mnist):\n",
    "    sink.write({\n",
    "        \"__key__\": f\"{i:06d}\",\n",
    "        \"png\": image,\n",
    "        \"cls\": cls\n",
    "    })\n",
    "sink.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e072c738-3e3f-41f3-bf96-625ad2aec214",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-rw-r-- 1 tmb tmb 235M Aug 26 03:27 mnist-000000.tar\n"
     ]
    }
   ],
   "source": [
    "!ls -lh mnist-000000.tar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0ce10b53-36a1-4f3f-b9de-cbd23487f725",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-r--r--r-- bigdata/bigdata   1 2021-08-26 03:27 000000.cls\n",
      "-r--r--r-- bigdata/bigdata 313 2021-08-26 03:27 000000.png\n",
      "-r--r--r-- bigdata/bigdata   1 2021-08-26 03:27 000001.cls\n",
      "-r--r--r-- bigdata/bigdata 312 2021-08-26 03:27 000001.png\n",
      "-r--r--r-- bigdata/bigdata   1 2021-08-26 03:27 000002.cls\n",
      "tar: write error\n"
     ]
    }
   ],
   "source": [
    "!tar tvf mnist-000000.tar | sed 5q"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "de3197ed-7e89-4701-ac5a-df4eb7b76102",
   "metadata": {},
   "source": [
    "# Generate a Large Rendered Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7f93e0ad-8e0a-480b-8174-5d1308dec9b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "!curl -sL https://filesamples.com/samples/font/ttf/OpenSans-Regular.ttf > OpenSans-Regular.ttf\n",
    "!curl -sL https://raw.githubusercontent.com/dwyl/english-words/master/words.txt > words.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "4e6048d4-0336-464e-8364-f0e92b329cac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "1080\n",
      "&c\n",
      "10-point\n",
      "10th\n",
      "11-point\n",
      "12-point\n",
      "16-point\n",
      "18-point\n",
      "1st\n"
     ]
    }
   ],
   "source": [
    "!head words.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be5a5631-63ba-4472-81f9-e816a9c0ce67",
   "metadata": {},
   "source": [
    "Use the Pillow library to render text (or figure out how to do this with Cairo).\n",
    "\n",
    "See here for more information: https://pillow.readthedocs.io/en/stable/reference/ImageDraw.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e5de55ec-a2e6-4d0c-840d-606121366b39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f542663f280>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAB2CAYAAAAz69PvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAdQklEQVR4nO3de3AUVfo38O8zt9wmkCFXuYRwCRchhCVIQGIWMGgIgq+WpT+FVXdF3C23Vst13132rfIta/d1WbdYWbdURBfRBeW3iC4EWEQCqATREARJhEAikdwgiSH3y1z6ef+YTv8SMplMkklC2udT1ZXp7jPd58x0njl9+vRpYmYIIYTQF8NQZ0AIIYT/SXAXQggdkuAuhBA6JMFdCCF0SIK7EELokAR3IYTQoQEJ7kSUTkQFRFRIRL8biH0IIYToHvm7nzsRGQFcALAUQCmAHAAPMvM3ft2REEKIbg1EzX0egEJm/paZ7QB2ALh7APYjhBCiG6YB2OYYACUd5ksBJF+fiIjWAlirziYNQD6EEELvqpk50tOKgQjuPmHmzQA2AwARyRgIQgjRe991t2IgmmXKAIzrMD9WXSaEEGKQDERwzwEQT0QTiMgC4L8A7BmA/QghhOiG35tlmNlJRL8E8BEAI4AtzJzv7/0IIYTont+7QvYpE9LmLoQQfZHLzHM9rZA7VIUQQockuAshhA5JcBdCCB2S4C6EEDokwV0IIXRoWAT35cuX48SJE8jJycHTTz891Nnp1k9/+lPk5OTgxIkTWLVqlcc0v/zlL5GTk4Pjx49j5cqVg5xD4Y3JZMILL7yAnJwcHDlyBHPneuyE0CfLli3TjuFf//rXftuuEN0ZsuEHeiMiIgJz5syByWTCp59+OtTZ6VZMTAySkpLgcrkQHR3tMc3YsWMxd+5ctLW1ISIiYpBzKLwxGAyYOHEi5s6di7q6OoSGhvpt2+Hh4UhKSoLRaER2drbftitEd4ZFzV0IIUTvSHAXQggdkuAuhBA6JMFdCCF0SIK7EELo0LDoLeONxWKByWQCEYGZoSgKHA4HXC5Xn7ZnNpthNptBRAAAl8sFu90ORVH8me1eIyJYLBYYjcZOeetPWW9URqMRZrMZRqMRAMDMcDqdcDgc6O9AdyaTCWazGQaDu17jcDhgt9v7neeOxyEALb9DfdyIH65hF9zb/1nCw8OxYsUKpKenIyEhAWFhYWhpaUF5eTlycnKwZ88eHD9+HA6Hw6fthoWFISMjA8uWLUNiYiLCw8Nht9tRXFyMTz75BB988AHOnj3b7+DSW0SEqVOn4q677kJqairi4+MxYsQI2O12lJSU4MSJE8jMzMSJEyd8Luv1LBYL7rnnHowYMQJtbW3IzMzEtWvXuk1vs9lw7733wmAwQFEUZGZmorKy0mv6jIwMBAcHo6GhAe+//z6cTmeXdIGBgUhJSUFGRgZuueUWjB07FhaLBfX19Th//jyOHj2KzMxMXLp0yev3MHr0aNxxxx0gIhw4cAAVFRUgIsyePRsPP/wwFi1ahMjISK2szz77rMf8+MJms2HFihXIyMjAzJkzYbPZ0NLSgqKiIhw5cgS7du3CxYsX/fLDJESvMPOQTwDY2/TII4+w3W5nRVH4hRde4Pnz5/Nnn33GLS0t7ImiKFxVVcV//vOfOTQ01Ou2iYjnzZvHWVlZ3Nzc7HF7LpeLi4uL+Ve/+hUHBgZ2u61169axoijscDj4mWee8Zhm/fr1zMzc2trKP/vZz7zmzWq18rPPPsuXLl1ip9Pptayvv/46jxs3zuv2uptMJhMfOnSImZkbGhr4rrvu8pr+wQcf1PLjcrl6LMedd97JdXV1zMy8f/9+tlgsXdJMnjyZ3333Xb527RoriuKxrA6HgwsKCnjNmjVev4fFixdzbW0t2+12Tk9PZ7PZzI8//jgXFxezy+Xq9Nm99dZb2vssFgvv2LGDmZlra2t58eLFXss1Z84cPnToELe0tHjMs9Pp5PPnz/NDDz3EK1eu1I7hjRs39ul7kkkmD9PJ7uLqsKu5T5gwAW+++SamTZuGy5cvIzc3FxcuXEBzczOio6OxYMECzJw5ExEREXjqqadQV1eHF198sduaWXJyMrZu3YopU6bA5XKhoKAA2dnZKC4uhtVqRXJyMpKSkjB+/Hi88MILMJlMePnll/tc0/NVSEgI/vCHP+AXv/gFLBYLWlpaUFBQgJMnT6KqqgpWqxWJiYlISEhAeHg41qxZg9jYWDzxxBO4fPlyr/bldDpx+PBhLFmyBMHBwZg3bx7279/vsUnBZDJh+fLlWrOGwWDA8uXLsXXrVo/pDQYDbrnlFoSGhkJRFBw+fLjLGcb06dOxdetWzJ07F0SEqqoqnDlzBvn5+WhpacHo0aORlJSE+Ph4xMfH46WXXkJERAQ2bNjg9WzFbDYjOjoaq1atwosvvgir1YrS0lJcunQJtbW1GDVqFHJycnr1WbWbNm0a3njjDcyZMwfMjKqqKnz++ec4e/YsnE4n4uLiMG/ePEyePBl/+9vfsH379j7tR4g+66lWPRgTevh16lhzb2lp4ZaWFn711Vd54sSJrD7oQ5vCwsL4+eef5+bmZlYUhS9fvsxxcXEetxsTE8OffvopK4rCbW1t/MYbb3BsbGynNEFBQbxmzRquqqpiRVG4srKSU1JSPG7PXzV3IuKf//znWo2wvLycH3/8cR4xYkSndBaLhdPS0vjLL79kRVFYURTesmULBwUF9boGsGjRIm5oaGBFUfjgwYNstVo9pouLi+OCggJ2uVxcX1/PiqJwUVERT5o0yWP6gIAA/uijj5iZuaamhlNTUzutt9lsvGfPHna5XOxyufjw4cO8YMECNhqNndKFh4fzM888w9XV1awoCtfW1vK9997rcZ/tNXdm5nfeeYdLSkq4sbGRN2zYwFOmTGGDwcAA2GAwsMlk6vR5+lJzDwgI4H/+85/aZ37y5Em+7bbbuuQ5MjKSn3vuOa6rq2OHw6Gll5q7TH6cuq25D3lg701wbz+V3rFjBwcHB3eb3mq1clZWlhZEV61a5THd008/zQ6Hg10uF3/44YccFhbmMZ3RaOTnnnuOXS4XK4rC27Zt44CAgC7p/BXcY2NjOT8/nxVF4aamJl61alWXwNFxSkhI4AsXLrCiKFxfX8/Lly/v9UEyZswYzsvL037AumviaW9eaGxs5E2bNnFTUxO3tbXx/fff7zH92LFjuaKigpmZ8/Pz2WazdVq/evVq7Yf79OnTPHny5G7zaDKZ+Mknn+SmpiZWFIVzcnK6bA/oHNwdDgc3Nzfzb37zG69NOYDvwX3x4sV87do1ZmYuLy/n5ORkr9tcv3691mwjwV0mP0/dBvdh1xWyrq4Or776Kpqbm7tN09jYiP379wNwn5rPmDGjS5rQ0FCsXr0aJpMJdXV1+Mtf/oLa2lqP23O5XNi+fTuuXLkCIsKiRYswfvx4v5THk6VLl2LKlCkAgIMHD+Lf//631x4x+fn5eOWVV+BwOGC1WvHoo49qzSa+qqioQF5eHgBg5MiRSEpK8phuxYoVMJvNqK2txdatW1FbWwuLxYI777zT4z5vueUWjBgxAgDw5ZdfdrpQGxISon0HLpcLGzduRFFRUbd5dDqd2Lp1K06cOAEiwowZM3DnnXd6LZfRaERmZiZeffVVtLa29vg59MRsNuPuu+/GiBEjwMzYsWOH16Ydu92Ot956C6Wlpf3etxC90a/gTkTFRHSWiE4T0Ul12Sgi+piILqp/bf7JqlthYSHy83t+3nZhYWF7HmGz2boEntmzZ2PChAkAgLy8PJw6dcrr9iorK3HmzBkA7gHC4uPj+5L9HhkMBqSnp8NkMqGtrQ27d+9GU1OT1/coioLdu3ejuroaRKT1NOmN9vZwwN2unpqaqnXraxcdHY1bb70VAJCbm4v8/HxcuHABAJCamopRo0Z1Sm80GrFgwQIEBgZCURRkZWV1Wh8XF4eEhAQQEcrKynDo0KEee5Q0NTVh586dUBQFgYGBSEtLg8Vi6TZ9XV0dNm/e3ONn6Kvg4GAsXLgQBoMBjY2N2Lt3b4/dHb/77jucOnVKesuIQeWPmvtiZp7N//OQ1t8ByGLmeABZ6rxfMDNKS0tRU1PTY9r2f2YiQkBAQJfgPn36dIwcORKAu0bZUzfC5uZmVFRUAHAHrcmTJ/elCD0KDAxEQkICAPcZyOnTp316X1lZmfaDZrPZMG3atF7v+8SJE6ivr4fBYMCPfvQj7fNpl5SUhNjYWLhcLnz22Wdobm7GkSNHwMyIiYnB/PnzO6W3Wq2YN28eDAYDrl692qUs48aN00bPzMvL6/bM6XpnzpxBXV0diAiJiYkICgrqNu2VK1f6fNHUk5EjR2LSpEkA3Gc73s402rW2tqKgoECCuxhUA9Fb5m4Ai9TXbwM4CuC3/tp4ZWWlT/8k3tIYDAaMHz9eC/jJycl46aWXvL6HiDqN722z+fWERBMVFQWr1QoAaGlpQUlJiU/vUxQFhYWFSE1NRXBwMEaPHt3rfZeVleGbb77B/PnzMWPGDERHR2sB12AwYOnSpQgJCUFNTQ2ys7Phcrlw7NgxNDQ0IDQ0FIsXL8aBAwe0nkQRERFITEwEM+PixYsoLi7utL/Y2FjtRqWSkhKfm02qqqpw7do12Gw2xMbGIiAgoNu0paWlaGxs7PVn0Z3o6Ghtf7W1taiurvbpfaWlpWDmLmdDQgyU/gZ3BnBQ7bHyOjNvBhDNzBXq+isAPA5sTkRrAazt7Q69tbX7ymAwwGazaf9oKSkpSElJ6dU2AgMD+50PT6xWK0wm99ficrl8bk5gZtTV1QFwn1m0/0D0Rk1NDXJzc5GcnIzw8HAkJCSgoKBAy9fSpUtBRLh06ZJWC//6669RXFyMhIQEpKWlISQkRMvHvHnzEBISAgD4/PPPuwTZsLAwLe+NjY0+dy9tbW1FW1sbAHe7vdls7jZtfX29X2vMoaGh2t3QbW1taGlp8el9DQ0NUnMXg6q/wT2FmcuIKArAx0R0vuNKZm7vqtiF+kOwGQC6S+OJv27n7nhre0FBAa5cudKr93/77bd+ycf1OgaADr2JfNL+2RBRry+otu/v6NGjWLt2LcxmMxYtWoT3338fgPvC6Lhx48DM2L9/v1bLrq6uRnZ2NhISEjBp0iTMmjULn332GQwGA2677TbtYun17e2eytqbfLanJyKvtWF/B9SOQwwoiuLz8ai3ISLEja9fwZ2Zy9S/lUT0IYB5AK4S0U3MXEFENwHo/r70IcLMnWrEmzdvxptvvtmrbbTXHP2tqalJq8GaTCYEBwf7tC8i0p4c5HK5+nyG096jJSoqCnPmzMHIkSPR2NiItLQ0WK1W1NXVISsrq1PQ3LdvH9asWYPAwECkp6fj2LFjsNlsmD17NogI3333Hc6fP99lX/X19VreQ0JCYDQafQqCAQEB2kXU1tbWPg+70BeNjY1a84rZbIbFYvFpbJqBOtMTojt9vqBKRCFEFNr+GsAdAPIA7AHwiJrsEQC7+5tJf1MUBeXl5do/aWRkJJqamtDQ0ODz5I/BpjyprKzUfngCAgJ8bjtvv44AuAOet7FevKmpqUFOTg6ICBMmTEBcXBysVivS0tJARCgsLMTZs2c7vefs2bMoKiqCwWBAamoqwsLCMHr0aEyfPh3MjK+++gpVVVVd9nX58mUtmI8ePdpr23lHo0aN0i72VlRUDNh34cn333+v5dlqtXa56Nydjs2AQgyG/vSWiQZwjIjOAPgSwD5mPgBgPYClRHQRQJo6f0NhZm3IAsB9QdVbu+1gam1t1fqbW61Wj330PQkLC8PUqVMBuGvE7T1nequ5uRmff/45XC4XIiMjMXXqVEyaNEnrfXP06NEug4qVl5fj5MmTANxdTMeOHYvZs2dj5MiRcLlcyM7O9nixtKSkRPsRuvnmm31+ZunkyZNhs9nAzDh37pxf+q/7qqqqSus1FRUV5VOXUyLC5MmT+9RUJkRf9floY+ZvmTlRnWYw8/9Tl3/PzLczczwzpzFzz/0Wh8CpU6dw9epVAO6ANGfOnCHOkZuiKDh48CCcTieCgoKwbNkyn2q0S5YsQVRUFAB3H/++BndmxvHjx9HU1ASj0Yjk5GSkpqYiMDAQdrtduzmsI6fTiQMHDkBRFAQHB+PWW29FamoqAPeP1dGjRz3uq7i4GPn5+WBmxMXFITk5ucf8mUwm3HPPPTCZTHA6nTh27NigBvfm5mbk5uaCmREZGelTntt7DQkxmH6wVYmysjJkZmaCmREWFoZ169YhMjJyqLMFAPj444+1/tMrV67ssSdPTEwMnnzySQQFBcHhcODdd9/tV1PF6dOntR++W2+9FfPnz4fRaMQ333yDc+fOeXxPdnY2KisrQUS44447tL76BQUFXbpAtmtsbMR7770Hp9MJs9mMp556SvuB8oSIsHTpUtx+++0A3GcMe/fuHdReKK2trdi7dy9aW1thNBrx2GOPaX31u7NkyRLMmjVLmmXEoPrBBndmxqZNm3Du3DkQEZYtW4a///3vmDlzpscmGpPJhOjoaCxcuBAPPPCA1ttmIHz77bfYtGkTHA4HbDYbNm7ciB//+Mdd7sRsbxd/8cUXtTtHjx8/jl27dvVr/42NjcjOzgYAzJw5EykpKWBmHDt2rNu2/MrKShw/fhyA+27V9iaiY8eOee1nvnv3bq1mn5qaij/96U8YN25cl0AYEBCAJUuWYMOGDRg1ahTsdjs2bdqES5cu9ausffGf//xHu+M0MTER69evx5gxY7rk2WKxIC0tDc8//zxMJpN0hRSDatgN+etP58+fx7p167Bp0ybExMTg/vvvx8KFC3H06FF8/fXXuHbtmjZs7KRJk3DzzTdj2rRpOHz4MD744IMB7d62ZcsWzJo1C6tXr8aMGTOwc+dO7Nu3TwuwISEhmD17NpYvX661yxcVFWHdunUeL172htPpxCeffIJVq1YhJCQEISEhcDgcXm+1b25uRlZWFlasWIHw8HAA7puwsrOzvfZf//777/Hb3/4W27Ztw/Tp0/Hoo48iKSkJ+/btw9mzZ9HU1ISYmBikpqYiIyMDNpsNLpcLu3btwmuvvTYkTzqqqanB888/j3feeQcxMTH4yU9+goSEBOzevRv5+flwOp0YM2YMFi5ciPT0dAQFBWHbtm146KGHbphrO+IHYChGgbx+Qg8jn3Uc8nfDhg0+jZZ2++23c7stW7Z0Gtq142QwGHjZsmV8+vTpTsOyOhwOttvtbLfb2el0dlq+c+dOj9vz98M6Ro0axX/961+1oXUVRWG73c6tra3c1tamjVJpt9v5iy++4JSUlC5DIPd1SkxM1EZzZGY+d+4cR0VFeX3P1KlTubq6WntPeXl5lyGUPU1ExElJSXz06FHte3Y6ndzW1satra3aMkVRuLGxkV977TWveek4KuT777/v82fSm4d1GAwGXr16NZeUlGjfg9Pp5NbWVi3P7cMi//GPf+QJEyZwSUmJjAopk7+n4f2wjqKiIrz77rswGo3Izc316T1XrlzBtm3bALjvjuzulFhRFBw4cABnzpzBAw88gIyMDEyZMgWRkZEICAiAy+VCfX09rl69ivPnz+PIkSP48MMPPdZG8/PzsX37diiK0m3b9FdffYVt27bB6XT2OC5JTU0Nfv/73yMzMxOPPvqoNiBYcHAwHA4HysrKUFBQgL1792LHjh0+D83gi4KCArz99tsYM2YMAPdn6O3Re4D7Aukrr7yCiRMnAnBf2C0vL+9xX8yM3Nxc3Hfffbjvvvtw7733Yvr06YiIiIDZbEZTUxPKysq0z+6TTz7x2o//6tWr+Ne//oWgoKBejSvTPpyCw+FAc3Oz1xvbFEXBe++9h7y8PDzxxBNISUlBbGwsQkJCYLfbUV5ejtOnT+Mf//gHDh06BJPJhC1btmDixIlazyIhBhLdCO2AvblDdaCNGDECkZGRCAkJ0dpJ29ra0NDQgKqqqkHtmdGRxWJBVFQUwsLCYLFY4HK50NjYiOrqau12f71o/w6sViuMRiPa2tpQV1eHysrKQe3T7qv26zFhYWGdKgRXrlzxeXgCIfool/9n0MZOJLgLIcTw1W1w/8H2lhFCCD2T4C6EEDokwV0IIXRIgrsQQuiQBHchhNAhCe5CCKFDEtyFEEKHJLgLIYQOSXAXQggdkuAuhBA6JMFdCCF0SIK7EELoUI/BnYi2EFElEeV1WDaKiD4moovqX5u6nIjoZSIqJKKviejGeDCpEEL8wPhSc98KIP26Zb8DkMXM8QCy1HkAWAYgXp3WAnjNP9kUQgjRGz0Gd2b+FEDNdYvvBvC2+vptAP+rw/J31AfxnAAQRkQ3+SmvQgghfNTXNvdoZq5QX18B0P749zEASjqkK1WXdUFEa4noJBHJY2mEEMLP+v2YPWbmvjxsg5k3A9gMyMM6hBDC3/pac7/a3tyi/q1Ul5cBGNch3Vh1mRBCiEHU1+C+B8Aj6utHAOzusPxhtdfMfAB1HZpvhBBCDJIem2WI6D0AiwBEEFEpgP8LYD2AfxHRYwC+A3C/mnw/gAwAhQCaAfx0APIshBCiB/KAbCGEGL7kAdlCCPFDIsFdCCF0SIK7EELokAR3IYTQIQnuQgihQxLchRBChyS4CyGEDklwF0IIHZLgLoQQOiTBXQghdEiCuxBC6JAEdyGE0CEJ7kIIoUMS3IUQQockuAshhA5JcBdCCB2S4C6EEDokwV0IIXRIgrsQQuiQBHchhNAhCe5CCKFDEtyFEEKHTEOdAVUjgIKhzsQAigBQPdSZGEBSvuFNz+XTc9kAYHx3K26U4F7AzHOHOhMDhYhOSvmGLynf8KXnsvVEmmWEEEKHJLgLIYQO3SjBffNQZ2CASfmGNynf8KXnsnlFzDzUeRBCCOFnN0rNXQghhB9JcBdCCB0a8uBOROlEVEBEhUT0u6HOT18Q0RYiqiSivA7LRhHRx0R0Uf1rU5cTEb2slvdrIpozdDnvGRGNI6IjRPQNEeUT0VPqcr2UL5CIviSiM2r5nleXTyCiL9Ry/DcRWdTlAep8obo+bkgL4CMiMhLRV0S0V53XTfmIqJiIzhLRaSI6qS7TxfHZH0Ma3InICOAVAMsA3AzgQSK6eSjz1EdbAaRft+x3ALKYOR5AljoPuMsar05rAbw2SHnsKyeAXzPzzQDmA3hS/Y70Ur42AEuYORHAbADpRDQfwJ8BvMTMkwFcA/CYmv4xANfU5S+p6YaDpwCc6zCvt/ItZubZHfq06+X47DtmHrIJwAIAH3WYXwdg3VDmqR9liQOQ12G+AMBN6uub4L5RCwBeB/Cgp3TDYQKwG8BSPZYPQDCAUwCS4b6r0aQu145TAB8BWKC+NqnpaKjz3kO5xsId4JYA2AuAdFa+YgAR1y3T3fHZ22mom2XGACjpMF+qLtODaGauUF9fARCtvh62ZVZP0X8E4AvoqHxqk8VpAJUAPgZQBKCWmZ1qko5l0Mqnrq8DED6oGe69jQD+NwBFnQ+HvsrHAA4SUS4RrVWX6eb47KsbZfgBXWNmJqJh3eeUiKwAdgF4mpnriUhbN9zLx8wuALOJKAzAhwCmDW2O/IeI7gJQycy5RLRoiLMzUFKYuYyIogB8TETnO64c7sdnXw11zb0MwLgO82PVZXpwlYhuAgD1b6W6fNiVmYjMcAf27cz8gbpYN+Vrx8y1AI7A3UwRRkTtlZ+OZdDKp64fCeD7wc1prywEsJKIigHsgLtp5m/QT/nAzGXq30q4f5znQYfHZ28NdXDPARCvXrm3APgvAHuGOE/+sgfAI+rrR+Buq25f/rB61X4+gLoOp483HHJX0f8B4Bwz/7XDKr2UL1KtsYOIguC+nnAO7iB/n5rs+vK1l/s+AIdZbby9ETHzOmYey8xxcP9/HWbmVdBJ+YgohIhC218DuANAHnRyfPbLUDf6A8gAcAHuds7/M9T56WMZ3gNQAcABdxveY3C3U2YBuAjgEIBRalqCu4dQEYCzAOYOdf57KFsK3G2aXwM4rU4ZOirfLABfqeXLA/CcunwigC8BFALYCSBAXR6ozheq6ycOdRl6UdZFAPbqqXxqOc6oU357DNHL8dmfSYYfEEIIHRrqZhkhhBADQIK7EELokAR3IYTQIQnuQgihQxLchRBChyS4CyGEDklwF0IIHfr/il4smm9OYiQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fontsize = 64\n",
    "image = Image.new(\"RGB\", (600,150), (0, 0, 0))\n",
    "draw = ImageDraw.Draw(image)\n",
    "font = ImageFont.truetype(\"OpenSans-Regular.ttf\", fontsize)\n",
    "draw.text((10, 0), \"hello world\", (255,255,255), font=font)\n",
    "a = np.asarray(image)\n",
    "imshow(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "beae1e21-461e-4512-8333-b606e8e747e8",
   "metadata": {},
   "source": [
    "Part 1:\n",
    "\n",
    "- use random samples from words.txt as text\n",
    "- use ImageDraw / ImageFont to generate images corresponding to the words\n",
    "- write sharded training data; your training data should contain:\n",
    "    - images with .jpg extension\n",
    "    - ground truth text with .gt.txt extension\n",
    "    - metainfo about the font name and font size with .json extension\n",
    "    \n",
    "Follow the MNIST example above to see how you can write."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "f31b4d01-bddc-4078-a1d4-27eeb7be9a05",
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_word_training_data(output, num_samples, pattern, wordfile, fontlist, ):\n",
    "    sink = wds.TarWriter(output, ...)\n",
    "    for i in range(num_samples):\n",
    "        ...\n",
    "        sink.write(...)\n",
    "        ...\n",
    "    sink.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c135f47f-ce13-49b8-b6ce-408e20ffb94d",
   "metadata": {},
   "source": [
    "Part 2:\n",
    "- write `ray` code to generate 20 shards in parallel (use `multiprocessing` if you can't use `ray`)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba1b4be5-e191-4cb9-8648-3dd36b32a239",
   "metadata": {},
   "source": [
    "# Write an Image Augmentation Tool\n",
    "\n",
    "The main function should look something like:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "9eb37861-cf68-4fc6-a040-d491cad95ebd",
   "metadata": {},
   "outputs": [],
   "source": [
    "def augment_image(image):\n",
    "    return ...\n",
    "\n",
    "def augment_shard(source_url, output, extension=\"jpg\"):\n",
    "    source = wds.WebDataset(source_url, ...)\n",
    "    sink = wds.TarWriter(output, ...)\n",
    "    for sample in source:\n",
    "        result = dict(sample)\n",
    "        result[extension] = augment(sample[extension])\n",
    "        sink.write(...)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5cab0a2-45ca-47e3-a4b8-74fecc9b96e4",
   "metadata": {},
   "source": [
    "Exercise:\n",
    "\n",
    "- write `ray` code to augment a list of shards in parallel (use `multiprocessing` if you can't use `ray`)\n",
    "- wrap up your code in a script using a command line library (`argparse` or [`typer`](https://pypi.org/project/typer/))\n",
    "- write a shell script to run your script over the 20 shards you generated above\n",
    "\n",
    "Optional:\n",
    "\n",
    "- follow the Kubernetes example from the lecture to schedule the 20 shard transformations as Kubernetes Jobs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96903ea7-be03-4605-9e6d-a31ea12ea73d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}