1.0--synthetic_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "from fastai.text import *\n",
    "from fastai.vision import *\n",
    "# import textwrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "PATH = Path('data/IAM_handwriting')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "device = torch.device('cpu')#torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## From CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "fname = 'edited_line.csv'  #'edited_word.csv'\n",
    "CSV = PATH/fname\n",
    "FOLDER = 'lines'\n",
    "\n",
    "df = pd.read_csv(CSV)\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Convert pg.csv from ids to text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(PATH/'pg.csv')\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "itos = pickle.load(open(PATH/'itos.pkl', 'rb'))\n",
    "len(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def convert(x):\n",
    "    return ''.join([itos[int(c)] for c in x[:-2].split(' ')])\n",
    "\n",
    "df['label'] = df.apply(lambda row: convert(row.char_ids), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## From Source"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#words\n",
    "df = pd.read_csv(f'{PATH}/ascii/words.txt', names=['filename','result','word'], escapechar='\\\\',\n",
    "                 delim_whitespace=True, skiprows=18, header=None, usecols=[0,1,8])\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#lines\n",
    "df = pd.read_csv(f'{PATH}/ascii/lines.txt', names=['filename','result','text'], escapechar='\\\\',\n",
    "                 delim_whitespace=True, skiprows=23, header=None, usecols=[0,1,8])\n",
    "df.apply(lambda row: row.text.replace('|', ' '), axis=1)\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### via manually created DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "maxTextLen = 32\n",
    "samples = []\n",
    "chars = set()\n",
    "\n",
    "f=open(f'{PATH}/ascii/words.txt')\n",
    "for line in f:\n",
    "    # ignore comment line\n",
    "    if not line or line[0]=='#':\n",
    "        continue\n",
    "\n",
    "    lineSplit = line.strip().split(' ')\n",
    "    assert len(lineSplit) >= 9\n",
    "\n",
    "    fileName = lineSplit[0]\n",
    "\n",
    "    # GT text are columns starting at 9\n",
    "    gtText = ''.join(lineSplit[8:])[:maxTextLen]\n",
    "    char_len = len(gtText)\n",
    "    chars = chars.union(set(list(gtText)))\n",
    "\n",
    "    # put sample into list\n",
    "    samples.append([fileName, gtText, char_len])\n",
    "    \n",
    "samples = np.stack(samples)\n",
    "df = pd.DataFrame(samples, columns=['filename', 'word', 'char_len'], )\n",
    "del samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df['char_len'] = df.char_len.astype('int32')\n",
    "df = df.loc[df['char_len'] > 3]\n",
    "df = df.loc[df['char_len'] < 20]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Edit Downloaded Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "dl_path = Path('data/handwriting_images')\n",
    "# b = pd.read_csv(dl_path/'ewd.csv')\n",
    "# a = pd.read_csv(dl_path/'downloads.csv')\n",
    "# img_path = dl_path/'downloads'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df = pd.concat([a,b], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "num_lines = df.text.apply(lambda x: len(x.split('\\n')))\n",
    "df['num_lines'] = num_lines\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df.sort_values('num_lines', inplace=True)\n",
    "df.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df.rename({'text': 'label'}, axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df.to_csv(dl_path/'dl.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "i = 109\n",
    "row = df.iloc[i]\n",
    "print(row.text)\n",
    "print(row.filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "t1 = '''main task is to give a formal proof that the\n",
    "program he proposes meets the equally formal\n",
    "functional specification. While designing proofs\n",
    "and programs hand in hand, the student gets\n",
    "ample opportunity to perfect his manipulative\n",
    "agility with the predicate calculus. Finally, in\n",
    "order to drive home the message that this'''\n",
    "\n",
    "t2 = '''introductory programming course is primarily\n",
    "a course in formal mathematics, we see to it\n",
    "that the programming language in question\n",
    "has not been implemented on campus so that\n",
    "students are protected from the temptation\n",
    "to test their programs. And this concludes\n",
    "the sketch of my proposal for an introductory\n",
    "programming course for freshmen.'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "row.text = t1\n",
    "row.filename = row.filename[:-4]+'a.png'\n",
    "\n",
    "line = DataFrame({\"filename\": row.filename[:-5]+'b.png', \"text\": t2}, index=[i+0.1])\n",
    "df = df.append(line, ignore_index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df = df.sort_index().reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(dl_path/'downloads.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "for i in range(1,11):\n",
    "    df = pd.read_csv(dl_path/'downloads.csv')\n",
    "    df['filename'] = df['filename'].apply(lambda x: f\"v{i}_{x}\")\n",
    "    df.to_csv(dl_path/f\"v{i}.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "a = pd.read_csv(dl_path/'v1.csv')\n",
    "b = pd.read_csv(dl_path/'v2.csv')\n",
    "c = pd.read_csv(dl_path/'v3.csv')\n",
    "d = pd.read_csv(dl_path/'v4.csv')\n",
    "e = pd.read_csv(dl_path/'v5.csv')\n",
    "f = pd.read_csv(dl_path/'v6.csv')\n",
    "g = pd.read_csv(dl_path/'v7.csv')\n",
    "h = pd.read_csv(dl_path/'v8.csv')\n",
    "i = pd.read_csv(dl_path/'v9.csv')\n",
    "j = pd.read_csv(dl_path/'v10.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ndf = pd.concat([df,a,b,c,d,e,f,g,h,i,j], ignore_index=True)\n",
    "ndf.sort_values('num_lines', inplace=True)\n",
    "ndf.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ndf.to_csv(dl_path/'downloaded_images.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "len(ndf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Remove test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "test = df.sample(15, random_state=42)\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# a01-122\n",
    "# a03-014\n",
    "# b04-162\n",
    "# b06-071\n",
    "# f04-093\n",
    "# g06-042j\n",
    "# j01-045\n",
    "# l01-030\n",
    "# l01-173\n",
    "# l03-004\n",
    "# p02-105\n",
    "# p03-040\n",
    "# r02-081\n",
    "# r03-096\n",
    "# r06-115"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "test.to_csv(PATH/'test_pg.csv', columns=['filename', 'label'], index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df = df.drop(test.index)\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df.to_csv(PATH/'edited_pg.csv', columns=['filename', 'label'], index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Edit Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Filter df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def cleanup(x):\n",
    "    x = re.sub(r\" \\'(\\w+)\", lambda x: x.group(0)[1:], x)     # match all space and apostrophe letters\n",
    "    x = re.sub(r\" [.,?!:;()]\", lambda x: x.group(0)[1:], x)  # match all space and punctuation\n",
    "    return x\n",
    "\n",
    "df['text'] = df.apply(lambda row: cleanup(row.text), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "res = df[df.char_len == 1]\n",
    "res = res[res['word'].str.contains(\"[.,\\-\\\"']\")]\n",
    "res.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df.drop(res.index, inplace=True)\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def cleanup(x): return x.replace('\"\"', '\" \"')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def respace_quotes(m):\n",
    "    m = m.group(0)   # entire matched string\n",
    "    m = m.replace('\"','\" ')\n",
    "#     m = m.replace(' \"',' \" ')\n",
    "    return m\n",
    "\n",
    "# def clean_quotes(x): return re.sub(r'\\\"(.+?)\\\"', despace_quotes, x)\n",
    "def clean_quotes(x): return re.sub(r'\\\"\\S', respace_quotes, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df['text'] = df.apply(lambda row: clean_quotes(row.text), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# df = pd.read_csv(PATH/'edited_line.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Fix Errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "err = df[df.result == 'err']\n",
    "len(err)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "itr = iter(err.index.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import pyperclip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "i = 348 #next(itr)\n",
    "row = df.iloc[i]\n",
    "print(i)#, row.result)\n",
    "im_path = PATH/'words'/f'{row.filename}.png'\n",
    "im = PIL.Image.open(im_path)\n",
    "fig,ax = plt.subplots(figsize=(16,3))\n",
    "# row.result = 'ok'\n",
    "ax.imshow(im) \n",
    "# pyperclip.copy(row.text)\n",
    "# row.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "row.result = 'err'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "row.text = 'went to her fiance*? 2\\'s house and'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### crop image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "w,h = im.size\n",
    "nim = im.crop((0,0,w,130))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "nim.save(im_path)\n",
    "nim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### append multiple rows"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "st=\"a06-095-05-04 ok 182 1331 1586 153 115 ATI The\\na06-095-05-05 ok 182 1536 1571 490 175 NN Community\\na06-095-06-00 ok 187 305 1755 81 145 ( (\\na06-095-06-01 ok 187 379 1768 380 125 JJ Common\\na06-095-06-02 ok 187 829 1754 370 123 NN Market\\na06-095-06-03 ok 187 1173 1751 60 155 ) )\\na06-095-06-04 ok 187 1266 1762 203 119 HVZ has\\na06-095-06-05 ok 187 1524 1758 486 168 VBN imparted\\na06-095-06-06 ok 187 2043 1800 128 83 AT an\\na06-095-07-00 ok 183 309 1941 442 157 NN impetus\\na06-095-07-01 ok 183 816 1958 261 118 CC and\\na06-095-07-02 ok 183 1121 1988 152 85 AT an\\na06-095-07-03 ok 183 1350 1971 493 89 JJ economic\\na06-095-07-04 ok 183 1895 1943 301 173 NN growth\\na06-095-08-00 ok 183 324 2127 102 91 IN to\\na06-095-08-01 ok 183 516 2125 170 104 ATI The\\na06-095-08-02 ok 183 743 2120 145 127 CD Six\\na06-095-08-03 ok 183 930 2217 10 21 . .\\na06-095-08-04 ok 183 1027 2110 320 157 IN Above\\na06-095-08-05 ok 183 1387 2113 143 115 ABN all\\na06-095-08-06 ok 183 1496 2216 46 72 , ,\\na06-095-08-07 ok 183 1602 2112 116 117 PP3 it\\na06-095-08-08 ok 183 1731 2128 95 105 BEZ is\\na06-095-08-09 ok 183 1861 2167 122 60 AT an\\na06-095-08-10 ok 183 2027 2128 220 124 NN idea\\na06-095-09-00 ok 182 323 2290 287 121 WDT which\\na06-095-09-01 ok 182 678 2295 172 117 HVZ has\\na06-095-09-02 ok 182 905 2310 372 179 VBN gripped\\na06-095-09-03 ok 182 1315 2294 283 128 NNS$ men's\\na06-095-09-04 ok 182 1654 2318 322 105 NNS minds\\na06-095-09-05 ok 182 1962 2404 23 43 , ,\\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "with open(\"tmp.txt\", \"w\") as text_file:\n",
    "    text_file.write(st)\n",
    "\n",
    "f=open('tmp.txt')\n",
    "samples = []\n",
    "for line in f:\n",
    "    # ignore comment line\n",
    "    if not line or line[0]=='#':\n",
    "        continue\n",
    "\n",
    "    lineSplit = line.strip().split(' ')\n",
    "    assert len(lineSplit) >= 9\n",
    "\n",
    "    fileName = lineSplit[0]\n",
    "    res = lineSplit[1]\n",
    "\n",
    "    # GT text are columns starting at 9\n",
    "    gtText = ''.join(lineSplit[8:])[:32]\n",
    "\n",
    "    # put sample into list\n",
    "    samples.append([fileName, res, gtText])\n",
    "    \n",
    "samples = np.stack(samples)\n",
    "tmp_df = pd.DataFrame(samples, columns=['filename', 'result', 'word'], )\n",
    "del samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "tmp_itr = iter(tmp_df[tmp_df.result == 'err'].index.values)\n",
    "\n",
    "i = next(tmp_itr)\n",
    "print(i);row = tmp_df.iloc[i]\n",
    "im_path = PATH/'words'/f'{row.filename}.png'\n",
    "im = PIL.Image.open(im_path)\n",
    "fig,ax = plt.subplots(figsize=(3,3))\n",
    "ax.imshow(im);ax.set_title(row.word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "row.result = 'ok'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "row.text = 'Leftwing'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df = df.append(tmp_df[tmp_df.result != 'err'], ignore_index=True, sort=False)\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### append single"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "fname = 'c04-134-01-06'\n",
    "im_path = PATH/'words'/f'{fname}.png'\n",
    "im = PIL.Image.open(im_path)\n",
    "im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df = df.append({'filename':fname, 'result':'ok', 'word':'Greensleeves'}, ignore_index=True)\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### paste images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "pw,ph = pim.size\n",
    "pw,ph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "w,h = im.size\n",
    "w,h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "nim = PIL.Image.new('RGB', (w+pw,ph), color=(255,255,255))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "nim.paste(pim, (0,0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "nim.paste(im, (pw,ph-h))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "nim.save(im_path)\n",
    "nim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### save csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df.to_csv(PATH/'edited_line.csv', columns=['filename', 'result', 'text'], index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add char_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# calculate character lengths\n",
    "lgts = df.label.apply(len)  \n",
    "df['char_len'] = lgts.astype('int32')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add num_lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "fname = 'edited_pg.csv'  #'edited_word.csv'\n",
    "CSV = PATH/fname\n",
    "FOLDER = 'paragraphs'\n",
    "\n",
    "df = pd.read_csv(CSV)\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "num_lines = df.label.apply(lambda x: len(x.split('\\n')))\n",
    "df['num_lines'] = num_lines\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df.to_csv(CSV, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "six_lines = df[df.num_lines>=6]\n",
    "six_lines.to_csv(PATH/'edited_cat6up.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Helpers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def standardize_imgs(imgs, baseheight):\n",
    "    resized_imgs = []\n",
    "    for img in imgs:\n",
    "        hpercent = (baseheight / float(img.size[1]))\n",
    "        wsize = int((float(img.size[0]) * float(hpercent)))\n",
    "        img = img.resize((wsize, baseheight), PIL.Image.ANTIALIAS)\n",
    "        resized_imgs.append(img)\n",
    "    return resized_imgs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def resize_max(im, size=1000):\n",
    "    \"Resize an image so that the largest dimension is of specified size\"\n",
    "    r,c = im.size\n",
    "    ratio = size/max(r,c)\n",
    "    return im.resize((int(r*ratio), int(c*ratio)), PIL.Image.ANTIALIAS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def variable_padding(im, top_left=False, size=None):\n",
    "    padLeft = random.randint(30,100)\n",
    "    padTop = random.randint(30,100)\n",
    "        \n",
    "    new_im = PIL.Image.new('RGB', (sz, sz), color=(255,255,255))  # new white image\n",
    "\n",
    "    new_im.paste(im, box=(padLeft, padTop))\n",
    "    return new_im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def square_max(im, top_left=False, size=None):\n",
    "    '''\n",
    "    Add whitespace to square an image by its largest dimension or specified size.\n",
    "    Args:\n",
    "        top_left: image is aligned with the top_left corner\n",
    "        size: size of final squared image.  If left blank size = largest dimension\n",
    "    '''\n",
    "    \n",
    "    r,c = im.size\n",
    "    if size is not None and size > max(r,c):\n",
    "        sz = size\n",
    "    else:\n",
    "        sz = max(r,c)\n",
    "        \n",
    "    new_im = PIL.Image.new('RGB', (sz, sz), color=(255,255,255))  # new white image\n",
    "    \n",
    "    # box logic\n",
    "    if top_left:\n",
    "        box = (0,0)\n",
    "    else:\n",
    "        if sz == r:\n",
    "            box = (0,random.randint(0,sz-c)) \n",
    "        elif sz == c:\n",
    "            box = (random.randint(0,sz-r),0)\n",
    "        else:\n",
    "            box = (random.randint(20,sz-r),random.randint(20,sz-c))\n",
    "            \n",
    "    new_im.paste(im, box=box)\n",
    "    return new_im"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def resize_dir(fn, src, targ=None):\n",
    "    if targ is None: targ = src\n",
    "    dirs = os.listdir(src)\n",
    "    for item in progress_bar(dirs):\n",
    "        if os.path.isdir(src/item): continue     # skip if src dir\n",
    "#         if os.path.isfile(targ/item): continue   # skip if file exists in targ dir\n",
    "        im = PIL.Image.open(src/item)\n",
    "        rsz = fn(im)\n",
    "        rsz.save(targ/item)\n",
    "        rsz.close()\n",
    "        im.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def resize_to_square(src_dir, targ_dir, size):\n",
    "    \"Resize and square all images in src_dir and save in targ_dir\"\n",
    "    resize_dir(partial(resize_max, size=size-50), src_dir, targ_dir)\n",
    "    resize_dir(partial(square_max, size=size), targ_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def show_sample(df, path, row=2, col=2, show_files=False):\n",
    "    fig, axes = plt.subplots(row,col, figsize=(20, 10))\n",
    "    samp = df.sample(row*col).values #=> outputs as an array [[filename, labels]]\n",
    "    for i,ax in enumerate(axes.flat):\n",
    "        row = samp[i]\n",
    "        ax.imshow(PIL.Image.open(path/row[0]))\n",
    "        title = row[1]+f\"\\n{row[2]}\" if show_files else row[1]\n",
    "        ax.set_title(title)\n",
    "\n",
    "#     plt.tight_layout(pad=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Synthesize Lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def create_img(imgs, targ_path, num_lines, max_size, pad=30):\n",
    "    w = 1\n",
    "    h = num_lines\n",
    "        \n",
    "    widths, heights = zip(*(i.size for i in imgs))\n",
    "    \n",
    "    median_height = int(np.median(heights))\n",
    "    stzd_imgs = standardize_imgs(imgs, median_height)\n",
    "    lines = [stzd_imgs[i:i + w] for i in range(0, len(stzd_imgs), w)]\n",
    "    \n",
    "    total_width = max([np.sum([word.size[0] for word in line]) for line in lines]) + (pad*(w+1))   \n",
    "    total_height = (median_height * h) + (pad*(h+1)) #sum(heights)\n",
    "\n",
    "    new_im = PIL.Image.new('RGB', (total_width, total_height), color=(255,255,255))\n",
    "\n",
    "    y_offset = pad\n",
    "    x_offset = pad\n",
    "    \n",
    "    for line in lines:\n",
    "        x_offset = pad\n",
    "        for word in line:\n",
    "            new_im.paste(word, (x_offset,y_offset))\n",
    "            x_offset += word.size[0] + pad\n",
    "        y_offset += median_height + pad\n",
    "    \n",
    "    if max_size: \n",
    "        resize_max(new_im, max_size).save(targ_path)\n",
    "    else:\n",
    "        new_im.save(targ_path/fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# number of words/image\n",
    "def create_synth_data(df, num, num_lines, src_path, targ_path, max_size=1000, offset=0):\n",
    "    d={}\n",
    "    for i in progress_bar(range(num)):\n",
    "        samp = df.sample(num_lines)\n",
    "        files = list(map(lambda x: x+'.png', samp.filename.values))\n",
    "        imgs  = [PIL.Image.open(src_path/f) for f in files]\n",
    "        \n",
    "        # split into rows with \\n\n",
    "        label = '\\n'.join([' '.join(row) for row in np.array_split(samp.text.values, num_lines)])\n",
    "#         label = ' '.join(samp.text.values)\n",
    "\n",
    "        fname = str(num_lines)+'_'+'{:04d}'.format(i+offset)+'.png'\n",
    "        create_img(imgs, targ_path/fname, num_lines, max_size)\n",
    "        [f.close() for f in imgs]\n",
    "        d[fname] = label\n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "src_path = PATH/'lines'\n",
    "synth_path = PATH/'edited_cat_lines'\n",
    "\n",
    "!rm -rf {synth_path}\n",
    "os.makedirs(synth_path, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#single\n",
    "num_lines = 3\n",
    "d = create_synth_data(df, 10, num_lines, src_path, synth_path)\n",
    "\n",
    "synth = pd.DataFrame({'filename': list(d.keys()), 'labels': list(d.values())})\n",
    "synth.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "show_sample(synth, synth_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "#multi\n",
    "for num_lines in progress_bar(range(3,15)):\n",
    "    d = create_synth_data(df, 1000, num_lines, src_path, synth_path)\n",
    "    synth = pd.DataFrame({'filename': list(d.keys()), 'label': list(d.values())})\n",
    "    \n",
    "    CSV = str(synth_path)+'_'+str(num_lines)+'.csv'\n",
    "    synth.to_csv(CSV, columns=['filename', 'label'], index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "a = pd.read_csv(PATH/'edited_cat_lines_3.csv')\n",
    "b = pd.read_csv(PATH/'edited_cat_lines_4.csv')\n",
    "c = pd.read_csv(PATH/'edited_cat_lines_5.csv')\n",
    "d = pd.read_csv(PATH/'edited_cat_lines_6.csv')\n",
    "e = pd.read_csv(PATH/'edited_cat_lines_7.csv')\n",
    "f = pd.read_csv(PATH/'edited_cat_lines_8.csv')\n",
    "g = pd.read_csv(PATH/'edited_cat_lines_9.csv')\n",
    "h = pd.read_csv(PATH/'edited_cat_lines_10.csv')\n",
    "i = pd.read_csv(PATH/'edited_cat_lines_11.csv')\n",
    "j = pd.read_csv(PATH/'edited_cat_lines_12.csv')\n",
    "k = pd.read_csv(PATH/'edited_cat_lines_13.csv')\n",
    "l = pd.read_csv(PATH/'edited_cat_lines_14.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "new = pd.concat([a,b,c,d,e,f,g,h,i,j,k,l], ignore_index=True)\n",
    "len(new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "CSV = str(synth_path) + '.csv'\n",
    "new.to_csv(CSV, columns=['filename', 'label'], index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "show_sample(new, synth_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Synthesize Words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## num words / line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def create_img(src_path, targ_path, files, fname, sz=None, pad=30):\n",
    "    if sz==None: sz=(1, len(files))  #(w,h)\n",
    "    w = sz[1]\n",
    "    h = sz[0]\n",
    "        \n",
    "    imgs = [ PIL.Image.open(src_path/f) for f in files ]\n",
    "    widths, heights = zip(*(i.size for i in imgs))\n",
    "    \n",
    "    median_height = int(np.median(heights))\n",
    "    stzd_imgs = standardize_imgs(imgs, median_height)\n",
    "    lines = [stzd_imgs[i:i + w] for i in range(0, len(stzd_imgs), w)]\n",
    "    \n",
    "    total_width = max([np.sum([word.size[0] for word in line]) for line in lines]) + (pad*(w+1))   \n",
    "    total_height = (median_height * h) + (pad*(h+1)) #sum(heights)\n",
    "\n",
    "    new_im = PIL.Image.new('RGB', (total_width, total_height), color=(255,255,255))\n",
    "\n",
    "    y_offset = pad\n",
    "    x_offset = pad\n",
    "    \n",
    "    for line in lines:\n",
    "        x_offset = pad\n",
    "        for word in line:\n",
    "            new_im.paste(word, (x_offset,y_offset))\n",
    "            x_offset += word.size[0] + pad\n",
    "        y_offset += median_height + pad\n",
    "        \n",
    "    new_im.save(targ_path/fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# number of words/image\n",
    "def create_synth_data(df, src_path, targ_path, num, sz, offset=0, randomize=False, pre=''):\n",
    "    d=[]\n",
    "    for i in progress_bar(range(num)):\n",
    "        if randomize:\n",
    "            r = random.randint(1,sz[0])\n",
    "            c = random.randint(1,sz[1])\n",
    "        else:\n",
    "            r,c = sz\n",
    "            \n",
    "        num_samp = np.product((r,c))\n",
    "        res = df.sample(num_samp)\n",
    "        files = list(map(lambda x: x+'.png', res.filename.values))\n",
    "        \n",
    "        # split into rows with \\n\n",
    "        label = '\\n'.join([' '.join(row) for row in np.array_split(res.word.values, r)])\n",
    "#         label = ' '.join(res.word.values)\n",
    "        \n",
    "        fname = pre+'{:04d}'.format(i+offset)+'.png'\n",
    "        create_img(src_path, targ_path, files, fname, (r,c))\n",
    "        row = {'filename': fname, 'label': label, 'source': files}\n",
    "        d.append(row)\n",
    "    return d"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "src_path = PATH/'words'\n",
    "synth_path = PATH/'edited_sm_synth'\n",
    "!rm -rf {synth_path}\n",
    "\n",
    "os.makedirs(synth_path, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "d = create_synth_data(df, src_path, synth_path, 50000, (4,3), randomize=True, pre='sm_')\n",
    "len(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "synth = pd.DataFrame(d)\n",
    "synth.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# resize_dir(partial(resize_max, size=512), src_path)\n",
    "# resize_dir(partial(square_max, size=1000), synth_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "show_sample(synth, synth_path, show_files=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "CSV = str(synth_path) + '.csv'\n",
    "synth.to_csv(CSV, columns=['filename', 'label'], index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## size of image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "synth_path = PATH/'large_synth_words_test'\n",
    "!rm -rf {synth_path}\n",
    "\n",
    "os.makedirs(synth_path, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def create_img(sz, fname, pad=30, median_height=None):\n",
    "    # TODO: randomize padding\n",
    "    new_im = Image.new('RGB', (sz,sz), color=(255,255,255))\n",
    "    \n",
    "    res   = df.sample(50)\n",
    "    files = list(map(lambda x: x+'.png', res.filename.values))\n",
    "    lbls  = res.word.values.tolist()\n",
    "    imgs  = [ PIL.Image.open(PATH/'words'/f) for f in files ]\n",
    "    \n",
    "    if median_height is None:\n",
    "        w, h  = zip(*(i.size for i in imgs))\n",
    "        # standardize heights and sort longest to shortest words\n",
    "        median_height = int(np.median(h))        # TODO: randomize this between mean/std\n",
    "        \n",
    "    stzd_imgs = standardize_imgs(imgs, median_height)\n",
    "    \n",
    "    #loop through standardized images and find the next image which satisfies the condition\n",
    "    labels = []\n",
    "    y_offset = pad\n",
    "    while y_offset+median_height+pad < sz:        \n",
    "        x_offset = pad\n",
    "        \n",
    "        gen = (i for i,x in enumerate(stzd_imgs) if x.size[0]+x_offset+pad <= sz)\n",
    "        lines = []\n",
    "        for idx in gen:\n",
    "            word = stzd_imgs.pop(idx)            \n",
    "            lines.append(lbls.pop(idx))\n",
    "            new_im.paste(word, (x_offset,y_offset))\n",
    "            x_offset += word.size[0] + pad\n",
    "        y_offset += median_height+pad\n",
    "        labels.append(' '.join(lines))\n",
    "\n",
    "    new_im.save(synth_path/fname)    \n",
    "    return '\\n'.join(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# size of image\n",
    "def create_synth_data(qty, sz, fname_offset=0):\n",
    "    d={}\n",
    "    for i in tqdm(range(qty)):\n",
    "        fname = '{:04d}'.format(i+fname_offset)+'.png'\n",
    "        p = random.randint(10,20)\n",
    "        h = random.randint(25,35)\n",
    "        d[fname] = create_img(sz, fname, pad=p, median_height=h)\n",
    "    return d\n",
    "\n",
    "d = create_synth_data(20000, 512)\n",
    "len(d)\n",
    "# ~25min to create 5000 1000x1000 images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "synth = pd.DataFrame({'filename': list(d.keys()), 'labels': list(d.values())})\n",
    "synth.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "show_sample(synth, synth_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Numericalize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Chars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# same as used in single word / multi-word\n",
    "itos = pickle.load(open(PATH/'char_itos.pkl', 'rb'))\n",
    "len(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "joined_labels = list(synth.labels) #list(map(lambda x: ' '.join(x), labels))\n",
    "\n",
    "stoi = collections.defaultdict(lambda: 82, {v:k for k,v in enumerate(itos)})\n",
    "ids = np.array([np.array([stoi[letter] for letter in word] + [3]) for word in joined_labels])\n",
    "\n",
    "# convert to strings (as labels)\n",
    "str_ids = np.array([' '.join(str(l) for l in w) for w in ids]).reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "synth['char_ids'] = str_ids\n",
    "synth.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "CSV = str(synth_path) + '.csv'\n",
    "synth.to_csv(CSV, columns=['filename', 'char_ids'], index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# # multi-line\n",
    "# CSV = str(targ_path)+'_'+str(num_lines)+'.csv'\n",
    "# synth.to_csv(CSV, columns=['filename', 'char_ids'], index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Add to existing CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "CSV = PATH/'large_synth_words_10000.csv'\n",
    "csv = pd.read_csv(CSV)\n",
    "len(csv)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# CSV = PATH/'synth_words_50000.csv'\n",
    "CSV = PATH/'large_synth_words_50000.csv'\n",
    "\n",
    "new = pd.concat([csv, synth[['filename', 'char_ids']]], ignore_index=True)\n",
    "new.to_csv(CSV, columns=['filename', 'char_ids'], index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "len(new)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Concatenate CSVs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "a = pd.read_csv(PATH/'cat_lines_11.csv')\n",
    "b = pd.read_csv(PATH/'cat_lines_12.csv')\n",
    "c = pd.read_csv(PATH/'cat_lines_13.csv')\n",
    "d = pd.read_csv(PATH/'cat_lines_14.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "new = pd.concat([a,b,c,d], ignore_index=True)\n",
    "len(new)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "new.to_csv(PATH/'cat_lines_11-14.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def to_string(row):\n",
    "    return ''.join([itos[int(c)] for c in row.split(' ')])\n",
    "\n",
    "\n",
    "fig, axes = plt.subplots(2,2, figsize=(20, 20))\n",
    "for i,ax in enumerate(axes.flat):\n",
    "    row = a.iloc[i]\n",
    "    im = Image.open(targ_path/row.filename)\n",
    "    ax.imshow(im)\n",
    "    label = to_string(row.char_ids)\n",
    "#     label = '\\n'.join(textwrap.wrap(row.labels, 70))\n",
    "    ax.set_title(label)\n",
    "\n",
    "plt.tight_layout(pad=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "itos = pickle.load(open(PATH/'synth_word_itos.pkl', 'rb'))\n",
    "len(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "joined_labels = list(synth.labels)\n",
    "\n",
    "stoi = collections.defaultdict(lambda: 2, {v:k for k,v in enumerate(itos)})\n",
    "ids = np.array([np.array([stoi[word] for word in line.split(' ')]+[3]) for line in joined_labels])\n",
    "\n",
    "# convert to strings (as labels)\n",
    "str_ids = np.array([' '.join(str(l) for l in w) for w in ids]).reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "synth['word_ids'] = str_ids\n",
    "synth.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Add to existing CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "CSV = PATH/'large_synth_word_ids_10000.csv'\n",
    "csv = pd.read_csv(CSV)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# CSV = PATH/'synth_words_50000.csv'\n",
    "CSV = PATH/'large_synth_word_ids_50000.csv'\n",
    "\n",
    "new = pd.concat([csv, synth[['filename', 'word_ids']]], ignore_index=True)\n",
    "new.to_csv(CSV, columns=['filename', 'word_ids'], index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "len(new)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "### Modify csv/itos to match previous versions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "itos_old = pickle.load(open(TMP_PATH/'synth_word_itos.pkl', 'rb'))\n",
    "\n",
    "# same as used in single word / multi-word\n",
    "itos = pickle.load(open(TMP_PATH/'char_itos.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "res = [''.join([itos_old[int(c)] for c in line.split(' ')]) for line in csv.char_ids]\n",
    "csv['words'] = res\n",
    "csv.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "joined_labels = list(csv.words) #list(map(lambda x: ' '.join(x), labels))\n",
    "\n",
    "stoi = collections.defaultdict(lambda: 2, {v:k for k,v in enumerate(itos)})\n",
    "ids = np.array([np.array([stoi[letter] for letter in word]+[3]) for word in joined_labels])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# convert to strings (as labels)\n",
    "str_ids = np.array([' '.join(str(l) for l in w) for w in ids]).reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "csv['char_ids'] = str_ids\n",
    "csv = csv[['filename', 'char_ids']]\n",
    "csv.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "### Verify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def label_text(pred):\n",
    "#     ints = to_np(pred).astype(int)\n",
    "#     ints = np.trim_zeros(ints)   # remove padding (0)\n",
    "    return ''.join([itos[int(i)] for i in pred])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(5,1, figsize=(10, 10))\n",
    "for i,ax in enumerate(axes.flat):\n",
    "    row = synth.iloc[i]\n",
    "    im = Image.open(synth_path/row.filename)\n",
    "    ax.imshow(im)\n",
    "    ax.set_title(label_text(row.char_ids.split(' ')))\n",
    "    \n",
    "plt.tight_layout(pad=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Batch Resize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "targ_path = PATH/'test_resize'\n",
    "os.makedirs(targ_path, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "src_path = PATH/'cat_lines'\n",
    "# targ_path = PATH/'resized_cat_lines'\n",
    "# os.makedirs(targ_path, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "resize_to_square(src_path, targ_path, 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# resize_dir(partial(resize_max, size=512), src_path)\n",
    "resize_dir(partial(square_max, size=1000), src_path, targ_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# V1 add padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "src_path = PATH/'fonts'\n",
    "targ_path = PATH/'fonts_resize'\n",
    "\n",
    "os.makedirs(targ_path, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def pad_one(fpath, i, targ_path):\n",
    "    dest = targ_path/fpath.name\n",
    "    img = PIL.Image.open(fpath)\n",
    "    bg_color = img.getpixel((0,0))\n",
    "    bottom_pad = (random.randint(10,100) if img.size[0] > 264 else 0)\n",
    "    img = PIL.ImageOps.expand(img, border=(random.randint(10,100),random.randint(10,100),0,bottom_pad), fill=bg_color)\n",
    "    img = resize_max(img)\n",
    "    img.save(dest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "il = ImageList.from_folder(src_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "parallel(partial(pad_one, targ_path=targ_path), il.items)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Convert datasets to new itos.pkl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "itos = pickle.load(open(TMP_PATH/'char_itos.pkl', 'rb'))\n",
    "len(itos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "voc = pickle.load(open(TMP_PATH/'itos.pkl', 'rb'))\n",
    "len(voc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "def convert_char_itos(old_itos, new_itos, path, old_fname, new_fname):\n",
    "    old_csv = pd.read_csv(path/old_fname)\n",
    "    # convert to text and remove _eos_ token\n",
    "    res = [''.join([old_itos[int(c)] for c in line.split(' ')[:-1]]) for line in old_csv.char_ids]    \n",
    "    # xxunk: 3, xxeos: 2\n",
    "    stoi = collections.defaultdict(lambda: 3, {v:k for k,v in enumerate(new_itos)})\n",
    "    ids = np.array([np.array([stoi[letter] for letter in word] + [2]) for word in list(res)])\n",
    "\n",
    "    # convert to strings (as labels)\n",
    "    str_ids = np.array([' '.join(str(l) for l in w) for w in ids]).reshape(-1,1)\n",
    "    old_csv['char_ids'] = str_ids\n",
    "    old_csv.to_csv(path/new_fname, columns=['filename', 'char_ids'], index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# old,new = 'small_synth_words.csv','sm_synth.csv'\n",
    "# old,new = 'multi_synth_words.csv','3x2_synth.csv'\n",
    "# old,new = 'paragraphs.csv','pg.csv'\n",
    "# old,new = 'mix_words_dl.csv','full_mix.csv'\n",
    "# old,new = 'mix_words.csv','mix.csv'\n",
    "old,new = 'downloaded_images.csv', 'dl.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "convert_char_itos(itos, voc, PATH, old, new)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Verify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "csv = pd.read_csv(PATH/new)\n",
    "\n",
    "res = [''.join([voc[int(c)] for c in line.split(' ')[:-1]]) for line in csv.char_ids]\n",
    "csv['text'] = res\n",
    "csv.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create Mix Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# dl = pd.read_csv(PATH/'downloaded_images.csv')\n",
    "# sm = pd.read_csv(PATH/'edited_sm_synth.csv')\n",
    "# cat = pd.read_csv(PATH/'edited_cat_lines.csv')\n",
    "# pg = pd.read_csv(PATH/'edited_pg.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "dl = pd.read_csv(PATH/'downloaded_images.csv')\n",
    "sm = pd.read_csv(PATH/'combo.csv')\n",
    "cat = pd.read_csv(PATH/'edited_cat_lines.csv')\n",
    "pg = pd.read_csv(PATH/'edited_pg.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "mix = pd.concat([dl,sm,cat,pg], ignore_index=True)\n",
    "len(mix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "mix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "mix.to_csv(PATH/'mix.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <=6 lines:  fonts+cat_lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "a = pd.read_csv(PATH/'edited_cat6up.csv')\n",
    "a.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "b = pd.read_csv(PATH/'combo6up.csv')\n",
    "b.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "mix = pd.concat([a,b,df], ignore_index=True)\n",
    "len(mix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "mix.sort_values('num_lines', inplace=True)\n",
    "mix.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "mix.to_csv(PATH/'combo_cat_pg_dl_sorted.csv', index=False)\n",
    "# max:  14 lines, 723 chars"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(PATH/'combo_cat_pg.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true
   },
   "source": [
    "# SentencePiece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "import sentencepiece as spm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "df.label.values[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "name = str(PATH/'spm_train')\n",
    "fname = name + '.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# puts dataset into format expected by sentencepiece:\n",
    "# .txt file entries separated by \\n\n",
    "def write_text(texts, filename):\n",
    "    with open(filename, 'w', encoding='utf-8') as f:\n",
    "        for text in texts:\n",
    "            f.write(text + \"\\n\")\n",
    "\n",
    "write_text(df.label.values, fname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "spm.SentencePieceTrainer.Train(\n",
    "    f\"--unk_id=3 --pad_id=0 --input={fname} --model_prefix={name}_8k --vocab_size=8000 --user_defined_symbols=\\n\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "sp = spm.SentencePieceProcessor()\n",
    "sp.Load(name+'_8k.model')\n",
    "sp.SetEncodeExtraOptions(\"bos:eos\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "vocab = {i: sp.id_to_piece(i) for i in range(8000)}\n",
    "vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "ids = sp.EncodeAsIds(df.label.values[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": [
    "sp.DecodeIds(ids+[0,0,0,0,0,0,0,0]) == df.label.values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {},
    "heading_collapsed": true,
    "hidden": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  },
  "notify_time": "30",
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}