BERT_GEC_Implementation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT_GEC_Implementation.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LfiBXakLdXPM",
        "colab_type": "text"
      },
      "source": [
        "# **Grammar Error Correction using BERT**\n",
        "\n",
        "\n",
        "***Use of BERT Masked Language Model (MLM) for Grammar Error Correction (GEC), without the use of annotated data***\n",
        "\n",
        "Sunil Chomal | sunilchomal@gmail.com"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PXyJCXB6p_jc",
        "colab_type": "code",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/GEC.png": {
              "data": "iVBORw0KGgoAAAANSUhEUgAAAiYAAAFRCAIAAADPY/H1AAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsMAAA7DAcdvqGQAAD8GSURBVHhe7Z39jx3lefeff6q/8ENVVVWVKG1VJIqsIiuqn4jSOsUp4iWJbFX4EYSa9iHQEEhoKKghBFKqwMawwawNPMYUYcyLwRuwl91zvC9eY69f19hrLzzX8h0urlxzz5z7zJkz95z196OvrJn73C/jc811f889Z87s//qcEEIIaQRaDiGEkIag5RBCCGkIWg4hhJCGoOUQQghpCFoOIYSQhqDlEEIIaQhaDiGEkIag5RBCCGkIWg4hhJCGoOUQQghpCFoOIYSQhqDlEEIIaQhaDiGEkIag5RBCCGkIWg4hhJCGoOUQQghpCFoOIYSQhqDlEEIIaQhaDiGEkIag5RBCCGkIWg4hhJCGoOUQQghpCFoOIYSQhqDlEEIIaQhaDiGEkIag5RBCCGkIWg4hhJCGoOUQQghpCFoOIYSQhqDlEEIIaQhaDiGEkIag5RBCCGkIWg4hhJCGoOUQQghpCFoOIYSQhqDlEEIIaQhaDiGEkIag5RBCCGkIWg4hhJCGoOUQQghpCFoOIYSQhqDlEEIIaQhaDiGEkIag5RBCCGkIWg4hhJCG8JZzYXl58fgnM52jC/PzlFWn0104tnj+/PnPPvsse7NawzqO2rGFhdXV1bNnzrjySLU2ahKy4ycWO7PT7oCp7uz0seMLTLTRUnyifWU5ktjd7tGx8T133/fvN916159c9/eU1aYtd9654+EnfjU2Pd1ZuXQpe9dSw6iVq4VRk5AdnZsZf/2ZHz699baHbrhu+x9QVlsevP7eJ+/4r92PznSnmGijovhEyyznypUrU1PTt2z7V9cRldfGzdveOXjoUguSgVGLV0uiJiH7uHN426N/6+ZZKq/ND1x78KMDTLTRUs9Eyyyn0539h+/9s2tMFWnDjd8XM0++8GfU+lIbotadm/neI5vc3EoV6cb7viFrHSbaaKk80dYs58LysiwYXTOqXP/2yJMnT5zEm5gERq2C0kZNQjb++jNuVqXK9cjYvSeXPsnewRQw0SqoJNHWLGd+bmHr3T9ybahy3XTrXTOdo3gTk1Ahan93+w8uXrz0wsQ+V95T/7TjJzLiw48948oHlHab33A161LaqM0fm73757e4KZUq120P3dCZnc7ewRS0f3osypphZ1OJShJtzXI6ne7GzdtcG6pcX99w89zsHN7EJFSIGi0nbdS6R6c3P3Ctm1Kpcm2465q5udnsHUxB+6dHmzW79+5fWDzxl9+81ZU3rJJEW7Ochfl514CKkbxveBOTUCFq7bccV2EYShg1GdrNp1SMRi7REspaTloVRY2WU10jkQmPP/Ub1BengeW898Fh+VdKDk4eQR2Ua7V8Q7UE1NSG0Nc23Dx5OLv0gTrLFz599oWXUVIySt5y7ED5TuxAgqTW9d/6rh0aA5UrYdRkaDeZUjEaiURTBU/dP7vhO6fPnHvjwPtSgpM/nw5FiYOkAKjpMg4VfvbEszY7bDYFO+mZYpEJVaSiqNFyqqv9maDnnJxJ4jSyK2c5PgSJo1y5srr1nodw6uP00m3X8MFHfyW7j/3yN5I2+c9Q0tWllZW/vmkrdtGJjoJ+ykfJb7hOcKjyKgaSVyVbpMQNHaOEUZOh3WRKxaj9iWYVPP9hOXquBtMh2BCFMAktdKe9Zo1sBy+sBTtxwyHFXM+DqChqtJzqan8myPlnTyB75rlZ3iIVXEOtEzwd9VWc9HYU3Q6O4o7BbhQdKg5ANtSE5CW8ioPpqYRRk6HdZErFqP2JZhU8/2E5KBTpeavIS8GGaga20J322MV20HKCndjhtKbreRAVRY2WU13tz4R4y3FnWNBynn/pNWmu630rXY+7U9kliRslfwy6ETxUFMq2gJdEdmiUlCth1GRoN5lSMWp/olkFz/+g5bh0CDYMuoVs29Pe9jag5bieZbeyiqKW2HKCb32kBmlbi9qfCXKqSU28ReO7107N/EmGM88ZiWuIC2uyKye0nruqR3/xnJQgqaQfdIjzXvsJjqLHkN8I5oP0pukEuaG1vEQJoyZDu8k0UjueulWaP/bifbrhKvSlWjppUu1PNKvg+e8sJ5gOwYZ68ksFKUT2udPe1rEfFvNpZTsJplgwl6t5T1HUhmI5OFAMAPLzFGTfi341SNtaNBKZIKcg6st5EzzJdBtopGxDrYxzEVmB/kUolAo41zHK3MLxL1qHvwXFKNptfiN4qLYHQY9Htu2arFwJoyZDu8k0qNsf2Xhx5dOszeefT7z9bNBy9r7/28VTc9/c8ce2bYxqtBw5hrVD/ILJ7ttS4g5ejnD7f252Jf0e80gkmip4/uNEtXO3PZmtB+QTB/YD5JyXEnfaa4LYbjVfUJ7vJJhiJT33q6KoDXGVgwWam56cBvkvJddoZUJjsqdyjcLphDyMObWKlDBqMrSbTIPCrC1O48pF7bGcDXddc3ju/SurV+75ZfbjVjke6TN/8Lbkhh/84ZnlpX4Pe7QSrfL5P6TEGURiVPEf5pyKotao5agDC85d8XZjQsG2rSYl+fv5tK01cK3vehiGRisTGtPwMseeP0Xr5p5KGDUZ2k2mQeVnbbfKeWLiQZnuv+hyDZiHXV7Y+f3A4de0xPWm2wB1xDlWLl+66f4/t/YAg8E6xnZiu4XKLUdkR4/UaCVa5fN/eIlTTVjxVF4PFEWtOcuxbyhMQv4zmETs3beohv+nbmMDFaStuyyD4bQ82APq1KvRygQKShg1GdpNpkH1tBzM13aVgybWe2QbngH/QD+QdmIHeurln+oQWLvgVTTPH5LURzWMIm1tn9gV3CiifFc9xUQbRRVFrTnLUUuQbTiB2I9+btXlm/0kC6Sa1tcK4iLWcmDIdvVjGdIHB2bCKCph1GRoN5kGlZ+1e1oOyi0yp8MM8pO7dqK2IYXqBLoh1abmJ6UHjGuvoYlsW5H2mXcUV5Kv0FNMtFFUUdRaYTn27ltrJKqelhO8TwNthydmwigqYdRkaDeZBpWflHVC1w0pzFsOylXVLEevoeG7GflXCu1YrhO329NygodaLibaKKooas1ZjnWC/MUxvfsW7gL7UZVbjn1VK7sehqFBMqHTHfRJhXVFrTGHrlf508BVKFLCqMnQbjINKt5y9KIZmtjvWkQ9Lcf2Zu1Htk+cOTZ97CPZxXb3+JTrx90+oF2VWw4OyblXT41WoskJqZ+tm1E7U7goao3ePiBOg1EFvEH6ZuHKGCqjEFgfKrIcvZ1XyF9bG174B8mEX4/9dnFxoL8CUlfU9G105TFyZm8XssNW/jRwFYqUMGoytJtMg4q0HGzndwU4QU/LkW1xFDQRUCLCAdhvcew1NCuxvazx55/b+llR6CZp54sxGq1EkxOyPAtkvrKz4uAaJIXLpfdMC/3eulYUtSFazrrXgJkwtnPX6TNns/3+aUPUrkLLGSRqMrSbTKkYjVaiNW85w5N+YeEyPUZFUaPlVNeAmSAaf/Hl5eULWVGfxETNTseaCVh9ohM7ZctZVf5kWcGlSpHlBMft2X/RMYgwEAqLFruo2VMJoyZDu8mUilH7E02kl3DeOPC+Wo47b1024aTNn9tYW+hjp1EBu4Lk4K+eewnbLhfi08cN4dIQrUT6fQf+d/FZJiqKGi2nugbPBNHEnr0rK5ez0n6IiVpw6pezx66R7fkq5yVMxZqHfsyRsxmJpLKnMtBWsu3GDfbvDsbVQScoRG+6rUPYsWKUMGoytJtMqRi1P9FwWurcbU94d97Ktl3lBOvADzQvUG6TAl6ifqAp4GqWHIYbwqWhSi+soSv3armKokbLqa6STNATPUZ79725urqatYwmJmrBqR+FthzbOB3d5ybZwLmozdEzZJuI9CzX5lKoDYv6lw2tbOvottZRtFB7QPMYJYyaDO0mUypG7U80PfNlW05Il2gKTmxrOcE6mOhRWRRMHNnOp1t5iin5IbSOzSP1GwEDFTlTUEVRo+VUV12ZINr/1rtZy2hiomZPUNnWrNB1tJxzWid4vqJQtgU9QVW2iaiC5ci2PRhbR7dtZZUWBl8tUcKoydBuMqVi1P5EK7Gc/JmZtxxXZxiWUz6EyKYhSrR/6z165D1VFDVaTnXVmAmigx/8LmscR0zU9BS06/2ix9AGz1c57UpOMttEpOdocNxg/zU+hdq+WqKEUZOh3WRKxaj9iYaTMHjCu/NWJJaja4VgnXotJ2aI4BPZ7Vh4SXbz/50iFUXtKrIc+w7WonozQXRkaiZrH0Fk1OT8Rv3nX3oNX8bo2aNXzGS76HzFhuLePdtEpDkg2/lxg/27g0Gdak+hRs2eShg1GdpNpo0pf+N1UMFq+Vu07W3WDWhEE00K8+etLcRJm6/j/CCYOLIdaTm6DYJDuDREoQgf+4B8ELx9+wPSf6TrFEUt1nJ07HiXU9mFZELZGNSi2jNBNL+wmHXRi8hMGET4yIaIY3vYcXQeNgwljJoM7SbTxjQ8y8k/mKB2rftEW5cqilqU5djJGksw+2pPtcRyatcwMmFs566lpVNZL6U0kwn5z0euQr0aUcuJjJoM7SbTxjSI5aiCi5sRtZy2Jdr6U1HUoiwHSxy3PsDs8EUn2TdOUuLuCsdHY+wK6CGmIYbQpZVWcw1VbhWJydGOLi9pneBw7lBjPHIYmSDaOT4R8xsCZkI1JYyaDO0m034llnDh4vkX3nwaHeKX/HjKgP0jBfAGgBJ4ycLJLgptQ5TANoLVilY57m8oYBf94HE4dVkRE20UVRS1KMvRK336ORezv7UQTOWygclarzNKBbvKiW+oDiFm8N4Hh2U331C2oaDlSFf20qS1nOBw+nWCXoot15AyQTSxZ2/WUTHMhGpKGDUZ2k2m/QqWgKkcz6oRA4BzuOetwWm0TrDhlh//FSxBvSRYrchyZEPa2lWONCl/1k41MdFGUUVR6+P2AbvmwPRtkUJM5Vh/WA+wlhPfUFrlDcOC+vZVZznaRMuxHRxONjCcNkfPJRpeJsQ8FYqZUE0JoyZDu8m0X1k70W03v+u8b+uUNMwOzliOqxZvOehQ1kZSoeixbBXERBtFFUWtD8sRYblz0NxZa1+Nt5yYhkHLcQ1V9lXZVs/Qy2XSudYJDodC2RbwUk8NKROmZ7LLGuUwE6opYdRkaDeZ9qsS50ChKN5yxC2wNlILCVaLtxzsSp/vz+y3hQOKiTaKKopa7Hc5mM11ssaGfukCBady2bbmEd8Qiyr0ML57zTDyDVXaAzwGllPjD1CCGkYmHJr8KOuiF8yEakoYNRnaTab9CjbgLnw5y7F+oPYTbKiWoyXBaj0tR6/p6eiCHs/guhoSzU6YjcnOhLWrKGr9fZcj6KSP9whgig9O5bam2xXKG4pXoZp9CeSvfWnlYfwAJajaM6Gvn0YPKRNsrO0qs3bZtW+TShg1GdpNpv0KlhC8C8BO8XALAGOQhva+A7uOkd2lc5/IhtQMViu3HPUY7Lq/o1OL1mWiiWwK2HmvMbXXcta9sDaCm2I7ZjasNxP6fQDUkKKmS9Khno6i9WE5fUVNhnaTab+CSdS4gKhd9d6rBq3LRBOlSgEVLSelypdQQdWYCXte2dfvY26HFDVJA/z37YVNJ3sjCVZF+hR0eRXnsVaQErtykhy7/lvftfejY4h8qyEpYdRkaDeZ9qv2Ww4WPfUe4Sgmmjvn4SuaOLteecOlgF3l2LlIcyr4MxLIfkq220WZ6P4sgnvYB3pAySCrrqKo0XKqq65MeHHi1Qp/zGNIUdNUKfJdzQ05Nd/74PD//s7/kfruuzrrIrL9N1vuRMrZvLIf8YKtZHsYShg1GdpNplSMRjHR8ue8SxzJF5sC+irOfziEfuxDISpLYT43tRD5q3aCPNJtvOqyVbvFWLJRyxX1oqjRcqqrJBN6ommwc3yi2p8sHEbU7EczXeu480/yxJboKY5dZI5FXrLdCkiDfL5ZtMPalTBqMrSbTKkYjWKi5c95lziioOVYR1H70Q1bE51AWkFeRfOSTNTkst3aHlDfDdGviqJGy6muwTNhbOeuyn+YfRhRc5+VsoG+zAooxnLcyapN7KvBfNMmw1PCqMnQbjKlYjSKiZY/513ioE4+BapZjl5Pe33/QfQZrBZjObKt19a0ZgUVRY2WU12DZ0KnO5vt988womZPU3Udd+FYUkLrjO/ehwtr7iR2TTTZbFubgcFWQ1LCqMnQbjKlYjSKiZY/513iyEs2BTT1bA6q/fS0HBH6F5BHwZwKWg4sSg8v+IcMKqgoamksx707I6oBM2HywyPZTiWGFDUNjSDnYvBx5ZIqqCDnrjuJRUgJYBNGdk8unZYNZItWc7uCfsobhhJGTYZ2k2nzcr/cHAmNYqIFz3mbOLJrU8Aaic1BlMRYjo6oL+VzKm859q4ElOtnTbXDaiqKWgLLsW9ZhedSt0eDZMKBd756HmI1Go7aulHCqMnQbjJtXleb5TDRUqkoagksR1dwWuLMFp+p1WwFLP20rYC7DFGu1zFRpzENkgmDw0yopoRRk6HdZFpN+O0L+sQPMPVHnYL+zNM9c9q2EuIbSqFIf16q1VzD/FHVJSbaKKooagksR73EfUsGz1BD6nmX4W92/b/garExMRNGUQmjJkO7ybSaZPa3z5jB7G8tRLaxoY+u0ccB2FVOfEN93ID4ygedAzuevi3f0B1VjWKijaKKopbs9gFdr4hV2CuVum1XOYI4jf22TYQKsiQSKxrq1f8iMRNGUQmjJkO7ybSa3DNmdFeRZQecAOsPNQzZtpYT37DoQWqK1NdCDFSjmGijqKKoJbMckXpG0HLUYHRx4yxHhJJ33v+o+atqImbCKCph1GRoN5lWll7F0rneTfTxlhPTMGg5eWuxR+VeGkRMtFFUUdTSfJcj/iEb6i7YcBfW1GC0RDekLe4yhBsJsKuGxUwYRSWMmgztJtNq+sXuH9/z5Z9Bm+y+DZPQL12gEstR84hviC9y0MPut5/DhTXX0B2VfWlAMdFGUUVRS/ldjoA7BWA57lE/KJTdkrsMceNAkqtqovWdCbq4dOWjroRRk6HdZFpNmNalQzUPeAOwf60gbzla0+0K5Q3Fq1DNvgTQMH9UdWl9J1pltTxDi6KW8sKaSpc7rrynUt2rBq3vTLAntDj9kN7k4fVcpIRRk6HdZErFaB0n2iDnf8sztChqo205eNOTXFUTreNMENFyakeGdpMpFaN1nGi0HKoPradMsFc7cYbhhP7ZE8/iaUsA57de8xTg92huH4re84dWqf6EQcKoydBuMqVi1M5Es/fcuvNfXi3KEZRUOP9HLkOLokbLqa52ZkI1lfwKSgrtJx2cc/b8k22crHo/IcrRRDITX7a15E8YJIyaDO0mUypGLUw0PYfxG8EG/oTHyGVoUdRoOdXVwkyoLPsBR5Azqei0Q7lFPuyguX7qwemIXe0nP0RMz+iwRiWMmgztJlMqRi1MNDlvdfoWufO/JEey/f7P/5HL0KKo0XKqq4WZUFmaQnr+lZ/QKFfFnND5IeTVnj3XroRRk6HdZErFqIWJpiczdt35HzyTBzz/88376qf5DC2KGi2nulqYCZWlZ5v++MmddppgOFlx8VfV1wltf1/Vs+falTBqMrSbTKkYtTDR7DnczJ/wyDdveYYWRW3NcmaPzv7Fxn90DaieSpsJ9UYNJ5N0q7+Csic0tvO7QvAxd8ETOj9ETM/osEYljNrsXHfjPX/k5lOqp9qZaDIXo4Kc5+78F+XP5AHP/5HL0FLLmZ27Y/v9rgFVrk1b7ux0ungTk8CoVVDaqM0tdLc//m03n1Ll2vLg9d3Zr+6bah4mWgWVJNqa5Zw+deon//G0a0OV684dDy8cW8SbmARGrYLSRu306VOPPf9DN6VS5br3yTuOHV/I3sEUMNEqqCTR1ixndXV1amr6uk23u2ZUkb6+4eZDkx9evnwZb2ISGLV+lTxqErKPO4c3/cufulmVKtKGu6753ZGDTLTRUnmirVmOcPHixf/Z/w6/0YmRvKFj43tOLp3CW5cQRi1eLYmahGz/B6/xG50Yid+Mv/7M0qmT2XuXDiZavHomWmY5wtlz544c+Xjr3T/auHmb64WCNtz4/Tu23y8G3ga/AYxaT7UtaufOn52a+fDun9+y+YFr3SRLQTfe943tj39b1jdt8BvAROupyET7ynIEWULOzy10Ot2F+XkqL3lnZmfn0i7z8zBq5Wph1NZCdmy2e3TaHSoFdY/OzC10mWijpchE+z3LIYQQQoYHLYcQQkhD0HIIIYQ0BC2HEEJIQ9ByCCGENAQthxBCSEPQcgghhDQELYcQQkhD0HIIIaTV7N335q/HfhupFydezZq1EloOIYS0GloOIYSQhqDlEEIIaQhaDiGEkIag5RBCCGkIWg4hhJCGoOUQQghpCFoOIYSQhqDlEEIIaQhaDiGEkIag5RBCCBkWkx8ecUZSWe++dyjrtB3QcgghpHXMzi2M7dzl/KNfHZmaybprDbQcQghpI0tLp3aOTzgXiZQ0nF9YzDpqE7QcQghpKcvLF/a8ss/ZSU+9OPHq6TNnsy5aBi2HEELay8rK5dffeMuZSonEoi5evJg1bh+0HEIIaTvvvnfIWUtQ+996d3V1NWvTSmg5hBAyAkzPdJ3BOB2a/Cir2mJoOYQQMhosLn4SvKFgbOeuTnc2q9RuaDmEEDIynD5z9sWJV63fiAktLZ3KXm49tBxCCBklVlYuv7r3DfjNxJ69y8sXshdGAVoOIYSMGKurq/vfenfvvjfFfrKiEYGWQwghpCFoOYQQQhqClkMIIaQhaDmEEEIagpZDCCGkIWg5hBBCGoKWQwghpCFoOYQQQhqClkMIIaQhaDmEEEIawlvOheXl4ycWO7PTC/PzlFV3dvrY8YXz589/9tln2ZvVGiRqi8c/mekcdcdMdTrdhWOLLYwaQ1ak1oZM4PRYpPjp8SvLWV1dPTo3M/76Mz98euttD91w3fY/oKy2PHj9vU/e8V+7H53pTq1cupS9a6mRqHW7R8fG99x937/fdOtdf3Ld31NWm7bceeeOh5/41dj0dKclUWPIytXCkAmcHssVPz1mlnPlypWPO4e3Pfq3riMqr80PXHvwowOXWpAMErWpqelbtv2rS1oqr42bt71z8FDyqDFk8WpJyAROj/HqOT1mltOdm/neI5tcY6pIN973DTHz5Av/Tnf2H773zy5RqSJtuPH78sE5bdQYsr7UhpAJnB77Uvn0uGY5F5aXZcHomlHlemTs3pNLn+BNTIJEbWx8j81Pqqf+7ZEnT544mb2DjcOQVVDakAmcHiuoZHpcs5z5Y7N3//wW16Yx3f7Ixosrn068/Sx2n3r5p1dWr9zzy2THE6nbHrqhMzuNNzEJ83MLW+/+kU3OIWn33v0YcWHxxF9+81b3aryknwF7GFw33XrXTOco/jvN01jI1pPShkxIOz2OqEqmxzXL6R6d3vzAta5NYxpRy9lw1zVzcyn/2Hin0924eZvLz3r1tQ03Tx6evnJldes9D8nuP+34yQsT+2yFvtQGy/n6hpvnZueyd7BxGgjZ+lPakAlpp8cRVcn0uGY5C/PzrkGTKrIcOejDc+/jKB978T6tiRKtn1DyvuFgkiCju+SsXX93+w8uXrzkbObPbvjO6TPn3jiwFhq8hGo4KpSgDkrEZq7/1nfFurArPPzYM8FWzShh1BoI2bpU8kRziU/FqChq7bUc2Vi5fOmm+//cVrPeg+2ESp4JLjNr1+NP/UYGgkOoYCeXVlb++qatsgvnsC4i23+z5U6sZmRhpD3YVU6wlWw3oIRRayBk61LJE80lPhWjoqi113J2PHUrDhHWortK8oVO8kxwmVm7SixH1yUwFYu8ZFc5Qt5ygq3Q4bCVMGoNhGxdKnmiucSnYlQUtfZajmzrtTV5FZaTfGVjlTwTXGbWLhiDM4Og5ThbEnfBMqholRNs1YwSRq2BkK1LJU80l/hUjIqilt5ybvjBH55ZXlo8NffNHX8Mj8H2L3b/WIwHr05234YzyYZrnlDJM8FlZu3C7QP2Gtqbbx9yloPLYgcnj2AXUsux6yQtlO1gq2aUMGoNhGxdKnmiucSnYlQUtfSWI4Kv4ID0+xst1BJ7ba0Nd7UlzwSXmcMQXAcjwjCc5YjsVTLc3gZHkd2TS6f1exqt5nYFvSmuASWMWjMha4mKFrI4N/q6jpo80VziR0ovzNRyhaaFl3nKVRS1VljOiCp5JrjMpGKUMGoxIZO1YFZ5sBvK7WXMwVWhN2s5tvl6tRxchsna/P53AdYt9r7/W1zFsW1jVKPlyDGsHeIX4LqRO3g5wu3/udmV9HvMRVGj5VRX8kxwmUnFKGHUykPW5l9BDdjb1WM5wXua2mM5+ObCXiKS45E+8wdvS+x3H1qhp4qiRsupruSZ4DKTilHCqJWHLDgX4zLm8H4FZS+cBmvaCoK9UCZGYq+1wlRQ/+DkEaxyfvbEs645+n/vg8MYJeb7vOSJ5hI/qPys7VY5T0w8qD80FGAednlh5/cDh1/TEtebbgPUEefAFxDWHmAw9vtvNLTdQuWWI7KjR6ooarSc6kqeCS4zqRgljFp5yJL8CkoG1c5LagZXOTKc/eoO/WBbvKr8YLAro8d8jZc80VziB9XTcjBf21UOmljvkW14hv1JIqSd2IGeevmnOgTWLngVzfOHpPcD6zfltk/sCm4UUb6rniqKGi2nupJngstMKkYJo1YesiS/gtKX0KqoZtByrLt8NNWRY5BOZBsugq6KLAfd2jolSp5oLvGDys/aPS0H5RaZ02EG+cldO7E/I1En0A2pNjU/KT1gXHeblW0r0j7zjuJK8hV6qihqtJzqSp4JLjOpGCWMWnnIMP+qGUBBy3FztEzoWGEUzfLlM7teOoN5BGsGLUevocmr0kT+lR60ZtHBrG/LsZOyTui6IYV5y0G5qprl6DU0fDcj/0qhHct14nZ7Wk7wUMtVFLV1azkV3iNp4j4UlGvATFhcHOhvH8joLjPXq4omJjt5xWuQqA01ZJjB7TW0Bn4F9egvnpMVCUaRCkU1bW9WMuLxE0tT07PSCbanu/N5O8kfTJOWM3jUXOIHFW85etEMTdxvDXtaju3N2o9snzhzbPrYR7KL7e7xKdePu31Auyq3HBySc6+eKopaoeVgSFQS8m9Biay7VutHW9l4SLdSEukKNjCRkiboHG+xOxXyGiQTTp85O7ZzV7ZTCRndZWZdsp9J26CYz8vxqhy1BkJmv6vHHO0sR4R3A9ivUmS3wq+g9KKcWkKwputNhaHRFtuuSf5gGracWqLmEj+o/KwdNAls53cFO/nk50nbiTgKmgg6xeEA7Lc4RVMlJlJg62dFoZuke06GeRVFrYfl4H9unTlGecvpy7FEaLVwsqvvGiJhS8plIxQpadKM5SwvXxh/8eVfj30V+ArI6C4z61LbLMcqleW0PGTrW8kTzSU+FaOiqPVtOdYP8SomaNzV98q7O92NgEHLsd6e70RK0ErGVcOXJnIMu976b+sK6AH2hjUjSmRctRx0Bf/IH7xIPzLI6A1YzsrK5Yk9eyUNhjp/2c/L+BSpn4UFTNNSsnzh02dfeBmFByeP2FZCfEMpFOHCjqDVXMP8UUF64QUfumEqqCyd47NwLbfbiipErZmQUUVKnmgu8akYFUWtvwtrKMTSQbcxQdtlUPkqx5Zgus93onW0K9kQD5D6cIUtP/4r9K/WIi/ZY0D5L1/+iXSLHoIHrwPZq5zDs5zV1dW9+95EGgx1/pLZXy+ViDA7WwvRKRvzu9TXSyL5lURMQ71IIlYhNiC7+YbuqFRSGZ2gpr1QI15lL7/kj80dhvZZpH6j1ljIqCIlTzSX+FSMiqIWtcrRaR0bFqmACRo1obzlZLW/9AbM7PpqvhMtR+Uf/vdWedW2Rf2s098/NtmWHnRXfSh48PZgpMKwLWf/W+9qGsQoaxZCRneZaYVpWsBMrbuKTOU6p2uF/LQe31BXKlJY1FALMZDKustQb7cV9Ru1xkJGFSl5ornEp2JUFLUoy9FtzNqY01UxlmNf7ctysCGDwjm0rQyBEntIem1NGqL8pQO/luYwj+DBN2k5Bz/4nTvReyprGUJGd5nppFexdK4vmuhlu2haj28YtJy8B9ijcoUHh3+7raivqDUcMiqo5InmEp+KUVHU+lvloNDNxUHLyV8i01ft1O9WLVpNW6mLYNC85UgJenN/7EBHkZpoEjx4VNOBhmc5R6Zm3Fkeo6xxCBndZaZVzM2vJZaj5hHf8HFzh+747jWTyzd0R2VfkuYN3G4rio9awyGjipQ80VziUzEqilp/3+WIMEeDoFvYOupS9lURfALAe4osRys7i9LDWzr3iWzIq+hBStzqB+VYdeUPXvoUT0KJLIkuXDyv/yMUCu7gVfGZML+w6E7xSGXtQ8joLjOtYm5+LZqytabbFcobih+gmn0JoGH+qFToE+XY1u9mio5tqJbTfMiGKvteNSAJh4ZvcCVPNJf4jSk4eeYVrKYTYH7DVhueiqJWaDlUT0VmwtLSqbGdu9wpHqmsixAyustMKkYxUVt/IbsaLGd4UXOJ35iGZznyOVu/+xiSiqJGy6mumExYXr6wc3zCnd/xynoJIaO7zKRi1DNq6zJk695yhho1l/iNaRDLUQUXN7SckVTPTFhZufzixKvu5O5LWUchZHSXmVSMyqM2EiGDf6BPuIhesRT09gp8uyZIHTQp+Q2TfkmGrtCJ3tYhFfSSpmAHtX9YQUeUwiYtZ9hRc4nfr8QSLlw8/8KbT6NDfEmMq/f2jxTAGwBK4CULJ7sotA1RAtsIVita5bi/oYBd9IPvs+uyoqKo0XKqq2cm2F8GVFPWUQgZ3WUmFaPyqLU/ZDAPfGul2/m/X6Ab9mdSMBIxhrwf6LoEfdov1dSxrK9Iz7Ac/VpO68ComrScYUfNJX6/giVgKtdvpuEcwdustE6wYf4nicFqRZYjG9LWrnL0C3IcUs9FVaSKokbLqa6emYDnO7mTuy9lHYWQ0V1mUjEqj1r7Q2ZXG0BmebvKEcQP7K19IusZ6kZ4yVWQV/M/jbIupTUxKPoU2TraEC8NqOSJ5hK/X1k70W03v+u8b+uUNMwOzliOqxZvOehQ1kZSQY9hcBVFjZZTXT0zQVhc/MSd3H0p6yWEjO4y86pVcA4tUs+otTxkwf+sGoy+2q/llP80quWWIww1ai7x+1WJc6BQFG854hbuptxgtXjLwa70+f7Mfls4oIqiNizLwf9cOre/bpH/mJQEjdS9BYOr9g7ziskEYXqm687veGVdhJDRXWZetarXcoQ2hwwzvvsyRg1Gr3rphryqP5MqsRyRNIn5aZRai7Mc1JHd5i+sgeFFzSV+v8JkiOlIL3w5y7F+oPYTbAh7sD9JDFbraTl6TU9HF/R4BldR1IZrOfGPgl7HliMcmvzIneKRytqHkNFdZl61qt1yhDaHDP9fgJkdPiS79u8XiA+hjthAz1WOCHVgXdi2tgEPA2jrLEekIz7/0mvLFz5t2HKEIUXNJX6/0skQHdq7AOwUD7cAMAZpaO87sOsY2dWfJAarlVuOegx27Q/hZbcWFUVtuJYj876+rfKfFF/VR0FrTX2+AHDviGCj4irDVDQGglQOdjgMxWeC0O9Dn6CscQgZ3WUmhM+YqCNTg8wd+ac+20v/uHIihTqnYBLRWUxLrPI9uHHLK9tXMRBufEKFXz33ErYxbs+xdA5FV/nbsazio9ZYyKhyJU80l/j9ChOUncfapnrvVYOKojZcy5F3WVcbsiH2rmtGV98uSmyEdJ2IavAtWQ9KHTwpAJVRwW63apUjuEfbRiprHEJGd5kJiXPkL+JjpoapyNQceYOT7GoP2Fble3DjWuUr21fzRwjPkI/M+JTdcyyUP/bL34gzoR+UBxUftcZCRpUreaK5xO9XdkJrp/ARv94jLIra0C0HHpN/FLSrbx3C1nHRUsuRDdSx6yGAym2zHGFl5fKeV/a5c71cWcsQMrrLTAjzr2ANwy5cZNuuGwSpqV8GuE4Ut9DJ9+DGLa9sX7VHqKYi2+Ir9jsDtBXyY+lukedZ9RW1ZkJGlSt5ornEp2JUFLWhWw42ZBT9ymsQy9HeBBTCcvJXz1poOcLy8gX7m7WstBIyustMlV53knk8aDlqMDrFBy3HeYNVvgcptOP2rKzqaTk9x0L58y+9Jv2UX1UT9Ru1ZkJGlSh5ornEp2JUFLWhW45+s4IvzUosR++gsC7i6suuMxIMZO+Lg2yHQ1KFTBBOnzmrT+bIiioho7vMhILPkLaXrWTK1klcS3RDeih6DrRVvoeSp0TnK9tX4y2naCxtJTXtN95BVYjasENGlSt5ornEp2JUFLWhW45si0/IKEELUen1Ma2GXcGuYNxlNLxkC7Vz1+EwVC0ThMXFT/DLtWy/EjK6y0xIL0NhmsaEPrdwHK1gBiiU3aIbnGQX8zjIz+P5Hty45ZXzr5ZYTs+xtBXK4a/av1O1qA01ZC2RffMjJU16evzgSp5oLvGpGBVFbViWMwxhwYQFzTBusehXlTNB6HRnm5m/7IROiSpHrbGQpdL6sxyhlqi5xG9eDXxNULuKojZKliMKLmhSaZBMECY/PJJtVUJGd5kZFC3HaZCoNROyVFqXliMMHjWX+M2LlkOtacBMGBAZ3WUmFaOEURtqyPDZovafN+lVTUE/uOBLNaHeJ0YXKXmiucSvJvt7QVztz98MJSXumdO2lRDfUApF+g2FVnMN80dVl4qiRsupruSZ4DKTilHCqA01ZPAGeAksAZ6ht1RU+HkT+sS6R7exIfYDx6LlREpmf3tDE2Z/ayGyjQ0saOzX3naVE98Q14TkVfGVDzoHdjx9W76hO6oaVRQ1Wk51Jc8El5lUjBJGbaghUyeQbXuJTO/CsKscQV5FNWxrK0F9SEsU6V87RAVaTqTcDU32OwIgyw44AdYfahiybS0nvqG7azfYUAsxUI0qihotp7qSZ4LLTCpGCaM21JD1tBy919y+Wv7zJltTRcupLL2KpXO9m+jjLSemYdBy8tZij8q9NIiKojbCllP0Djam5JngMjNSmEf6nSmCs0+NGnb/qoRRqxyyGMVbjpTg1Z4/b0KfsB8VqslAvLDWl36x+8f3fPln0Ca7b8Mk3A8KSyxHzSO+Ib7IQQ+7334OF9ZcQ3dU9qUBVRS1EbOcnm7fpJJngsvMGGGawJV691K5hmEJMrXpkdByBlRPy0EFKe/r500oBOouEjiU1PvE6CIlTzSX+NWEaV06VPPADAbw7UuR5WhNtyuUN5QJE9XsSwAN80dVl4qiNsKWk1zJM8FlZoz0I60rTyJrOY0pYdSqhYxKnmgu8akYFUVtUMtx99g5Z4aR2jqCeMa3/u/XXIm4CIwaJfBk2/CJiQdtk/xYKBfQVnrL3zXojlZKBlHyTHCZ2VP2plj7wRbgAzI8CffaogTSj8DyYTn/1xBsz/hwrZ+phXzPu155Q+sL0sR+MM8fVXDQakoYtQoho0TJE80lPhWjoqgNajnuHrug5cgG6sAGgiXwGzTU7fwNfMELa6gPp9HLlyhEZSnEkeQ7HETJM8FlZozsKsdeitHr+6ig9yyprOVIK6xO9KKNbNgmqGO9J9hz8MJa8KiCg6KTfpUwatVCRiVPNJf4VIyKojao5WDeF9Rm7LaznPIS9KOIhWghOhQFLUcdRQrVfqwPac18h4MoeSa4zIyRtRw7d+tEjwqY8a2ClqCFui6RbS23BHsOWk7wqIKDopN+lTBq1UJGJU80l/hUjIqiVsN3OfYeO53cpVy24QSY/ddGM1e9XIltaGU7l90BLSff4SBKngkuM2M0JMuR7fztts4VaDnVQkYlTzSX+FSMiqI2qOUE7/yT2RwzO5xALEF9AsqXoGHPG/jEcvTKmBqJdRS1n6Dl1HtHYPJMcJkZI2s5du7Wib6a5QT/YgJGUQUtR6+zaVfBo6LlXM1Knmgu8akYFUVtUMvJ32MnrvBFz5+/dODX+s0NSoCahAK3sIWwjXznWifvNCgXUBK0nHyHgyh5JrjMjJG1HJHM6Vl3X07i1SwHrWTbWQgoMjOtIz1YI8kfFS3nalbyRHOJT8WoKGprljM71914zx+5BnUJyx17w9iJM8e6x6dsiVvxjIrSZsLs0dm/2PiPLjmpnkoYNYasmhIn2jCnx3WsMsuZW+huf/zbrkGNyi9f8iWuSfu15cHru7Nf3ebbPLOzc3dsv98lJ1WuTVvu7HS62TvYOAxZBaUNmTDs6XFdqmR6XLOc06dPPfb8D10bqlz3PnnHseMLeBOTcPrUqZ/8x9MuP6ly3bnj4YVji9k72DgMWQWlDZnA6bGCSqbHNctZXV39uHN407/8qWtGFWnDXdf87sjBy5cv401MgkRtamr6uk23uxSlivT1DTcfmvwwYdQYsn6VPGQCp8d+VT49rlmOcPHixf0fvMZLljGSN3T89WeWTp3EW5cQidr/7H+HXw/ESCavsfE9J5dOZe9dIhiyeLUkZAKnx3j1nB4zyxHOnT87NfPh3T+/ZfMD17peKOjG+76x/fFvi4G3wW/A2XPnjhz5eOvdP9q4eZvLWAracOP379h+v3xYbsPkJTBkPdW2kAmcHnsqcnr8ynIEWULOH5vtHp1emJ+n8uoenZlb6KZd5udZi9rcQqfTdUdLQfLOzM7OtSpqDFm5WhgygdNjuSKnx9+zHEIIIWR40HIIIYQ0BC2HEEJIQ9ByCCGENAQthxBCSEPQcgghhDQELYcQQkhD0HIIIYQ0RFssZ+++N3899ttIvTjxataMEELI6EDLIYQQ0hC0HEIIIQ1ByyGEENIQtBxCCCENQcshhBDSELQcQgghDUHLIYQQ0hC0HEIIIQ1ByyGEENIQtBxCCCENkcxyJj884oykst5971DWKSGEkBaTcpUzO7cwtnOX849+dWRqJuuOEEJIu0l8YW1p6dTO8QnnIpGShvMLi1lHhBBCWk/673KWly/seWWfs5OeenHi1dNnzmZdEEIIGQVacfvAysrl1994y5lKicSiLl68mDUmhBAyIrTCcsC77x1y1hLU/rfeXV1dzdoQQggZHVpkOcL0TNcZjNOhyY+yqoQQQkaNdlmOsLj4SfCGgrGduzrd2awSIYSQEaR1liOcPnP2xYlXrd+ICS0tncpeJoQQMpq00XKElZXLr+59A34zsWfv8vKF7AVCCCEjS0stR1hdXd3/1rt7970p9pMVEUIIGWXaazmEEELWGbQcQgghDUHLIYQQ0hC0HEIIIQ1ByyGEENIQtBxCCCENQcshhBDSELQcQgghDUHLIYQQ0hC0HEIIIQ3hLefC8vLxE4ud2emF+XnKqjs7fez4wvnz5z/77LPszWoNErXF45/MdI66Y6Y6ne7CscV2Ro2Qq5CvLGd1dfXo3Mz468/88Omttz10w3Xb/4Cy2vLg9fc+ecd/7X50pju1culS9q6lRqLW7R4dG99z933/ftOtd/3JdX9PWW3acuedOx5+4ldj09Od9kSNkKuWzHKuXLnycefwtkf/1s2zVF6bH7j24EcHLrVg/pKoTU1N37LtX908S+W1cfO2dw4eakPUCLmaySynOzfzvUc2ubmVKtKN931D1jrJr9V0urP/8L1/dnMrVaQNN35f1jq8wkZIQtYs58Ly8vjrz7hZlSrXI2P3nlz6BG9iEiRqY+N77JRK9dS/PfLkyRMns3eQENI4a5Yzf2z27p/f4qZUqly3PXRDZ3Yab2IS5ucWtt79IzelUuW66da7ZjpHs3eQENI4a5bTPTq9+YFr3ZRKlWvDXdfMzc3iTUxCp9PduHmbm1Kpcn19w81zs3PZO0gIaZw1y1mYn3fzKRUjed/wJiZBRnfzKRWjtFEj5CqHllNdtJxRFC2HkITQcqqLljOKouUQkhBaTnXRckZRtBxCEkLLqS5aziiKlkNIQgaynB1P3SrNH3vxPt1wFfpSLZ00KVrOKIqWQ0hC+rCc2x/ZeHHlUzQTJt5+Nmg5e9//7eKpuW/u+GPbNkY1Wo4cw9ohfsFk920pcQcvR7j9Pze7kn6Ped1bzt/d/oOLF796QszBySOugtXuvfsXFk/85TdvdeVtEy2HkIT0bTniNK5c1B7L2XDXNYfn3r+yeuWeX2Y/bpXjkT7zB29LbvjBH55ZXur3sK8Sy3lhYp9sP/7Ub2TQhx97xlawouUQQnoykOW4Vc4TEw/KdL/W6xfAPOzyws7vBw6/piWuN90GqCPOsXL50k33/7m1BxgM1jG2E9stVG45Ijt6pK42y7lyZXXrPQ9pOQ5DXv3ahpsnD3/1IAaxpX/a8RNsSGXZRsM/u+E7p8+ce+PA2hkiraST5QufPvvCy1806rGEqlG0HEISUqflYL62qxw0sd4j2/AM+Af6gbQTO9BTL/9Uh8DaBa+ief6QpD6qYRRpa/vEruBGEeW76qmrxHIwnPMb2IndtqucEsu5tLLy1zdt1bZoYv1s2KLlEJKQ6t/lwAbchlSzloNyi8zpMIP85K6dqG1IoTqBbki1qflJ6QHj2mtoIttWpH3mHcWV5Cv01FViOViRyAYWIrATC5ZBkZaDyiLt3NUftmg5hCRkuKscW66qZjl6DQ3fzci/UmjHcp243Z6WEzzUcl09liPb4ihYoBTZAy2HENKToViOXjRDE/tdi6in5djerP3I9okzx6aPfSS72O4en3L9uNsHtKtyy8EhOffqqUEmr0530EeCNmw56gp2xWOlniTb2hBf89ByCCGgfsvBdn5XgBP0tBzZFkdBEwElIhyA/RbHXkOzEtvLGn/+ua2fFYVukna+GKNBJq9fj/12cXGgP7fTsOXAMPLX1vQ7GC2Ec4gDYff5l15bvvApLYcQIvRhOZTTgJYztnPX6TNns/3+acBy1qVoOYQkhJZTXQNajmj8xZeXly9kRX1Cy6kmWg4hCaHlVNfgliOa2LN3ZeVyVtoPtJxqouUQkhBaTnWVTF7qKDHau+/N1dXVrGU0tJxqouUQkhBaTnXVZTmi/W+9m7WMhpZTTbQcQhJCy6muGi1HdPCD32WN46DlVBMth5CEJLac/I3XQQWr5W/RtrdZN6B6LUd0ZGomax/B1WA5//Tlz0hd+SCi5RCSkHVrOfkHE9Su2i1HNL+wmHXRixG1HPuQgp6i5RCyzhhhy1EFFzcjajljO3ctLZ3KeimFllNNtBxCElKP5YglXLh4/oU3n0an+CU/njJg/0gBvAGgBF6ycLKLQtsQJbCNYLWiVY77GwrYRT94HE5dVjQMyxHtHJ+I+bFOXZZj//TAwwV/d8D9eQLxjOu/9V1XIi6CBwqgBI8VsA1/9sSztgmGyDcR4W/zCG8ceJ+WQ8h6ojbLEUvAVI5n1YgBwDnc89bgNFon2HDLj/8KlqBeEqxWZDmyIW3tKkealD9rp5qGZDmiiT17s46KqctyZH7XZ6OJgpYjG6gjDoGn1+RLYB7WSGTbdS6yq5xgE2yI/djns2nzwUXLISQhdVqOXbjItpvfdd63dUoa4vgEtRxXLd5y0KGsjaRC0WPZKmh4lhPz+LW6LAceI6jN2G1nOeUl6EcR29BCdCiylhNsYv92jnaOtrWIlkNIQtpoOeIWWBuphQSrxVsOdqXP92f228IBNSTLmZ7Jrh+WU5fliPTyl5qEsxysPL4YNrv8lS+xDa1s57KbtxzXhJZDyDpm6BfW4BMi6wdqP8GGajlaEqzW03L0mp6OLujxDK5hWM6hyY+yLnpRl+U8+ovnZE7XB0UHr2uJDahPQPkSNHR/1MB1LiViOXqpLdgEPuQOwFYYULQcQhJSp+UE7wKwUzzcAsAYpKG978CuY2R36dwnsiE1g9XKLUc9Brvu7+jUototp69nENRlOfAD6VCdQFwBQ+jfHYANKLIuyZdIQ1sIq8h3rnWKmgQPQArrEi2HkITUf2Gtnar3XjWoXsvp90lrdVlOT2G1gbUIto+fWJruztsSt+Jps2g5hCTkarEcLHrqPcIaLWfPK/v6fZ50Y5Yjyq9FgquTkRAth5CE1GM5V6fqspwXJ16t8FdzmrSc9SRaDiEJoeVU1yCTl/rNzvGJan8blJZTTbQcQhJCy6muwS1nbOeumJ/gBKHlVBMth5CE0HKqa3DL6XRns/3+oeVUEy2HkIS0xXLcLzdHQgNazuSHR7KdSrTBcuzvOkdFtBxCEkLLqa5BJq8D73z14NFq0HKqiZZDSELqtBz89gX94geY+qNOQX/m6Z45bVsJ8Q2lUKQ/L9VqrmH+qOpS2smrLsvBD2vQJ36eGXy8zfKFT5994WUUHpw8YlsJ8Q2lUPT4l8+K1mquYf6o6hIth5CE1Gk5MvvbZ8xg9rcWItvY0EfX6OMA7ConvqE+bkB85YPOgR1P35Zv6I6qRq0Py5HZXx8NIMLsby1EtrGBBY19Bppd5cQ3xG965FXxlfc+OCy7+YbuqGoULYeQhNRpOe4ZM7qryLIDToD1hxqGbFvLiW9Y9CA1ReprIQaqUevDcvRHnZj07W88gSw74ARYf6hhyLa1nPiG0sraSbChFmKgGkXLISQhNX+Xo1exdK53E3285cQ0DFpO3lrsUbmXBtH6sByRXsXSud5N9PGWE9MwaDl5a7FH5V4aRLQcQhJSp+X8YveP7/nyz6BNdt+GSeiXLlCJ5ah5xDfEFznoYffbz+HCmmvojsq+NKDWh+UEnyStX7pAJZaj5hHfEF/koIfx3Wsml2+YfwR1XaLlEJKQOi0H07p0qOYBbwD2rxXkLUdrul2hvKF4FarZlwAa5o+qLq0Pyyl52LOAb1+KLEdrul2hvKF4FarZlwAa5o+qLtFyCElIzRfWriqtD8u52kTLISQhtJzqouWMomg5hCSEllNdtJxRFC2HkITQcqqLljOKouUQkhBaTnXRckZRtBxCEkLLqS5aziiKlkNIQmg51UXLGUXRcghJyJrlzM51N97zR24+pXoq7eQ1e3T2Lzb+o5tPqZ6i5RCSkDXLmVvobn/8224+pcq15cHru7NfPUq5eWZn5+7Yfr+bT6lybdpyZ6fTzd5BQkjjrFnO6dOnHnv+h25Kpcp175N3HDu+gDcxCadPnfrJfzztplSqXHfueHjh2GL2DhJCGmfNclZXVz/uHN70L3/qZlWqSBvuuuZ3Rw5evnwZb2ISJGpTU9PXbbrdzapUkb6+4eZDkx+mjRohVzlrliNcvHhx/wev8RudGInfjL/+zNKpk3jrEiJR+5/97/AbnRiJ34yN7zm5dCp77wghKcgsRzh3/uzUzId3//yWzQ9c6yZZCrrxvm9sf/zbsr5pg9+As+fOHTny8da7f7Rx8zY3yVLQhhu/f8f2+2V9Q78hJDlfWY6wuro6f2y2e3R6YX6eyqt7dGZuodu2KzNrUZtb6HS67mgpSN6Z2dk5Xk8jpA38nuUQQgghw4OWQwghpCFoOYQQQhqClkMIIaQhaDmEEEIagpZDCCGkIWg5hBBCGoKWQwghpCFoOYQQQhqClkMIIaQhaDmEEEIagpZDCCGkIWg5hBBCGoKWQwghpCFoOYQQQhqClkMIIaQhaDmEEEIagpZDCCGkIWg5hBBCGuHzz/8/wJ3Ofrc18acAAAAASUVORK5CYII=",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "image/png"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 358
        },
        "outputId": "ef6f3aeb-785e-4144-fb0e-6b6d7202b6e2"
      },
      "source": [
        "%%html\n",
        "<img src='/nbextensions/google.colab/GEC.png' />"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<img src='/nbextensions/google.colab/GEC.png' />"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2u7iMiMEdNez",
        "colab_type": "text"
      },
      "source": [
        " **High level workflow**\n",
        " \n",
        "•\tTokenize the sentence using Spacy\n",
        "\n",
        "•\tCheck for spelling errors using Hunspell\n",
        "\n",
        "•\tFor all preposition, determiners & helper verbs, create a set of probable sentences\n",
        "\n",
        "•\tCreate a set of sentences with each word “masked”, deleted or an additional determiner, preposition or helper verb added\n",
        "\n",
        "•\tUsed BERT Masked Language Model to determine possible suggestions for masks\n",
        "\n",
        "•\tUse the GED model to select appropriate solutions\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d0be7sVJ8hFj",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 306
        },
        "outputId": "1444be73-79a0-4fba-b0c8-adde9af794a4"
      },
      "source": [
        "# install pytorch_pretrained_bert the previous version of Pytorch-Transformers\n",
        "!pip install -U pytorch_pretrained_bert"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already up-to-date: pytorch_pretrained_bert in /usr/local/lib/python3.6/dist-packages (0.6.2)\n",
            "Requirement already satisfied, skipping upgrade: boto3 in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (1.9.199)\n",
            "Requirement already satisfied, skipping upgrade: regex in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (2019.6.8)\n",
            "Requirement already satisfied, skipping upgrade: tqdm in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (4.28.1)\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (1.16.4)\n",
            "Requirement already satisfied, skipping upgrade: torch>=0.4.1 in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (1.1.0)\n",
            "Requirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from pytorch_pretrained_bert) (2.21.0)\n",
            "Requirement already satisfied, skipping upgrade: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch_pretrained_bert) (0.2.1)\n",
            "Requirement already satisfied, skipping upgrade: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch_pretrained_bert) (0.9.4)\n",
            "Requirement already satisfied, skipping upgrade: botocore<1.13.0,>=1.12.199 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch_pretrained_bert) (1.12.199)\n",
            "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (2019.6.16)\n",
            "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (2.8)\n",
            "Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch_pretrained_bert) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\" in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.199->boto3->pytorch_pretrained_bert) (2.5.3)\n",
            "Requirement already satisfied, skipping upgrade: docutils<0.15,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.13.0,>=1.12.199->boto3->pytorch_pretrained_bert) (0.14)\n",
            "Requirement already satisfied, skipping upgrade: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil<3.0.0,>=2.1; python_version >= \"2.7\"->botocore<1.13.0,>=1.12.199->boto3->pytorch_pretrained_bert) (1.12.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "41MNRTbh7qBm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch\n",
        "from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ne5HvQLD7rCU",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0c788c2d-2e8b-467f-b250-1c136dc91dc8"
      },
      "source": [
        "# Check to confirm that GPU is available\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "n_gpu = torch.cuda.device_count()\n",
        "torch.cuda.get_device_name(0)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Tesla K80'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "af63QP2z7s7s",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# OPTIONAL: if you want to have more information on what's happening, activate the logger as follows\n",
        "import logging\n",
        "logging.basicConfig(level=logging.INFO)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-72xZeds8KNQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "bfa6ded0-ab3a-4966-fdf1-5bad70bd5122"
      },
      "source": [
        "# Load pre-trained model tokenizer (vocabulary)\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I0804 13:09:47.153560 139915230685056 tokenization.py:190] loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased-vocab.txt from cache at /root/.pytorch_pretrained_bert/26bc1ad6c0ac742e9b52263248f6d0f00068293b33709fae12320c0e35ccfbbb.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YCaEFtjS7vSq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from keras.preprocessing.sequence import pad_sequences\n",
        "import numpy as np\n",
        "\n",
        "def check_GE(sents):\n",
        "    \"\"\"Check of the input sentences have grammatical errors\n",
        "\n",
        "    :param list: list of sentences\n",
        "    :return: error, probabilities\n",
        "    :rtype: (boolean, (float, float))\n",
        "    \"\"\"\n",
        "    \n",
        "  # Create sentence) and label lists\n",
        "  # We need to add special tokens at the beginning and end of each sentence\n",
        "  # for BERT to work properly\n",
        "  sentences = [\"[CLS] \" + sentence + \" [SEP]\" for sentence in sents]\n",
        "  labels =[0]\n",
        "\n",
        "  tokenized_texts = [tokenizer.tokenize(sent) for sent in sentences]\n",
        "\n",
        "  # Padding Sentences\n",
        "  # Set the maximum sequence length. The longest sequence in our training set\n",
        "  # is 47, but we'll leave room on the end anyway.\n",
        "  # In the original paper, the authors used a length of 512.\n",
        "  MAX_LEN = 128\n",
        "\n",
        "  predictions = []\n",
        "  true_labels = []\n",
        "\n",
        "  # Pad our input tokens\n",
        "  input_ids = pad_sequences(\n",
        "      [tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_texts], \n",
        "      maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\"\n",
        "      )\n",
        "\n",
        "  # Index Numbers and Padding\n",
        "  input_ids = [tokenizer.convert_tokens_to_ids(x) for x in tokenized_texts]\n",
        "\n",
        "  # pad sentences\n",
        "  input_ids = pad_sequences(input_ids, maxlen=MAX_LEN, \n",
        "                            dtype =\"long\", truncating=\"post\",padding =\"post\")\n",
        "\n",
        "  # Attention masks\n",
        "  # Create attention masks\n",
        "  attention_masks = []\n",
        "\n",
        "  # Create a mask of 1s for each token followed by 0s for padding\n",
        "  for seq in input_ids:\n",
        "    seq_mask = [float(i > 0) for i in seq]\n",
        "    attention_masks.append(seq_mask)\n",
        "\n",
        "  prediction_inputs = torch.tensor(input_ids)\n",
        "  prediction_masks = torch.tensor(attention_masks)\n",
        "  prediction_labels = torch.tensor(labels)\n",
        "\n",
        "  with torch.no_grad():\n",
        "    # Forward pass, calculate logit predictions\n",
        "    logits = modelGED(prediction_inputs, token_type_ids=None, \n",
        "                      attention_mask=prediction_masks)\n",
        "\n",
        "  # Move logits and labels to CPU\n",
        "  logits = logits.detach().cpu().numpy()\n",
        "  # label_ids = b_labels.to(\"cpu\").numpy()\n",
        "\n",
        "  # Store predictions and true labels\n",
        "  predictions.append(logits)\n",
        "  # true_labels.append(label_ids)\n",
        "\n",
        "#   print(predictions)\n",
        "  flat_predictions = [item for sublist in predictions for item in sublist]\n",
        "#   print(flat_predictions)\n",
        "  prob_vals = flat_predictions\n",
        "  flat_predictions = np.argmax(flat_predictions, axis=1).flatten()\n",
        "  # flat_true_labels = [item for sublist in true_labels for item in sublist]\n",
        "#   print(flat_predictions)\n",
        "  return flat_predictions, prob_vals"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eA4nvXBn7Kf_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 119
        },
        "outputId": "fe608925-f984-4222-96e8-d6122e41cdfb"
      },
      "source": [
        "# load previously trained BERT Grammar Error Detection model\n",
        "\n",
        "# from self google drive\n",
        "# from google.colab import drive\n",
        "# drive.mount('/content/drive')\n",
        "# !cp './drive/My Drive/Colab Notebooks/S89A/bert-based-uncased-GED.pth' .\n"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-08-04 13:11:07--  https://drive.google.com/drive/folders/1BoxpDeWZaNC8O3M3222pEAH-aW69sZ14/bert-based-uncased-GED.pth\n",
            "Resolving drive.google.com (drive.google.com)... 74.125.141.139, 74.125.141.101, 74.125.141.102, ...\n",
            "Connecting to drive.google.com (drive.google.com)|74.125.141.139|:443... connected.\n",
            "HTTP request sent, awaiting response... 400 Bad Request\n",
            "2019-08-04 13:11:07 ERROR 400: Bad Request.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4ZjdtFfmzC9i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#\n",
        "# CREDIT: https://stackoverflow.com/a/39225039\n",
        "#\n",
        "\n",
        "import requests\n",
        "\n",
        "def download_file_from_google_drive(id, destination):\n",
        "  print(\"Trying to fetch {}\".format(destination))\n",
        "\n",
        "  def get_confirm_token(response):\n",
        "    for key, value in response.cookies.items():\n",
        "      if key.startswith('download_warning'):\n",
        "        return value\n",
        "\n",
        "    return None\n",
        "\n",
        "  def save_response_content(response, destination):\n",
        "    CHUNK_SIZE = 32768\n",
        "\n",
        "    with open(destination, \"wb\") as f:\n",
        "      for chunk in progress_bar(response.iter_content(CHUNK_SIZE)):\n",
        "        if chunk: # filter out keep-alive new chunks\n",
        "          f.write(chunk)\n",
        "\n",
        "  URL = \"https://docs.google.com/uc?export=download\"\n",
        "\n",
        "  session = requests.Session()\n",
        "\n",
        "  response = session.get(URL, params = { 'id' : id }, stream = True)\n",
        "  token = get_confirm_token(response)\n",
        "\n",
        "  if token:\n",
        "    params = { 'id' : id, 'confirm' : token }\n",
        "    response = session.get(URL, params = params, stream = True)\n",
        "\n",
        "  save_response_content(response, destination)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YglAn18CzxJW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def progress_bar(some_iter):\n",
        "    try:\n",
        "        from tqdm import tqdm\n",
        "        return tqdm(some_iter)\n",
        "    except ModuleNotFoundError:\n",
        "        return some_iter"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N72u8k_FzGkB",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "4eeffeb8-7d5a-46f9-ae0a-9eb322c5a348"
      },
      "source": [
        "# load previously trained BERT Grammar Error Detection model\n",
        "\n",
        "# download from public google drive link\n",
        "download_file_from_google_drive(\"1al7v87aRxebSUCXrN2Sdd0jGUS0zZ3vn\", \"./bert-based-uncased-GED.pth\")"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Trying to fetch ./bert-based-uncased-GED.pth\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "13367it [00:02, 5337.72it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YdekFl7a7ftt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "886c531f-fb89-4834-86a3-5814967d1de4"
      },
      "source": [
        "# https://pytorch.org/tutorials/beginner/saving_loading_models.html\n",
        "\n",
        "from pytorch_pretrained_bert import BertForSequenceClassification\n",
        "\n",
        "modelGED = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", \n",
        "                                                      num_labels=2)\n",
        "\n",
        "# restore model\n",
        "modelGED.load_state_dict(torch.load('bert-based-uncased-GED.pth'))\n",
        "modelGED.eval()"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I0804 13:26:48.230247 139915230685056 modeling.py:580] loading archive file https://s3.amazonaws.com/models.huggingface.co/bert/bert-base-uncased.tar.gz from cache at /root/.pytorch_pretrained_bert/9c41111e2de84547a463fd39217199738d1e3deb72d4fec4399e6e241983c6f0.ae3cef932725ca7a30cdcb93fc6e09150a55e2a130ec7af63975a16c153ae2ba\n",
            "I0804 13:26:48.234201 139915230685056 modeling.py:588] extracting archive file /root/.pytorch_pretrained_bert/9c41111e2de84547a463fd39217199738d1e3deb72d4fec4399e6e241983c6f0.ae3cef932725ca7a30cdcb93fc6e09150a55e2a130ec7af63975a16c153ae2ba to temp dir /tmp/tmp8lw1qyri\n",
            "I0804 13:26:55.947669 139915230685056 modeling.py:598] Model config {\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "I0804 13:26:59.686025 139915230685056 modeling.py:648] Weights of BertForSequenceClassification not initialized from pretrained model: ['classifier.weight', 'classifier.bias']\n",
            "I0804 13:26:59.687417 139915230685056 modeling.py:651] Weights from pretrained model not used in BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForSequenceClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): BertLayerNorm()\n",
              "      (dropout): Dropout(p=0.1)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1)\n",
              "  (classifier): Linear(in_features=768, out_features=2, bias=True)\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ECUddA_Y9KA3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "e7955d41-fc21-43d5-e440-6d83e43d2128"
      },
      "source": [
        "# Load pre-trained model (weights) for Masked Language Model (MLM)\n",
        "model = BertForMaskedLM.from_pretrained('bert-large-uncased')\n",
        "model.eval()"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I0804 13:27:20.014006 139915230685056 modeling.py:580] loading archive file https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased.tar.gz from cache at /root/.pytorch_pretrained_bert/214d4777e8e3eb234563136cd3a49f6bc34131de836848454373fa43f10adc5e.abfbb80ee795a608acbf35c7bf2d2d58574df3887cdd94b355fc67e03fddba05\n",
            "I0804 13:27:20.015491 139915230685056 modeling.py:588] extracting archive file /root/.pytorch_pretrained_bert/214d4777e8e3eb234563136cd3a49f6bc34131de836848454373fa43f10adc5e.abfbb80ee795a608acbf35c7bf2d2d58574df3887cdd94b355fc67e03fddba05 to temp dir /tmp/tmpds94oeo6\n",
            "I0804 13:27:41.119252 139915230685056 modeling.py:598] Model config {\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 1024,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 4096,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"num_attention_heads\": 16,\n",
            "  \"num_hidden_layers\": 24,\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n",
            "I0804 13:27:53.759980 139915230685056 modeling.py:651] Weights from pretrained model not used in BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForMaskedLM(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(30522, 1024, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 1024)\n",
              "      (token_type_embeddings): Embedding(2, 1024)\n",
              "      (LayerNorm): BertLayerNorm()\n",
              "      (dropout): Dropout(p=0.1)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (12): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (13): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (14): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (15): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (16): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (17): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (18): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (19): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (20): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (21): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (22): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "        (23): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (key): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (value): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "              (LayerNorm): BertLayerNorm()\n",
              "              (dropout): Dropout(p=0.1)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=1024, out_features=4096, bias=True)\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=4096, out_features=1024, bias=True)\n",
              "            (LayerNorm): BertLayerNorm()\n",
              "            (dropout): Dropout(p=0.1)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "    (pooler): BertPooler(\n",
              "      (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "      (activation): Tanh()\n",
              "    )\n",
              "  )\n",
              "  (cls): BertOnlyMLMHead(\n",
              "    (predictions): BertLMPredictionHead(\n",
              "      (transform): BertPredictionHeadTransform(\n",
              "        (dense): Linear(in_features=1024, out_features=1024, bias=True)\n",
              "        (LayerNorm): BertLayerNorm()\n",
              "      )\n",
              "      (decoder): Linear(in_features=1024, out_features=30522, bias=False)\n",
              "    )\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jrrIyrNv-cEF",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        },
        "outputId": "470c5da5-6c96-4d16-939e-72d3e5b16bad"
      },
      "source": [
        "# Load pre-trained model tokenizer (vocabulary)\n",
        "tokenizerLarge = BertTokenizer.from_pretrained('bert-large-uncased')"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "I0804 13:27:59.746210 139915230685056 tokenization.py:190] loading vocabulary file https://s3.amazonaws.com/models.huggingface.co/bert/bert-large-uncased-vocab.txt from cache at /root/.pytorch_pretrained_bert/9b3c03a36e83b13d5ba95ac965c9f9074a99e14340c523ab405703179e79fc46.542ce4285a40d23a559526243235df47c5f75c197f04f37d1a0c124c32c9a084\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p20iABcF9nYe",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        },
        "outputId": "02e365e2-91b1-4273-a30e-c21883a25300"
      },
      "source": [
        "# install the packages for Hunspell\n",
        "\n",
        "!sudo apt-get install libhunspell-1.6-0 libhunspell-dev\n",
        "!pip install cyhunspell"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Reading package lists... Done\n",
            "Building dependency tree       \n",
            "Reading state information... Done\n",
            "libhunspell-1.6-0 is already the newest version (1.6.2-1).\n",
            "libhunspell-dev is already the newest version (1.6.2-1).\n",
            "The following package was automatically installed and is no longer required:\n",
            "  libnvidia-common-410\n",
            "Use 'sudo apt autoremove' to remove it.\n",
            "0 upgraded, 0 newly installed, 0 to remove and 4 not upgraded.\n",
            "Requirement already satisfied: cyhunspell in /usr/local/lib/python3.6/dist-packages (1.3.2)\n",
            "Requirement already satisfied: cacheman>=2.0.6 in /usr/local/lib/python3.6/dist-packages (from cyhunspell) (2.1.0)\n",
            "Requirement already satisfied: psutil>=2.1.0 in /usr/local/lib/python3.6/dist-packages (from cacheman>=2.0.6->cyhunspell) (5.4.8)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from cacheman>=2.0.6->cyhunspell) (1.12.0)\n",
            "Requirement already satisfied: future>=0.16.0 in /usr/local/lib/python3.6/dist-packages (from cacheman>=2.0.6->cyhunspell) (0.16.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rG1ow8ew9n_Y",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        },
        "outputId": "87d8ee05-ba36-4d0b-b91d-7e023530c339"
      },
      "source": [
        "from hunspell import Hunspell\n",
        "import os\n",
        "\n",
        "# download the gn_GB dictionary for hunspell\n",
        "download_file_from_google_drive(\"1jC5BVF9iZ0gmRQNmDcZnhfFdEYv8RNok\", \"./en_GB-large.dic\")\n",
        "download_file_from_google_drive(\"1g8PO8kdw-YmyOY_HxjnJ5FfdJFX4bsPv\", \"./en_GB-large.aff\")\n",
        "\n",
        "gb = Hunspell(\"en_GB-large\", hunspell_data_dir=\".\")"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Trying to fetch ./en_GB-large.dic\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "27it [00:00, 1980.52it/s]\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Trying to fetch ./en_GB-large.aff\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "1it [00:00, 1558.64it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s4WCEE35jJKQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# List of common determiners\n",
        "# det = [\"\", \"the\", \"a\", \"an\"]\n",
        "det = ['the', 'a', 'an', 'this', 'that', 'these', 'those', 'my', 'your', 'his', \n",
        "       'her', 'its', 'our', 'their', 'all', 'both', 'half', 'either', 'neither', \n",
        "       'each', 'every', 'other', 'another', 'such', 'what', 'rather', 'quite']\n",
        "\n",
        "# List of common prepositions\n",
        "prep = [\"about\", \"at\", \"by\", \"for\", \"from\", \"in\", \"of\", \"on\", \"to\", \"with\", \n",
        "        \"into\", \"during\", \"including\", \"until\", \"against\", \"among\", \n",
        "        \"throughout\", \"despite\", \"towards\", \"upon\", \"concerning\"]\n",
        "\n",
        "# List of helping verbs\n",
        "helping_verbs = ['am', 'is', 'are', 'was', 'were', 'being', 'been', 'be', \n",
        "                 'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', \n",
        "                 'shall', 'should', 'may', 'might', 'must', 'can', 'could']"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SrGKWPZYNoOX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# test sentences\n",
        "\n",
        "org_text = []\n",
        "org_text.append(\"They drank the pub .\")\n",
        "org_text.append(\"I am looking forway to see you soon .\")\n",
        "org_text.append(\"The cat sat at mat .\")\n",
        "org_text.append(\"Giant otters is an apex predator .\")\n",
        "org_text.append('There is no a doubt, tracking system has brought many benefits in this information age .')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QojckfCt9OAd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import spacy\n",
        "import numpy as np\n",
        "\n",
        "def create_spelling_set(org_text):\n",
        "  \"\"\" Create a set of sentences which have possible corrected spellings\n",
        "  \"\"\"\n",
        "  \n",
        "  sent = org_text\n",
        "  sent = sent.lower()\n",
        "  sent = sent.strip().split()\n",
        "\n",
        "\n",
        "  nlp = spacy.load(\"en\")\n",
        "  proc_sent = nlp.tokenizer.tokens_from_list(sent)\n",
        "  nlp.tagger(proc_sent)\n",
        "\n",
        "  sentences = []\n",
        "\n",
        "  for tok in proc_sent:\n",
        "    # check for spelling for alphanumeric\n",
        "    if tok.text.isalpha() and not gb.spell(tok.text):\n",
        "      new_sent = sent[:]\n",
        "      # append new sentences with possible corrections\n",
        "      for sugg in gb.suggest(tok.text):\n",
        "        new_sent[tok.i] = sugg\n",
        "        sentences.append(\" \".join(new_sent))\n",
        "\n",
        "  spelling_sentences = sentences\n",
        "\n",
        "  # retain new sentences which have a \n",
        "  # minimum chance of correctness using BERT GED\n",
        "  new_sentences = []\n",
        "  \n",
        "  for sent in spelling_sentences:\n",
        "    no_error, prob_val = check_GE([sent])\n",
        "    exps = [np.exp(i) for i in prob_val[0]]\n",
        "    sum_of_exps = sum(exps)\n",
        "    softmax = [j/sum_of_exps for j in exps]\n",
        "    if(softmax[1] > 0.6):\n",
        "      new_sentences.append(sent)\n",
        "  \n",
        "  \n",
        "  # if no corrections, append the original sentence\n",
        "  if len(spelling_sentences) == 0:\n",
        "    spelling_sentences.append(\" \".join(sent))\n",
        "\n",
        "  # eliminate dupllicates\n",
        "  [spelling_sentences.append(sent) for sent in new_sentences]\n",
        "  spelling_sentences = list(dict.fromkeys(spelling_sentences))\n",
        "\n",
        "  return spelling_sentences"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C7OdVrA9O7O0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_grammar_set(spelling_sentences):\n",
        "  \"\"\" create a new set of sentences with deleted determiners, \n",
        "      prepositions & helping verbs\n",
        "      \n",
        "  \"\"\"\n",
        "  \n",
        "  new_sentences = []\n",
        "\n",
        "  for text in spelling_sentences:\n",
        "    sent = text.strip().split()\n",
        "    for i in range(len(sent)):\n",
        "      new_sent = sent[:]\n",
        "      \n",
        "      if new_sent[i] not in list(set(det + prep + helping_verbs)):\n",
        "        continue\n",
        "      \n",
        "      del new_sent[i]\n",
        "      text = \" \".join(new_sent)\n",
        "      \n",
        "      # retain new sentences which have a \n",
        "      # minimum chance of correctness using BERT GED\n",
        "      no_error, prob_val = check_GE([text])\n",
        "      exps = [np.exp(i) for i in prob_val[0]]\n",
        "      sum_of_exps = sum(exps)\n",
        "      softmax = [j/sum_of_exps for j in exps]\n",
        "      if(softmax[1] > 0.6):\n",
        "        new_sentences.append(text)\n",
        "  \n",
        "  # eliminate dupllicates\n",
        "  [spelling_sentences.append(sent) for sent in new_sentences]\n",
        "  spelling_sentences = list(dict.fromkeys(spelling_sentences))\n",
        "  return spelling_sentences"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j34k7n2p9Y5q",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def create_mask_set(spelling_sentences):\n",
        "  \"\"\"For each input sentence create 2 sentences\n",
        "     (1) [MASK] each word\n",
        "     (2) [MASK] for each space between words\n",
        "  \"\"\"\n",
        "  sentences = []\n",
        "\n",
        "  for sent in spelling_sentences:\n",
        "    sent = sent.strip().split()\n",
        "    for i in range(len(sent)):\n",
        "      # (1) [MASK] each word\n",
        "      new_sent = sent[:]\n",
        "      new_sent[i] = '[MASK]'\n",
        "      text = \" \".join(new_sent)\n",
        "      new_sent = '[CLS] ' + text + ' [SEP]'\n",
        "      sentences.append(new_sent)\n",
        "\n",
        "      # (2) [MASK] for each space between words\n",
        "      new_sent = sent[:]\n",
        "      new_sent.insert(i, '[MASK]')\n",
        "      text = \" \".join(new_sent)\n",
        "      new_sent = '[CLS] ' + text + ' [SEP]'\n",
        "      sentences.append(new_sent)\n",
        "\n",
        "  return sentences"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mMC3vhj49ZjD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import math\n",
        "from difflib import SequenceMatcher\n",
        "\n",
        "def check_grammar(org_sent, sentences, spelling_sentences):\n",
        "  \"\"\" check grammar for the input sentences\n",
        "  \"\"\"\n",
        "  \n",
        "  n = len(sentences)\n",
        "  \n",
        "  # what is the tokenized value of [MASK]. Usually 103\n",
        "  text = '[MASK]'\n",
        "  tokenized_text = tokenizerLarge.tokenize(text)\n",
        "  mask_token = tokenizerLarge.convert_tokens_to_ids(tokenized_text)[0]\n",
        "\n",
        "  LM_sentences = []\n",
        "  new_sentences = []\n",
        "  i = 0 # current sentence number\n",
        "  l = len(org_sent.strip().split())*2 # l is no of sentencees\n",
        "  mask = False # flag indicating if we are processing space MASK\n",
        "\n",
        "  for sent in sentences:\n",
        "    i += 1\n",
        "    \n",
        "    print(\".\", end=\"\")\n",
        "    if i%50 == 0:\n",
        "      print(\"\")\n",
        "    \n",
        "    # tokenize the text\n",
        "    tokenized_text = tokenizerLarge.tokenize(sent)\n",
        "    indexed_tokens = tokenizerLarge.convert_tokens_to_ids(tokenized_text)\n",
        "\n",
        "    # Create the segments tensors.\n",
        "    segments_ids = [0] * len(tokenized_text)\n",
        "\n",
        "    # Convert inputs to PyTorch tensors\n",
        "    tokens_tensor = torch.tensor([indexed_tokens])\n",
        "    segments_tensors = torch.tensor([segments_ids])\n",
        "\n",
        "    # Predict all tokens\n",
        "    with torch.no_grad():\n",
        "        predictions = model(tokens_tensor, segments_tensors)\n",
        "\n",
        "    # index of the masked token\n",
        "    mask_index = (tokens_tensor == mask_token).nonzero()[0][1].item()\n",
        "    # predicted token\n",
        "    predicted_index = torch.argmax(predictions[0, mask_index]).item()\n",
        "    predicted_token = tokenizerLarge.convert_ids_to_tokens([predicted_index])[0]\n",
        "    \n",
        "    # second best prediction. Can you used to create more options\n",
        "#     second_index = torch.topk(predictions[0, mask_index], 2).indices[1].item()\n",
        "#     second_prediction = tokenizer.convert_ids_to_tokens([second_index])[0]\n",
        "\n",
        "    text = sent.strip().split()\n",
        "    mask_index = text.index('[MASK]')\n",
        "\n",
        "    if not mask:\n",
        "      # case of MASKed words\n",
        "      \n",
        "      mask = True\n",
        "      text[mask_index] = predicted_token\n",
        "      try:\n",
        "        # retrieve original word\n",
        "        org_word = spelling_sentences[i//l].strip().split()[mask_index-1]\n",
        "#         print(\">>> \" + org_word)\n",
        "      except:\n",
        "#         print(spelling_sentences[i%l - 1])\n",
        "#         print(tokenized_text)\n",
        "#         print(\"{0} {1} {2}\".format(i, l, mask_index))\n",
        "        print(\"!\", end=\"\")\n",
        "        continue\n",
        "  #     print(\"{0} - {1}\".format(org_word, predicted_token))\n",
        "      # check if the prediction is an inflection of the original word\n",
        "  #   if org_word.isalpha() and predicted_token not in gb_infl[org_word]:\n",
        "  #     continue\n",
        "      # use SequenceMatcher to see if predicted word is similar to original word\n",
        "      if SequenceMatcher(None, org_word, predicted_token).ratio() < 0.6:\n",
        "        if org_word not in list(set(det + prep + helping_verbs)) or predicted_token not in list(set(det + prep + helping_verbs)):\n",
        "          continue\n",
        "      if org_word == predicted_token:\n",
        "        continue\n",
        "    else:\n",
        "      # case for MASKed spaces\n",
        "      \n",
        "      mask = False\n",
        "  #     print(\"{0}\".format(predicted_token))\n",
        "      # only allow determiners / prepositions  / helping verbs in spaces\n",
        "      if predicted_token in list(set(det + prep + helping_verbs)) :\n",
        "        text[mask_index] = predicted_token\n",
        "      else:\n",
        "        continue\n",
        "\n",
        "  #   if org_word == \"in\":\n",
        "  #     print(\">>>>>> \" + predicted_token)\n",
        "  #   print(tokenized_text)\n",
        "  #   print(mask_index)\n",
        "  \n",
        "    text.remove('[SEP]')\n",
        "    text.remove('[CLS]')\n",
        "    new_sent = \" \".join(text)\n",
        "    \n",
        "  #   print(new_sent)\n",
        "    # retain new sentences which have a \n",
        "    # minimum chance of correctness using BERT GED\n",
        "    no_error, prob_val = check_GE([new_sent])\n",
        "    exps = [np.exp(i) for i in prob_val[0]]\n",
        "    sum_of_exps = sum(exps)\n",
        "    softmax = [j/sum_of_exps for j in exps]\n",
        "    if no_error and softmax[1] > 0.996:\n",
        "  #     print(org_word)\n",
        "  #     print(predicted_token)\n",
        "  #     print(SequenceMatcher(None, org_word, predicted_token).ratio())\n",
        "  #     print(\"{0} - {1}, {2}\".format(prob_val[0][1], prob_val[0][0], prob_val[0][1] - prob_val[0][0]))\n",
        "\n",
        "  #     print(\"{0} - {1:.2f}\".format(new_sent, softmax[1]*100) )\n",
        "      print(\"*\", end=\"\")\n",
        "      new_sentences.append(new_sent)\n",
        "  #   print(\"{0}\\t{1}\".format(predicted_token, second_prediction))\n",
        "\n",
        "  print(\"\")\n",
        "  \n",
        "  # remove duplicate suggestions\n",
        "  spelling_sentences = []\n",
        "  [spelling_sentences.append(sent) for sent in new_sentences]\n",
        "  spelling_sentences = list(dict.fromkeys(spelling_sentences))\n",
        "  spelling_sentences\n",
        "  \n",
        "  return spelling_sentences"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h5OizN4gFigm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 799
        },
        "outputId": "9b696903-4c7d-4760-d71e-a395cfc19ffd"
      },
      "source": [
        "# org_text = []\n",
        "# with open(\"./drive/My Drive/Colab Notebooks/S89A/CoNLL_2013_DS.txt\") as file:\n",
        "#   org_text = file.readlines()\n",
        "\n",
        "# predict for each of the test samples\n",
        "\n",
        "for sent in org_text:\n",
        "  \n",
        "  print(\"Input Sentence >>> \" + sent)\n",
        "  \n",
        "  sentences = create_spelling_set(sent)\n",
        "  spelling_sentences = create_grammar_set(sentences)\n",
        "  sentences = create_mask_set(spelling_sentences)\n",
        "  \n",
        "  print(\"processing {0} possibilities\".format(len(sentences)))\n",
        "  \n",
        "  sentences = check_grammar(sent, sentences, spelling_sentences)\n",
        "\n",
        "  print(\"Suggestions & Probabilities\")\n",
        "  \n",
        "  if len(sentences) == 0:\n",
        "    print(\"None\")\n",
        "    continue\n",
        "\n",
        "  no_error, prob_val =  check_GE(sentences)\n",
        "\n",
        "  for i in range(len(prob_val)):\n",
        "    exps = [np.exp(i) for i in prob_val[i]]\n",
        "    sum_of_exps = sum(exps)\n",
        "    softmax = [j/sum_of_exps for j in exps]\n",
        "    print(\"{0} - {1:0.4f}%\".format(sentences[i], softmax[1]*100))\n",
        "  \n",
        "  print(\"-\"*60)\n",
        "  print()"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input Sentence >>> They drank the pub .\n",
            "processing 10 possibilities\n",
            "......*....\n",
            "Suggestions & Probabilities\n",
            "they drank at the pub . - 99.8071%\n",
            "------------------------------------------------------------\n",
            "\n",
            "Input Sentence >>> I am looking forway to see you soon .\n",
            "processing 126 possibilities\n",
            ".......*..................*..................*.*......\n",
            "..............*.........!........*...................\n",
            "...*......!...*.*.*............\n",
            "Suggestions & Probabilities\n",
            "i am looking forward to see you soon . - 99.7493%\n",
            "i am looking to Norway to see you soon . - 99.7757%\n",
            "i am looking for a way to see you soon . - 99.7564%\n",
            "i am looking forward to seeing you soon . - 99.7722%\n",
            "am i looking forward to see you soon . - 99.6467%\n",
            "i look forward to see you soon . - 99.7621%\n",
            "------------------------------------------------------------\n",
            "\n",
            "Input Sentence >>> The cat sat at mat .\n",
            "processing 12 possibilities\n",
            "..........*..\n",
            "Suggestions & Probabilities\n",
            "the cat sat at the mat . - 99.8284%\n",
            "------------------------------------------------------------\n",
            "\n",
            "Input Sentence >>> Giant otters is an apex predator .\n",
            "processing 26 possibilities\n",
            ".....*...............*......\n",
            "Suggestions & Probabilities\n",
            "giant otters are an apex predator . - 99.8082%\n",
            "------------------------------------------------------------\n",
            "\n",
            "Input Sentence >>> There is no a doubt, tracking system has brought many benefits in this information age .\n",
            "processing 152 possibilities\n",
            "..................................................\n",
            "......................*..*..........................\n",
            "..................................................\n",
            "..\n",
            "Suggestions & Probabilities\n",
            "there is no doubt, the tracking system has brought many benefits in this information age . - 99.6639%\n",
            "there is no doubt, tracking the system has brought many benefits in this information age . - 99.6644%\n",
            "------------------------------------------------------------\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}