Flair Tutorials 1-5.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Flair Tutorials 1-5.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "fnsgc6IAbQLw",
        "colab_type": "code",
        "outputId": "0374fe65-d7ad-42e8-d2be-47ca5e712fbf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "!pip install flair"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: flair in /usr/local/lib/python3.6/dist-packages (0.4.4)\n",
            "Requirement already satisfied: matplotlib>=2.2.3 in /usr/local/lib/python3.6/dist-packages (from flair) (3.1.2)\n",
            "Requirement already satisfied: tabulate in /usr/local/lib/python3.6/dist-packages (from flair) (0.8.6)\n",
            "Requirement already satisfied: gensim>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from flair) (3.6.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from flair) (2019.12.9)\n",
            "Requirement already satisfied: segtok>=1.5.7 in /usr/local/lib/python3.6/dist-packages (from flair) (1.5.7)\n",
            "Requirement already satisfied: urllib3<1.25,>=1.20 in /usr/local/lib/python3.6/dist-packages (from flair) (1.24.3)\n",
            "Requirement already satisfied: tqdm>=4.26.0 in /usr/local/lib/python3.6/dist-packages (from flair) (4.28.1)\n",
            "Requirement already satisfied: pytest>=3.6.4 in /usr/local/lib/python3.6/dist-packages (from flair) (3.6.4)\n",
            "Requirement already satisfied: torch>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from flair) (1.3.1)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (from flair) (0.4.2)\n",
            "Requirement already satisfied: ipython-genutils==0.2.0 in /usr/local/lib/python3.6/dist-packages (from flair) (0.2.0)\n",
            "Requirement already satisfied: sklearn in /usr/local/lib/python3.6/dist-packages (from flair) (0.0)\n",
            "Requirement already satisfied: transformers>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from flair) (2.2.2)\n",
            "Requirement already satisfied: bpemb>=0.2.9 in /usr/local/lib/python3.6/dist-packages (from flair) (0.3.0)\n",
            "Requirement already satisfied: sqlitedict>=1.6.0 in /usr/local/lib/python3.6/dist-packages (from flair) (1.6.0)\n",
            "Requirement already satisfied: hyperopt>=0.1.1 in /usr/local/lib/python3.6/dist-packages (from flair) (0.1.2)\n",
            "Requirement already satisfied: ipython==7.6.1 in /usr/local/lib/python3.6/dist-packages (from flair) (7.6.1)\n",
            "Requirement already satisfied: pymongo in /usr/local/lib/python3.6/dist-packages (from flair) (3.10.0)\n",
            "Requirement already satisfied: mpld3==0.3 in /usr/local/lib/python3.6/dist-packages (from flair) (0.3)\n",
            "Requirement already satisfied: langdetect in /usr/local/lib/python3.6/dist-packages (from flair) (1.0.7)\n",
            "Requirement already satisfied: deprecated>=1.2.4 in /usr/local/lib/python3.6/dist-packages (from flair) (1.2.7)\n",
            "Requirement already satisfied: tiny-tokenizer[all] in /usr/local/lib/python3.6/dist-packages (from flair) (3.0.1)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (2.6.1)\n",
            "Requirement already satisfied: numpy>=1.11 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (1.17.4)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (2.4.5)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (1.1.0)\n",
            "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.4.0->flair) (1.3.3)\n",
            "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.4.0->flair) (1.9.0)\n",
            "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from gensim>=3.4.0->flair) (1.12.0)\n",
            "Requirement already satisfied: attrs>=17.4.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (19.3.0)\n",
            "Requirement already satisfied: atomicwrites>=1.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (1.3.0)\n",
            "Requirement already satisfied: py>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (1.8.0)\n",
            "Requirement already satisfied: more-itertools>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (8.0.2)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (42.0.2)\n",
            "Requirement already satisfied: pluggy<0.8,>=0.5 in /usr/local/lib/python3.6/dist-packages (from pytest>=3.6.4->flair) (0.7.1)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision->flair) (4.3.0)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sklearn->flair) (0.21.3)\n",
            "Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->flair) (1.10.40)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->flair) (0.0.35)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->flair) (0.1.85)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers>=2.0.0->flair) (2.21.0)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->flair) (0.16.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->flair) (2.4)\n",
            "Requirement already satisfied: prompt-toolkit<2.1.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (2.0.10)\n",
            "Requirement already satisfied: jedi>=0.10 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (0.15.1)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (4.4.1)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (4.3.3)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (2.1.3)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (4.7.0)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (0.7.5)\n",
            "Requirement already satisfied: backcall in /usr/local/lib/python3.6/dist-packages (from ipython==7.6.1->flair) (0.1.0)\n",
            "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.6/dist-packages (from deprecated>=1.2.4->flair) (1.11.2)\n",
            "Requirement already satisfied: kytea; extra == \"all\" in /usr/local/lib/python3.6/dist-packages (from tiny-tokenizer[all]->flair) (0.1.4)\n",
            "Requirement already satisfied: natto-py; extra == \"all\" in /usr/local/lib/python3.6/dist-packages (from tiny-tokenizer[all]->flair) (0.9.0)\n",
            "Requirement already satisfied: SudachiPy; extra == \"all\" in /usr/local/lib/python3.6/dist-packages (from tiny-tokenizer[all]->flair) (0.4.2)\n",
            "Requirement already satisfied: boto>=2.32 in /usr/local/lib/python3.6/dist-packages (from smart-open>=1.2.1->gensim>=3.4.0->flair) (2.49.0)\n",
            "Requirement already satisfied: olefile in /usr/local/lib/python3.6/dist-packages (from pillow>=4.1.1->torchvision->flair) (0.46)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn->flair) (0.14.1)\n",
            "Requirement already satisfied: botocore<1.14.0,>=1.13.40 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->flair) (1.13.40)\n",
            "Requirement already satisfied: s3transfer<0.3.0,>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->flair) (0.2.1)\n",
            "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers>=2.0.0->flair) (0.9.4)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers>=2.0.0->flair) (7.0)\n",
            "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.0.0->flair) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.0.0->flair) (2019.11.28)\n",
            "Requirement already satisfied: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers>=2.0.0->flair) (2.8)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.1.0,>=2.0.0->ipython==7.6.1->flair) (0.1.7)\n",
            "Requirement already satisfied: parso>=0.5.0 in /usr/local/lib/python3.6/dist-packages (from jedi>=0.10->ipython==7.6.1->flair) (0.5.2)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.6/dist-packages (from pexpect; sys_platform != \"win32\"->ipython==7.6.1->flair) (0.6.0)\n",
            "Requirement already satisfied: cffi in /usr/local/lib/python3.6/dist-packages (from natto-py; extra == \"all\"->tiny-tokenizer[all]->flair) (1.13.2)\n",
            "Requirement already satisfied: sortedcontainers~=2.1.0 in /usr/local/lib/python3.6/dist-packages (from SudachiPy; extra == \"all\"->tiny-tokenizer[all]->flair) (2.1.0)\n",
            "Requirement already satisfied: dartsclone~=0.6.0 in /usr/local/lib/python3.6/dist-packages (from SudachiPy; extra == \"all\"->tiny-tokenizer[all]->flair) (0.6)\n",
            "Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.14.0,>=1.13.40->boto3->transformers>=2.0.0->flair) (0.15.2)\n",
            "Requirement already satisfied: pycparser in /usr/local/lib/python3.6/dist-packages (from cffi->natto-py; extra == \"all\"->tiny-tokenizer[all]->flair) (2.19)\n",
            "Requirement already satisfied: Cython in /usr/local/lib/python3.6/dist-packages (from dartsclone~=0.6.0->SudachiPy; extra == \"all\"->tiny-tokenizer[all]->flair) (0.29.14)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IYVcrEN2bhv_",
        "colab_type": "code",
        "outputId": "d01f24ae-d45f-4810-8563-ba9ab7ab8fb4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 138
        }
      },
      "source": [
        "from flair.data import Sentence\n",
        "from flair.models import SequenceTagger\n",
        "\n",
        "# make a sentence\n",
        "sentence = Sentence('I love Berlin .')\n",
        "\n",
        "# load the NER tagger\n",
        "tagger = SequenceTagger.load('ner')\n",
        "\n",
        "# run NER over sentence\n",
        "tagger.predict(sentence)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 08:41:38,000 loading file /root/.flair/models/en-ner-conll03-v0.4.pt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Sentence: \"I love Berlin .\" - 4 Tokens]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cshuA3V2b2HY",
        "colab_type": "code",
        "outputId": "8e14ef63-2bf9-45bc-ef63-cd2623c6af48",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        }
      },
      "source": [
        "print(sentence)\n",
        "print('The following NER tags are found: ')\n",
        "\n",
        "# iterate over entities and print\n",
        "for entity in sentence.get_spans('ner'):\n",
        "  print(entity)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sentence: \"I love Berlin .\" - 4 Tokens\n",
            "The following NER tags are found: \n",
            "LOC-span [3]: \"Berlin\"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wRMN33flcUW9",
        "colab_type": "text"
      },
      "source": [
        "#Tutorial 1: NLP Base Types"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oQVM-uqrceFF",
        "colab_type": "text"
      },
      "source": [
        "## A. Creating a Sentence"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "14iYVpEtcJ-_",
        "colab_type": "code",
        "outputId": "6e520b01-f087-40a5-d547-bd53aca72cf4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# The sentence object holds a sentence that we may want to embed or tag\n",
        "from flair.data import Sentence\n",
        "\n",
        "# Make a sentence object by passing a whitespace tokenized string\n",
        "sentence = Sentence('The grass is green .')\n",
        "\n",
        "# Print the object to see what's in there\n",
        "print(sentence)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sentence: \"The grass is green .\" - 5 Tokens\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CPd6Q49ncyQj",
        "colab_type": "code",
        "outputId": "82851e73-0989-4454-a7c1-aeac0e9f8a94",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "# using the token id\n",
        "print(sentence.get_token(4))\n",
        "\n",
        "# using the index itself\n",
        "print(sentence[3])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Token: 4 green\n",
            "Token: 4 green\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mNE0l-Djc9ec",
        "colab_type": "code",
        "outputId": "ff6ca645-5972-4ab2-84df-b0ecdc03723b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 108
        }
      },
      "source": [
        "for token in sentence:\n",
        "  print(token)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Token: 1 The\n",
            "Token: 2 grass\n",
            "Token: 3 is\n",
            "Token: 4 green\n",
            "Token: 5 .\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MQKcm6_idKxa",
        "colab_type": "text"
      },
      "source": [
        "## B. Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YtnhL8d_dDwv",
        "colab_type": "code",
        "outputId": "365c24c6-830e-4717-997f-0065fd67b453",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "from flair.data import Sentence\n",
        "\n",
        "# Make a sentence object by passing an untokenized string\n",
        "# and the 'use_tokenizer' flag\n",
        "sentence = Sentence('The grass is green.', use_tokenizer = True)\n",
        "\n",
        "# Print the object to see what's in there\n",
        "print(sentence)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sentence: \"The grass is green .\" - 5 Tokens\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MDiCa0V6diCu",
        "colab_type": "text"
      },
      "source": [
        "### B1. Adding Custom Tokenizers"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LK2UFquzdb-U",
        "colab_type": "code",
        "outputId": "790fb9b4-10c6-454f-eda6-e5292100a033",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "from flair.data import Sentence, segtok_tokenizer\n",
        "\n",
        "# Make a sentence object by passing an untokenized string\n",
        "# and a tokenizer\n",
        "sentence = Sentence('The grass is green.', use_tokenizer = segtok_tokenizer)\n",
        "\n",
        "# Print the object to see what's in there\n",
        "print(sentence)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sentence: \"The grass is green .\" - 5 Tokens\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6VERbfdZeG-i",
        "colab_type": "text"
      },
      "source": [
        "## C. Adding Tags to Tokens"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "op7f_9Mad68u",
        "colab_type": "code",
        "outputId": "11c3e955-654a-4129-cde9-aa82b09907db",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# add a tag to a word in the sentence\n",
        "sentence[3].add_tag('ner', 'color')\n",
        "\n",
        "# print the sentence with all tags of this type\n",
        "print(sentence.to_tagged_string())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "The grass is green <color> .\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m_Xmuve-ebQM",
        "colab_type": "code",
        "outputId": "cb5588ea-d6de-495d-d3c0-0b505adda28b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "# get token 3 in the sentence\n",
        "token = sentence[3]\n",
        "\n",
        "# get the 'ner' tag of the token\n",
        "tag = token.get_tag('ner')\n",
        "\n",
        "# print token\n",
        "print(f'\"{token}\" is tagged as \"{tag.value}\" with confidence score \"{tag.score}\"')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\"Token: 4 green\" is tagged as \"color\" with confidence score \"1.0\"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9p8cpQVofAtI",
        "colab_type": "text"
      },
      "source": [
        "## D. Adding Labels to Sentences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u7veDIb6eyAK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "sentence = Sentence('France is the current world cup winner.')\n",
        "\n",
        "# add a label to a sentence\n",
        "sentence.add_label('sports')\n",
        "\n",
        "# a sentence can also belong to multiple classes\n",
        "sentence.add_labels(['sports', 'world cup'])\n",
        "\n",
        "# you can also set the labels while initializing the sentence\n",
        "sentence = Sentence('France is the current world cup winner.',\n",
        "                    labels = ['sports', 'world cup'])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Wj6s6D3fcn9",
        "colab_type": "code",
        "outputId": "18dc3f26-5999-456e-94c5-324733c17aa6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 92
        }
      },
      "source": [
        "sentence = Sentence('France is the current world cup winner.',\n",
        "                    labels = ['sports', 'world cup'])\n",
        "\n",
        "print(sentence)\n",
        "for label in sentence.labels:\n",
        "  print(label)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sentence: \"France is the current world cup winner.\" - 7 Tokens - Labels: [sports (1.0), world cup (1.0)] \n",
            "sports (1.0)\n",
            "world cup (1.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F2tZAeh-fxbV",
        "colab_type": "text"
      },
      "source": [
        "# Tutorial 2: Tagging your Text"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R30AKAPawvga",
        "colab_type": "text"
      },
      "source": [
        "## A. Tagging with Pre-Trained Sequence Tagging Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CFyjp2lWfqEd",
        "colab_type": "code",
        "outputId": "48595e20-35f5-4c90-a177-c1f5e907cbb5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        }
      },
      "source": [
        "from flair.models import SequenceTagger\n",
        "\n",
        "tagger = SequenceTagger.load('ner')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 08:42:01,108 loading file /root/.flair/models/en-ner-conll03-v0.4.pt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qfkh6vmWxJus",
        "colab_type": "code",
        "outputId": "7e50ff64-a05e-4897-a1b8-fe235a2e2ed0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "sentence = Sentence('George Washington went to Washington .')\n",
        "\n",
        "# predict NER tags\n",
        "tagger.predict(sentence)\n",
        "\n",
        "# print sentence with predicted tags\n",
        "print(sentence.to_tagged_string())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "George <B-PER> Washington <E-PER> went to Washington <S-LOC> .\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kL72kToNxqE9",
        "colab_type": "text"
      },
      "source": [
        "### A1. Getting Annotated Spans"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u0DOrbiGxUKX",
        "colab_type": "code",
        "outputId": "dc984e82-64f0-4bb2-bfc3-956002ae4d37",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "for entity in sentence.get_spans('ner'):\n",
        "  print(entity)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "PER-span [1,2]: \"George Washington\"\n",
            "LOC-span [5]: \"Washington\"\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aCopOu-MxwpS",
        "colab_type": "code",
        "outputId": "11189f9e-1aef-4c72-c5fd-56fe2d639f12",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        }
      },
      "source": [
        "print(sentence.to_dict(tag_type = 'ner'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'text': 'George Washington went to Washington .', 'labels': [], 'entities': [{'text': 'George Washington', 'start_pos': 0, 'end_pos': 17, 'type': 'PER', 'confidence': 0.9967881441116333}, {'text': 'Washington', 'start_pos': 26, 'end_pos': 36, 'type': 'LOC', 'confidence': 0.9993711113929749}]}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ehr08DCayeDD",
        "colab_type": "text"
      },
      "source": [
        "### A2. Tagging a German sentence"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "luBhbu3Cx5vZ",
        "colab_type": "code",
        "outputId": "c00b509e-6bbf-48ed-de8b-c972ff7d0cfa",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        }
      },
      "source": [
        "# load model\n",
        "tagger = SequenceTagger.load('de-ner')\n",
        "\n",
        "# make German sentence\n",
        "sentence = Sentence('George Washington ging nach Washington.')\n",
        "\n",
        "# predict NER tags\n",
        "tagger.predict(sentence)\n",
        "\n",
        "# print sentence with predicted tags\n",
        "print(sentence.to_tagged_string())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 08:42:02,858 loading file /root/.flair/models/de-ner-conll03-v0.4.pt\n",
            "George <B-PER> Washington <E-PER> ging nach Washington. <S-LOC>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "997TKC9QzhdB",
        "colab_type": "text"
      },
      "source": [
        "### A3. Tagging Multilingual Text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t872XPuxyu9K",
        "colab_type": "code",
        "outputId": "139cd0bd-3e17-4cc3-b0dd-8737ce1dc973",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        }
      },
      "source": [
        "# load model\n",
        "tagger = SequenceTagger.load('pos-multi')\n",
        "\n",
        "# text with English and German sentences\n",
        "sentence = Sentence('George Washington went to Washington . Dort kaufte er einen Hut .')\n",
        "\n",
        "# predict PoS tags\n",
        "tagger.predict(sentence)\n",
        "\n",
        "# print sentence with predicted tags\n",
        "print(sentence.to_tagged_string())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 08:42:46,893 loading file /root/.flair/models/pos-multi-v0.1.pt\n",
            "George <PROPN> Washington <PROPN> went <VERB> to <ADP> Washington <PROPN> . <PUNCT> Dort <ADV> kaufte <VERB> er <PRON> einen <DET> Hut <NOUN> . <PUNCT>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "drIPaZys0OTK",
        "colab_type": "text"
      },
      "source": [
        "## A4. Experimental: Semantic Frame Detection"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bx8Et-HTz5u2",
        "colab_type": "code",
        "outputId": "827a40b2-7877-4e71-acff-b05be59eec55",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 92
        }
      },
      "source": [
        "# load model\n",
        "tagger = SequenceTagger.load('frame')\n",
        "\n",
        "# make English sentence\n",
        "sentence_1 = Sentence('George returned to Berlin to return his hat .')\n",
        "sentence_2 = Sentence('He had a look at different hats .')\n",
        "\n",
        "# predict NER tags\n",
        "tagger.predict(sentence_1)\n",
        "tagger.predict(sentence_2)\n",
        "\n",
        "# print sentence with predicted tags\n",
        "print(sentence_1.to_tagged_string())\n",
        "print(sentence_2.to_tagged_string())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 08:42:55,838 loading file /root/.flair/models/en-frame-ontonotes-v0.4.pt\n",
            "George <_> returned <return.01> to <_> Berlin <_> to <_> return <return.02> his <_> hat <_> . <_>\n",
            "He <_> had <have.03> a <_> look <look.01> at <_> different <_> hats <_> . <_>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uFRDMtos1Cam",
        "colab_type": "text"
      },
      "source": [
        "### A5. Tagging a List of Sentences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NDLrDtd90w36",
        "colab_type": "code",
        "outputId": "c0843629-0297-4052-f406-9ea900786710",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 110
        }
      },
      "source": [
        "# your text of many sentences\n",
        "text = \"This is a sentence. This is another sentence. I love Berlin.\"\n",
        "\n",
        "# use a library to split into sentences\n",
        "from segtok.segmenter import split_single\n",
        "\n",
        "sentences = [Sentence(sent, use_tokenizer = True) for sent in split_single(text)]\n",
        "\n",
        "# predict tags for list of sentences\n",
        "tagger: SequenceTagger = SequenceTagger.load('ner')\n",
        "tagger.predict(sentences)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 08:43:04,144 loading file /root/.flair/models/en-ner-conll03-v0.4.pt\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Sentence: \"This is a sentence .\" - 5 Tokens,\n",
              " Sentence: \"This is another sentence .\" - 5 Tokens,\n",
              " Sentence: \"I love Berlin .\" - 4 Tokens]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tvp-vGqQ1rrQ",
        "colab_type": "text"
      },
      "source": [
        "## B. Tagging with Pre-Trained Text Classification Models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nVBPga-S1fPJ",
        "colab_type": "code",
        "outputId": "d4f9718f-6d5a-4668-a8e1-a20357b00af8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "from flair.models import TextClassifier\n",
        "\n",
        "classifier = TextClassifier.load('en-sentiment')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2019-12-20 08:43:05,756 loading file /root/.flair/models/imdb-v0.4.pt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-0r0R09119jP",
        "colab_type": "code",
        "outputId": "370151dc-b2d9-42e7-e44e-e24d2128fa8c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "sentence = Sentence('This film hurts. It is so bad that I am confused.')\n",
        "\n",
        "# predict NER tags\n",
        "classifier.predict(sentence)\n",
        "\n",
        "# print sentence with predicted labels\n",
        "print(sentence.labels)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[NEGATIVE (0.9598667025566101)]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FTduYjzV_kAn",
        "colab_type": "text"
      },
      "source": [
        "# Tutorial 3: Word Embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xb2qApd-_4A4",
        "colab_type": "text"
      },
      "source": [
        "## A. Classic Word Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HNm9wbfD_Y1w",
        "colab_type": "code",
        "outputId": "cc0f3a64-3a76-4854-c7cb-907fd8571417",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 73
        }
      },
      "source": [
        "from flair.embeddings import WordEmbeddings\n",
        "\n",
        "# init embedding\n",
        "glove_embedding = WordEmbeddings('glove')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/smart_open/smart_open_lib.py:402: UserWarning: This function is deprecated, use smart_open.open instead. See the migration notes for details: https://github.com/RaRe-Technologies/smart_open/blob/master/README.rst#migrating-to-the-new-open-function\n",
            "  'See the migration notes for details: %s' % _MIGRATION_NOTES_URL\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GzVP1bWcAMOa",
        "colab_type": "code",
        "outputId": "cf26f45a-39d0-42ff-b1bf-c881ee38e709",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# create sentence\n",
        "sentence = Sentence('I got another confession to make.')\n",
        "\n",
        "# embed a sentence using glove\n",
        "glove_embedding.embed(sentence)\n",
        "\n",
        "# now check out the embedded tokens\n",
        "for token in sentence:\n",
        "  print(token)\n",
        "  print(token.embedding)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Token: 1 I\n",
            "tensor([-0.0465,  0.6197,  0.5665, -0.4658, -1.1890,  0.4460,  0.0660,  0.3191,\n",
            "         0.1468, -0.2212,  0.7924,  0.2991,  0.1607,  0.0253,  0.1868, -0.3100,\n",
            "        -0.2811,  0.6051, -1.0654,  0.5248,  0.0642,  1.0358, -0.4078, -0.3801,\n",
            "         0.3080,  0.5996, -0.2699, -0.7603,  0.9422, -0.4692, -0.1828,  0.9065,\n",
            "         0.7967,  0.2482,  0.2571,  0.6232, -0.4477,  0.6536,  0.7690, -0.5123,\n",
            "        -0.4433, -0.2187,  0.3837, -1.1483, -0.9440, -0.1506,  0.3001, -0.5781,\n",
            "         0.2017, -1.6591, -0.0792,  0.0264,  0.2205,  0.9971, -0.5754, -2.7266,\n",
            "         0.3145,  0.7052,  1.4381,  0.9913,  0.1398,  1.3474, -1.1753,  0.0040,\n",
            "         1.0298,  0.0646,  0.9089,  0.8287, -0.4700, -0.1058,  0.5916, -0.4221,\n",
            "         0.5733, -0.5411,  0.1077,  0.3978, -0.0487,  0.0646, -0.6144, -0.2860,\n",
            "         0.5067, -0.4976, -0.8157,  0.1641, -1.9630, -0.2669, -0.3759, -0.9585,\n",
            "        -0.8584, -0.7158, -0.3234, -0.4312,  0.4139,  0.2837, -0.7093,  0.1500,\n",
            "        -0.2154, -0.3762, -0.0325,  0.8062], device='cuda:0')\n",
            "Token: 2 got\n",
            "tensor([ 0.5464,  0.1902,  0.5130, -0.7673, -0.2382, -0.0658,  0.2446,  0.3202,\n",
            "        -0.1321, -0.5108,  0.6910,  0.2446,  0.0753,  0.3406,  0.3774, -0.2765,\n",
            "        -0.2294,  0.3206, -0.4311,  0.3724,  0.4389,  0.6733,  0.1064, -0.2867,\n",
            "         0.1670,  0.0036, -0.6208, -0.3173,  0.5547, -0.4729, -0.0725,  1.1329,\n",
            "         0.4597,  0.5982,  0.4312, -0.1136, -0.7058,  0.3756,  0.0485,  0.4399,\n",
            "        -0.0782, -0.3277,  0.3706, -0.8684, -0.2289, -0.0492, -0.2156, -0.0500,\n",
            "         0.8218, -1.0077, -0.3538, -0.1255, -0.4025,  0.7098, -0.2467, -2.4466,\n",
            "        -0.0993,  0.4970,  1.0912,  0.7368,  0.2660,  1.0726, -1.1285,  0.2243,\n",
            "         0.4752, -0.3435,  0.4480,  0.3436, -0.2933,  0.3400, -0.0612,  0.1267,\n",
            "         0.0493, -0.3384,  0.0146,  0.5229, -0.6815, -0.2861, -0.5318,  0.0998,\n",
            "         0.2078,  0.0620, -0.8066, -0.4071, -1.5768, -0.4713,  0.4164, -0.1998,\n",
            "        -0.9482, -0.4067, -0.0120, -0.5402, -0.1542, -0.2559, -0.6889, -0.0638,\n",
            "         0.0613,  0.4414,  0.0822,  0.4268], device='cuda:0')\n",
            "Token: 3 another\n",
            "tensor([-1.3669e-01,  1.6266e-01,  3.2851e-01, -2.3838e-01,  3.7632e-01,\n",
            "         5.1200e-01,  4.3825e-01,  2.6590e-01,  1.0699e-01,  1.0075e-01,\n",
            "         9.9575e-02,  2.3082e-01,  3.0345e-02, -3.3396e-01,  3.8298e-01,\n",
            "        -2.4366e-01, -1.1523e-02, -3.5039e-01, -2.0009e-01,  4.6843e-01,\n",
            "         7.9308e-01, -3.0159e-01,  3.2797e-02,  2.8515e-01,  5.0240e-01,\n",
            "        -2.7093e-01, -4.2409e-01, -4.1382e-01,  9.4871e-02,  7.2463e-02,\n",
            "        -5.5992e-01,  4.6250e-01,  2.7367e-01, -1.1924e-01, -5.5437e-02,\n",
            "        -5.3994e-02, -2.2305e-01,  1.9366e-01,  4.2554e-01,  1.2814e-01,\n",
            "        -3.0268e-01,  1.1386e-01,  6.6691e-01, -5.6826e-01,  4.9106e-01,\n",
            "         3.5975e-01,  4.1939e-01,  5.5777e-02, -5.2086e-01, -4.4886e-01,\n",
            "         5.8267e-02, -8.2595e-02,  8.2818e-02,  1.1480e+00, -4.4502e-01,\n",
            "        -3.0693e+00, -3.8092e-01, -2.2423e-01,  1.6572e+00,  6.9771e-01,\n",
            "         5.7998e-02,  8.7766e-01, -3.2617e-02,  2.4173e-01,  7.2651e-01,\n",
            "        -2.4736e-01,  5.0189e-01,  5.8423e-01, -2.2143e-01,  1.8851e-01,\n",
            "        -4.6677e-01, -2.2507e-02, -2.1069e-01, -8.6704e-02,  3.1232e-01,\n",
            "         2.7984e-01, -2.1408e-01, -4.1711e-01, -1.1358e+00,  1.8686e-02,\n",
            "         8.9265e-01, -6.7525e-04, -2.6069e-01, -1.4054e-01, -1.1586e+00,\n",
            "        -3.2087e-01,  4.6940e-01,  1.9181e-01,  1.3760e-01, -5.5373e-01,\n",
            "         4.2719e-01,  1.4677e-01, -1.8532e-01, -1.6940e-01, -2.7725e-01,\n",
            "        -2.9067e-01,  1.7023e-01, -1.3190e-01,  4.8349e-01,  1.1721e-02],\n",
            "       device='cuda:0')\n",
            "Token: 4 confession\n",
            "tensor([ 0.7941, -0.0173,  0.3188, -0.3286, -0.0214,  1.2454,  0.3956,  0.4233,\n",
            "         0.2123, -0.0473,  0.0073,  0.7784, -0.0931,  1.3929,  0.5594,  0.4399,\n",
            "         0.1916,  0.1948, -0.4169,  0.7541, -0.0249, -0.6088, -0.7124,  0.2858,\n",
            "        -0.3600,  0.2607,  0.8570,  0.2394, -0.0722,  0.4884,  0.6362, -0.4175,\n",
            "        -0.3617, -0.2068, -0.5214, -0.0688, -0.1354, -0.1724, -0.0233, -0.0152,\n",
            "        -0.5621,  0.8566,  0.7419,  0.3400, -0.9375, -0.8577,  0.6919, -0.4567,\n",
            "         0.6837, -0.4721,  0.0693,  0.4299,  0.6115,  0.2925, -0.4352, -0.7271,\n",
            "        -0.5204, -0.5540,  0.4211, -0.2366,  0.2904, -0.1001, -0.2693, -0.9198,\n",
            "         0.2626, -1.1150,  0.4721,  0.1068, -0.7125,  0.0293, -0.2485,  0.0941,\n",
            "         1.1618, -0.8272, -0.1434,  0.5714,  0.3778, -0.6977, -0.7972, -0.1159,\n",
            "         0.0222,  0.3447,  0.3680,  0.1735, -1.2078, -0.4418,  0.5111,  0.4080,\n",
            "         0.0044,  0.3721, -0.0764, -0.9032, -0.3794,  0.5969,  0.9123, -0.6411,\n",
            "         0.3538, -0.4895, -0.3920,  0.1282], device='cuda:0')\n",
            "Token: 5 to\n",
            "tensor([-1.8970e-01,  5.0024e-02,  1.9084e-01, -4.9184e-02, -8.9737e-02,\n",
            "         2.1006e-01, -5.4952e-01,  9.8377e-02, -2.0135e-01,  3.4241e-01,\n",
            "        -9.2677e-02,  1.6100e-01, -1.3268e-01, -2.8160e-01,  1.8737e-01,\n",
            "        -4.2959e-01,  9.6039e-01,  1.3972e-01, -1.0781e+00,  4.0518e-01,\n",
            "         5.0539e-01, -5.5064e-01,  4.8440e-01,  3.8044e-01, -2.9055e-03,\n",
            "        -3.4942e-01, -9.9696e-02, -7.8368e-01,  1.0363e+00, -2.3140e-01,\n",
            "        -4.7121e-01,  5.7126e-01, -2.1454e-01,  3.5958e-01, -4.8319e-01,\n",
            "         1.0875e+00,  2.8524e-01,  1.2447e-01, -3.9248e-02, -7.6732e-02,\n",
            "        -7.6343e-01, -3.2409e-01, -5.7490e-01, -1.0893e+00, -4.1811e-01,\n",
            "         4.5120e-01,  1.2112e-01, -5.1367e-01, -1.3349e-01, -1.1378e+00,\n",
            "        -2.8768e-01,  1.6774e-01,  5.5804e-01,  1.5387e+00,  1.8859e-02,\n",
            "        -2.9721e+00, -2.4216e-01, -9.2495e-01,  2.1992e+00,  2.8234e-01,\n",
            "        -3.4780e-01,  5.1621e-01, -4.3387e-01,  3.6852e-01,  7.4573e-01,\n",
            "         7.2102e-02,  2.7931e-01,  9.2569e-01, -5.0336e-02, -8.5856e-01,\n",
            "        -1.3580e-01, -9.2551e-01, -3.3991e-01, -1.0394e+00, -6.7203e-02,\n",
            "        -2.1379e-01, -4.7690e-01,  2.1377e-01, -8.4008e-01,  5.2536e-02,\n",
            "         5.9298e-01,  2.9604e-01, -6.7644e-01,  1.3916e-01, -1.5504e+00,\n",
            "        -2.0765e-01,  7.2220e-01,  5.2056e-01, -7.6221e-02, -1.5194e-01,\n",
            "        -1.3134e-01,  5.8617e-02, -3.1869e-01, -6.1419e-01, -6.2393e-01,\n",
            "        -4.1548e-01, -3.8175e-02, -3.9804e-01,  4.7647e-01, -1.5983e-01],\n",
            "       device='cuda:0')\n",
            "Token: 6 make.\n",
            "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
            "        0., 0., 0., 0.], device='cuda:0')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "26kjKHc-AlPN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "german_embedding = WordEmbeddings('de-crawl')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2jV4aiTpBpJI",
        "colab_type": "text"
      },
      "source": [
        "## B. Flair Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nnMYQ60DA2P9",
        "colab_type": "code",
        "outputId": "8d7da8af-fd84-4ea3-d44f-90fe28d574a7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "from flair.embeddings import FlairEmbeddings\n",
        "\n",
        "# init embedding\n",
        "flair_embedding_forward = FlairEmbeddings('news-forward')\n",
        "\n",
        "# create a sentence\n",
        "sentence = Sentence('Christmas came early this year.')\n",
        "\n",
        "# embed words in sentence\n",
        "flair_embedding_forward.embed(sentence)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Sentence: \"Christmas came early this year.\" - 5 Tokens]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yyVcOPzVCA3y",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# init forward embedding for German\n",
        "flair_embedding_forward = FlairEmbeddings('de-forward')\n",
        "flair_embedding_backward = FlairEmbeddings('de-backward')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Iq33G88zCodB",
        "colab_type": "text"
      },
      "source": [
        "## C. Stacked Embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4gFPUaY7CTZF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flair.embeddings import WordEmbeddings, CharacterEmbeddings\n",
        "\n",
        "# init standard GloVe embedding\n",
        "glove_embedding = WordEmbeddings('glove')\n",
        "\n",
        "# init Flair forward and backwards embeddings\n",
        "flair_embedding_forward = FlairEmbeddings('news-forward')\n",
        "flair_embedding_backward = FlairEmbeddings('news-backward')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dOJ7hKkzDB6r",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flair.embeddings import WordEmbeddings, FlairEmbeddings, StackedEmbeddings\n",
        "\n",
        "# create a StackedEmbedding object that combines GloVe and \n",
        "# forward / backward Flair embeddings\n",
        "\n",
        "stacked_embeddings = StackedEmbeddings([\n",
        "                                        glove_embedding,\n",
        "                                        flair_embedding_forward,\n",
        "                                        flair_embedding_backward\n",
        "])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mp9bKqHgEM5i",
        "colab_type": "code",
        "outputId": "58a5c932-476b-48f4-ec23-59178f0db812",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 399
        }
      },
      "source": [
        "sentence = Sentence('Boston is in the state of Massachusetts.')\n",
        "\n",
        "# just embed a sentence using the StackedEmbedding as you would\n",
        "# with any single embedding.\n",
        "stacked_embeddings.embed(sentence)\n",
        "\n",
        "# now check out the embedded tokens\n",
        "for token in sentence:\n",
        "  print(token)\n",
        "  print(token.embedding)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Token: 1 Boston\n",
            "tensor([ 7.1747e-01,  2.8692e-01, -4.3357e-02,  ...,  1.7524e-04,\n",
            "         1.3637e-03,  4.0130e-03], device='cuda:0')\n",
            "Token: 2 is\n",
            "tensor([-5.4264e-01,  4.1476e-01,  1.0322e+00,  ...,  1.2819e-04,\n",
            "        -3.6461e-02,  1.1889e-01], device='cuda:0')\n",
            "Token: 3 in\n",
            "tensor([ 0.0857, -0.2220,  0.1657,  ..., -0.0008, -0.0158,  0.0045],\n",
            "       device='cuda:0')\n",
            "Token: 4 the\n",
            "tensor([-0.0382, -0.2449,  0.7281,  ...,  0.0023, -0.0072,  0.0021],\n",
            "       device='cuda:0')\n",
            "Token: 5 state\n",
            "tensor([ 1.1835e-03, -1.6506e-01,  1.2236e+00,  ..., -1.6769e-03,\n",
            "        -1.0730e-01, -2.1249e-02], device='cuda:0')\n",
            "Token: 6 of\n",
            "tensor([-0.1529, -0.2428,  0.8984,  ...,  0.1088,  0.0714,  0.0622],\n",
            "       device='cuda:0')\n",
            "Token: 7 Massachusetts.\n",
            "tensor([ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ..., -7.2782e-05,\n",
            "        -2.2795e-02,  9.4885e-05], device='cuda:0')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G_ejvh6oE4y5",
        "colab_type": "text"
      },
      "source": [
        "# Tutorial 4: List of All Word Embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6ozwG4mxGCfT",
        "colab_type": "text"
      },
      "source": [
        "## A. Combining BERT and Flair"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gEAs4MFXEpZa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flair.embeddings import FlairEmbeddings, BertEmbeddings\n",
        "\n",
        "# init Flair embeddings\n",
        "flair_forward_embedding = FlairEmbeddings('multi-forward')\n",
        "flair_backward_embedding = FlairEmbeddings('multi-backward')\n",
        "\n",
        "# init multilingual BERT\n",
        "bert_embedding = BertEmbeddings('bert-base-multilingual-cased')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A1J9lGoMHv-d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flair.embeddings import StackedEmbeddings\n",
        "\n",
        "# now create the StackedEmbedding object that combines\n",
        "# all embeddings\n",
        "stacked_embeddings = StackedEmbeddings(\n",
        "    embeddings = [flair_forward_embedding, \n",
        "                  flair_backward_embedding,\n",
        "                  bert_embedding]\n",
        ")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LZmrUjIuIqHf",
        "colab_type": "code",
        "outputId": "92d8aaee-4409-4489-b053-7ce2f6d95da5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 345
        }
      },
      "source": [
        "sentence = Sentence('We see things they never see.')\n",
        "\n",
        "# just embed a sentence using the StackedEmbedding as you\n",
        "# would with any single embedding.\n",
        "stacked_embeddings.embed(sentence)\n",
        "\n",
        "# now check out the embedded tokens.\n",
        "for token in sentence:\n",
        "  print(token)\n",
        "  print(token.embedding)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Token: 1 We\n",
            "tensor([-0.4077,  0.0265,  0.0092,  ...,  0.4373,  0.7063,  1.0879],\n",
            "       device='cuda:0')\n",
            "Token: 2 see\n",
            "tensor([1.8586e-01, 7.4364e-03, 8.7382e-04,  ..., 1.0972e+00, 3.7068e-01,\n",
            "        8.3195e-01], device='cuda:0')\n",
            "Token: 3 things\n",
            "tensor([2.7703e-01, 3.3039e-02, 1.2601e-04,  ..., 1.8076e+00, 6.2378e-01,\n",
            "        4.4064e-01], device='cuda:0')\n",
            "Token: 4 they\n",
            "tensor([-1.1517e-01,  1.9221e-02,  1.2209e-04,  ...,  1.2136e+00,\n",
            "         6.5522e-01,  5.4608e-01], device='cuda:0')\n",
            "Token: 5 never\n",
            "tensor([-7.3337e-01,  1.7415e-02,  1.8182e-05,  ...,  1.9867e+00,\n",
            "         8.5260e-01,  7.6160e-01], device='cuda:0')\n",
            "Token: 6 see.\n",
            "tensor([-3.4127e-01,  8.6519e-03,  3.3226e-06,  ...,  1.6097e+00,\n",
            "         4.0221e-01,  2.0394e-01], device='cuda:0')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ctsLhQB2Jt4E",
        "colab_type": "text"
      },
      "source": [
        "# Tutorial 5: Document Embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ynZ4zQhoJ_xD",
        "colab_type": "text"
      },
      "source": [
        "## A. Document Embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZRT3Eh3VKDaA",
        "colab_type": "text"
      },
      "source": [
        "### A1. Pooling"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "foci0L10Jksy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flair.embeddings import WordEmbeddings, FlairEmbeddings\n",
        "from flair.embeddings import DocumentPoolEmbeddings, Sentence\n",
        "\n",
        "# initialize the word embeddings\n",
        "glove_embedding = WordEmbeddings('glove')\n",
        "flair_embedding_forward = FlairEmbeddings('news-forward')\n",
        "flair_embedding_backward = FlairEmbeddings('news-backward')\n",
        "\n",
        "# initialize the document embeddings, mode = mean\n",
        "document_embeddings = DocumentPoolEmbeddings([glove_embedding,\n",
        "                                              flair_embedding_forward,\n",
        "                                              flair_embedding_backward])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N_xZC7XRK3Kr",
        "colab_type": "code",
        "outputId": "7cfab55a-4fe4-4f14-a4ab-d16cf1ad2ec9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "source": [
        "# create an example sentence\n",
        "sentence = Sentence('The grass is green. And the sky is blue.')\n",
        "\n",
        "# embed the sentence with our document embedding\n",
        "document_embeddings.embed(sentence)\n",
        "\n",
        "# now check out the embedded sentence\n",
        "print(sentence.get_embedding())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([-0.2494,  0.1465,  0.4193,  ...,  0.0008, -0.0205, -0.0008],\n",
            "       device='cuda:0', grad_fn=<CatBackward>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QjHuvJ8XLJb1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "document_embeddings = DocumentPoolEmbeddings([glove_embedding,\n",
        "                                              flair_embedding_forward,\n",
        "                                              flair_embedding_backward],\n",
        "                                             pooling = 'min')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5j4xXxB2LmiR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# instantiate pre-trained word embeddings\n",
        "embeddings = WordEmbeddings('glove')\n",
        "\n",
        "# document pool embeddings\n",
        "document_embeddings = DocumentPoolEmbeddings([embeddings],\n",
        "                                             fine_tune_mode = 'nonlinear')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gL-2wIzUMd47",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flair.embeddings import OneHotEmbeddings\n",
        "\n",
        "# instantiate one-hot encoded word embeddings\n",
        "# embeddings = OneHotEmbeddings(corpus)\n",
        "\n",
        "# # document pool embeddings\n",
        "# document_embeddings = DocumentPoolEmbeddings([embeddings],\n",
        "#                                              fine_tune_mode = 'none')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uhryMckUOoE8",
        "colab_type": "text"
      },
      "source": [
        "### A2. RNN"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Peb8KUf5NQp_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flair.embeddings import WordEmbeddings, DocumentRNNEmbeddings\n",
        "\n",
        "glove_embedding = WordEmbeddings('glove')\n",
        "\n",
        "document_embeddings = DocumentRNNEmbeddings([glove_embedding])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FTgaj2HWPDDw",
        "colab_type": "code",
        "outputId": "60249f29-4b8a-4d50-9b0b-651fa05a67e2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 346
        }
      },
      "source": [
        "# create an example sentence\n",
        "sentence = Sentence('The grass is green. And the sky is blue.')\n",
        "\n",
        "# embed the sentence with our document embedding\n",
        "document_embeddings.embed(sentence)\n",
        "\n",
        "# now check out the embedded sentence.\n",
        "print(sentence.get_embedding())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([ 0.0716, -0.0029,  0.0113, -0.0852, -0.1412, -0.0882, -0.1054, -0.0443,\n",
            "        -0.2207,  0.1358, -0.0730, -0.1728,  0.0569,  0.2528,  0.1463,  0.0390,\n",
            "        -0.1720, -0.1887,  0.0682, -0.2321,  0.0260,  0.0275, -0.1433, -0.0222,\n",
            "        -0.0617, -0.1305,  0.1678, -0.0938, -0.0282, -0.0858,  0.0849,  0.0312,\n",
            "         0.1670,  0.1790,  0.0671, -0.2630, -0.1673, -0.1985,  0.0852,  0.1641,\n",
            "         0.0219, -0.1363,  0.2222, -0.1169, -0.1913, -0.1164,  0.0226, -0.1652,\n",
            "         0.2111, -0.1139,  0.4214, -0.1383, -0.1504,  0.0193,  0.0126, -0.0391,\n",
            "        -0.0957, -0.2721,  0.0869,  0.2812, -0.1523, -0.0153, -0.2355, -0.0652,\n",
            "         0.0448, -0.1141,  0.1463, -0.1454,  0.1379,  0.0849,  0.0250, -0.0075,\n",
            "         0.1289,  0.0090, -0.0008,  0.0645,  0.2504, -0.1422,  0.0176,  0.1865,\n",
            "         0.2174,  0.1410, -0.0827, -0.0324, -0.2778, -0.0379, -0.2863,  0.0455,\n",
            "        -0.2348, -0.1785, -0.2250,  0.0749, -0.0767,  0.1159, -0.2740, -0.0188,\n",
            "        -0.1326,  0.0304, -0.0614, -0.1590,  0.1693,  0.3065, -0.2462, -0.0259,\n",
            "        -0.0532, -0.1089, -0.1187,  0.0836,  0.1838, -0.0464, -0.0735, -0.1065,\n",
            "         0.1209, -0.0876,  0.0251, -0.0247,  0.2439,  0.1061,  0.0280, -0.1691,\n",
            "        -0.1463,  0.2087,  0.0096,  0.0700, -0.0394, -0.1053,  0.0546, -0.1235],\n",
            "       device='cuda:0', grad_fn=<CatBackward>)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hDxEF4U8Pgw_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from flair.embeddings import WordEmbeddings, DocumentRNNEmbeddings\n",
        "\n",
        "glove_embedding = WordEmbeddings('glove')\n",
        "\n",
        "document_lstm_embeddings = DocumentRNNEmbeddings([glove_embedding],\n",
        "                                                 rnn_type = 'LSTM')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VnrhpWosP5WM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# document_embeddings = classifier.document_embeddings\n",
        "\n",
        "# sentence = Sentence('The grass is green. And the sky is blue.')\n",
        "\n",
        "# document_embeddings.embed(sentence)\n",
        "\n",
        "# print(sentence.get_embedding())"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}