Lexlata-Final-Model-Classification-Alternative.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normal modelimize alternatif olarak oluşturduğumuz bir modeldir. Normal modelimizin aksine burada girdinin doküman türünü sınıflandırıyoruz. Bunun sonucu olarak yalnızca girilen dokümanın türünde çıktılar alınıyor.\n",
    "\n",
    "Girdilerin (CSV değerlendirme dosyasının) özelliklerine bağlı olarak 2 modelimizin performansı farklılık gösterebileceğinden dolayı duruma göre birini seçeceğiz."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Yüklü değillerse gerekli kütüphanelerin yüklenmesi\n",
    "\n",
    "#!pip install torch\n",
    "#!pip install transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gerekli kütüphanelerin içe aktarılması\n",
    "\n",
    "from transformers import BertTokenizer, BertModel\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Cümle temsillerinin deserialize edilerek RAM'e alınması\n",
    "\n",
    "import pickle\n",
    "\n",
    "embeddings = []\n",
    "\n",
    "for i in range(0, 501):\n",
    "    with open(f'embeddings/sbert_embeddings{i}.json', 'rb') as file:\n",
    "        obj = pickle.load(file)\n",
    "        embeddings.extend(obj)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No sentence-transformers model found with name C:\\Users\\Furkan/.cache\\torch\\sentence_transformers\\sfurkan_LexBERT-turkish-uncased. Creating a new one with MEAN pooling.\n",
      "Some weights of the model checkpoint at C:\\Users\\Furkan/.cache\\torch\\sentence_transformers\\sfurkan_LexBERT-turkish-uncased were not used when initializing BertModel: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight']\n",
      "- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertModel were not initialized from the model checkpoint at C:\\Users\\Furkan/.cache\\torch\\sentence_transformers\\sfurkan_LexBERT-turkish-uncased and are newly initialized: ['bert.pooler.dense.weight', 'bert.pooler.dense.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "# Huggingface servisinde barındırılan BERT modelimizin indirilmesi\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "model = SentenceTransformer('sfurkan/LexBERT-turkish-uncased')\n",
    "model.max_seq_length = 65536"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Veri kümemizin içe aktarılması\n",
    "\n",
    "TARGETED_DATASET_PATH = r'data.csv'\n",
    "\n",
    "texts = pd.read_csv(TARGETED_DATASET_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "4142it [00:01, 2694.50it/s]\n"
     ]
    }
   ],
   "source": [
    "# Herhangi bir cümlenin indexini alıp içinde bulunduğu dokümanın indexini döndüren dictionary veri yapısının oluşturulması\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "sentenceIndex2DocIndex = {}\n",
    "lastSentenceIndex = 0\n",
    "currentDocIndex = 0\n",
    "for text in tqdm(texts.iloc):\n",
    "    text = text['data_text']\n",
    "    sentences = text.strip().split('.')\n",
    "    \n",
    "    for idx, sentence in enumerate(sentences):\n",
    "        sentenceIndex2DocIndex[idx + lastSentenceIndex] = currentDocIndex\n",
    "    \n",
    "    lastSentenceIndex += len(sentences)\n",
    "    currentDocIndex += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Doküman türünü çıkarabilen 'text classification' modelimizin indirilmesi\n",
    "\n",
    "from transformers import pipeline\n",
    "\n",
    "classificationPipeline = pipeline(\"text-classification\", model=\"sfurkan/LexBERT-textclassification-turkish-uncased\", tokenizer = \"sfurkan/LexBERT-textclassification-turkish-uncased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Girilen dokümanın BERT tarafından kullanılabilmesi için ilk 512 kelimeyi alacak şekilde kırpılması. (Yalnızca 'text classification' için kullanılır)\n",
    "\n",
    "def cropDocument(document, cropLimit=512):\n",
    "    totalWords = []\n",
    "    sentences = document.replace('\\n', '. ').split('.')\n",
    "    for sent in sentences:\n",
    "        sent = sent.strip()\n",
    "        words = sent.split(' ')\n",
    "        for word in words[:-1]:\n",
    "            if len(totalWords) >= cropLimit:\n",
    "                return totalWords\n",
    "            totalWords.append(word)\n",
    "        totalWords.append(f\"{words[-1]}.\")\n",
    "    return ' '.join(totalWords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Girilen dokümanın türünü döndürecek fonksiyon\n",
    "\n",
    "def getDocumentType(inputText):\n",
    "    return classificationPipeline(cropDocument(inputText))[0][\"label\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████| 4142/4142 [00:02<00:00, 1699.64it/s]\n"
     ]
    }
   ],
   "source": [
    "# Doküman içerisindeki cümle temsillerinin organizasyonunun 'liste içinde liste' şeklinde bir veri yapısında sağlanması\n",
    "documentEmbeddings = []\n",
    "def initializeDocEmbeddings():\n",
    "    embeddingIndex = 0\n",
    "    for idx in tqdm(range(texts.shape[0])):\n",
    "        text = texts.loc[idx][\"data_text\"]\n",
    "        sents = text.strip().split('.')\n",
    "        sentEmbeddings = []\n",
    "        for sent in sents:\n",
    "            sentEmbeddings.append(embeddings[embeddingIndex])\n",
    "            embeddingIndex += 1\n",
    "        documentEmbeddings.append(sentEmbeddings)\n",
    "initializeDocEmbeddings()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Verilen girdi dokümanı kullanarak benzer dokümanları döndüren fonksiyon\n",
    "\n",
    "from scipy.spatial.distance import cosine\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "def getSimilarDocuments(inputText, topk=10):\n",
    "    # Girdinin doküman türünün belirlenmesi\n",
    "    docType = getDocumentType(inputText)\n",
    "\n",
    "    inputEmbedding = model.encode(inputText)\n",
    "\n",
    "    documentScores = []    \n",
    "    for idx, docEmb in enumerate(documentEmbeddings):\n",
    "        # Iterate edilen dokümanın kategorisi, aranan kategori değilse atlanması\n",
    "        if texts.iloc[idx][\"kategori\"] != docType:\n",
    "            continue\n",
    "        \n",
    "        similarities = cosine_similarity(docEmb, [inputEmbedding])\n",
    "        meanSim = 0\n",
    "        for sim in similarities:\n",
    "            meanSim += sim[0]\n",
    "        meanSim /= len(similarities)\n",
    "        documentScores.append((idx, meanSim))\n",
    "\n",
    "    unfilteredDocuments = []\n",
    "    maxSim = -1\n",
    "    for idx, docScore in documentScores:\n",
    "        if docScore > maxSim:\n",
    "            maxSim = docScore\n",
    "            unfilteredDocuments.append((texts.iloc[idx], docScore))\n",
    "    \n",
    "    documentsToReturn = sorted(unfilteredDocuments, key=lambda x: x[1], reverse=True)[:min(len(unfilteredDocuments), topk)]\n",
    "    \n",
    "    documentRows = []\n",
    "    for document, score in documentsToReturn:\n",
    "        documentRows.append((*document.values, score))\n",
    "        \n",
    "        #dfRows.append(pd.DataFrame([document, pd.Series(score)], columns=texts.columns))\n",
    "    similarDF = pd.DataFrame(documentRows, columns=[*texts.columns, 'Similarity Score'])\n",
    "    \n",
    "    return similarDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>kategori</th>\n",
       "      <th>baslik</th>\n",
       "      <th>rega_no</th>\n",
       "      <th>mukerrer_no</th>\n",
       "      <th>rega_tarihi</th>\n",
       "      <th>kurum</th>\n",
       "      <th>mevzuat_no</th>\n",
       "      <th>belge_sayi</th>\n",
       "      <th>mevzuat_tarihi</th>\n",
       "      <th>donem</th>\n",
       "      <th>sira_no</th>\n",
       "      <th>madde_sayisi</th>\n",
       "      <th>data_text</th>\n",
       "      <th>url</th>\n",
       "      <th>kanunum_url</th>\n",
       "      <th>Similarity Score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>129</td>\n",
       "      <td>Kanun</td>\n",
       "      <td>YATIRIM ORTAMININ İYİLEŞTİRİLMESİ AMACIYLA BAZ...</td>\n",
       "      <td>29796</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2016-08-09</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6728</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2016-07-15</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>78</td>\n",
       "      <td>Resmî Gazete Tarihi: 09.08.2016 Resmî Gazete S...</td>\n",
       "      <td>http://www.mevzuat.gov.tr/Metin.Aspx?MevzuatKo...</td>\n",
       "      <td>https://www.kanunum.com/content/2654308</td>\n",
       "      <td>0.703920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>29</td>\n",
       "      <td>Kanun</td>\n",
       "      <td>GELİR VERGİSİ KANUNU İLE BAZI KANUNLARDA DEĞİŞ...</td>\n",
       "      <td>30836</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2019-07-19</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7186</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2019-07-17</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>35</td>\n",
       "      <td>Resmî Gazete Tarihi: 19.07.2019 Resmî Gazete S...</td>\n",
       "      <td>http://www.resmigazete.gov.tr/eskiler/2019/07/...</td>\n",
       "      <td>https://www.kanunum.com/content/8613190</td>\n",
       "      <td>0.699733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>Kanun</td>\n",
       "      <td>İŞ KANUNU İLE BAZI KANUN VE KANUN HÜKMÜNDE KAR...</td>\n",
       "      <td>29116</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2014-09-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6552</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014-09-10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>149</td>\n",
       "      <td>Resmî Gazete Tarihi: 11.09.2014 Resmî Gazete S...</td>\n",
       "      <td>https://www.mevzuat.gov.tr/mevzuat?MevzuatNo=6...</td>\n",
       "      <td>https://www.kanunum.com/content/1233870</td>\n",
       "      <td>0.685094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>Kanun</td>\n",
       "      <td>SAĞLIK BAKANLIĞI VE BAĞLI KURULUŞLARININ TEŞKİ...</td>\n",
       "      <td>28886</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2014-01-18</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6514</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2014-01-02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>56</td>\n",
       "      <td>Resmî Gazete Tarihi: 18.01.2014 Resmî Gazete S...</td>\n",
       "      <td>https://www.resmigazete.gov.tr/eskiler/2014/01...</td>\n",
       "      <td>https://www.kanunum.com/content/942558</td>\n",
       "      <td>0.674757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>Kanun</td>\n",
       "      <td>BÜTÇE KANUNLARINDA YER ALAN BAZI HÜKÜMLERİN İL...</td>\n",
       "      <td>28350</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2012-07-11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6338</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2012-06-29</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>19</td>\n",
       "      <td>Resmî Gazete Tarihi: 11.07.2012 Resmî Gazete S...</td>\n",
       "      <td>https://www.resmigazete.gov.tr/eskiler/2012/07...</td>\n",
       "      <td>https://www.kanunum.com/content/344634</td>\n",
       "      <td>0.668466</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id kategori                                             baslik rega_no  \\\n",
       "0  129    Kanun  YATIRIM ORTAMININ İYİLEŞTİRİLMESİ AMACIYLA BAZ...   29796   \n",
       "1   29    Kanun  GELİR VERGİSİ KANUNU İLE BAZI KANUNLARDA DEĞİŞ...   30836   \n",
       "2    9    Kanun  İŞ KANUNU İLE BAZI KANUN VE KANUN HÜKMÜNDE KAR...   29116   \n",
       "3    5    Kanun  SAĞLIK BAKANLIĞI VE BAĞLI KURULUŞLARININ TEŞKİ...   28886   \n",
       "4    3    Kanun  BÜTÇE KANUNLARINDA YER ALAN BAZI HÜKÜMLERİN İL...   28350   \n",
       "\n",
       "   mukerrer_no rega_tarihi  kurum mevzuat_no  belge_sayi mevzuat_tarihi  \\\n",
       "0          0.0  2016-08-09    NaN       6728         NaN     2016-07-15   \n",
       "1          1.0  2019-07-19    NaN       7186         NaN     2019-07-17   \n",
       "2          1.0  2014-09-11    NaN       6552         NaN     2014-09-10   \n",
       "3          0.0  2014-01-18    NaN       6514         NaN     2014-01-02   \n",
       "4          0.0  2012-07-11    NaN       6338         NaN     2012-06-29   \n",
       "\n",
       "   donem  sira_no  madde_sayisi  \\\n",
       "0    NaN      NaN            78   \n",
       "1    NaN      NaN            35   \n",
       "2    NaN      NaN           149   \n",
       "3    NaN      NaN            56   \n",
       "4    NaN      NaN            19   \n",
       "\n",
       "                                           data_text  \\\n",
       "0  Resmî Gazete Tarihi: 09.08.2016 Resmî Gazete S...   \n",
       "1  Resmî Gazete Tarihi: 19.07.2019 Resmî Gazete S...   \n",
       "2  Resmî Gazete Tarihi: 11.09.2014 Resmî Gazete S...   \n",
       "3  Resmî Gazete Tarihi: 18.01.2014 Resmî Gazete S...   \n",
       "4  Resmî Gazete Tarihi: 11.07.2012 Resmî Gazete S...   \n",
       "\n",
       "                                                 url  \\\n",
       "0  http://www.mevzuat.gov.tr/Metin.Aspx?MevzuatKo...   \n",
       "1  http://www.resmigazete.gov.tr/eskiler/2019/07/...   \n",
       "2  https://www.mevzuat.gov.tr/mevzuat?MevzuatNo=6...   \n",
       "3  https://www.resmigazete.gov.tr/eskiler/2014/01...   \n",
       "4  https://www.resmigazete.gov.tr/eskiler/2012/07...   \n",
       "\n",
       "                               kanunum_url  Similarity Score  \n",
       "0  https://www.kanunum.com/content/2654308          0.703920  \n",
       "1  https://www.kanunum.com/content/8613190          0.699733  \n",
       "2  https://www.kanunum.com/content/1233870          0.685094  \n",
       "3   https://www.kanunum.com/content/942558          0.674757  \n",
       "4   https://www.kanunum.com/content/344634          0.668466  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Örnek kullanım\n",
    "\n",
    "INPUT_TEXT = \"Elektronik eşya vergilendirmesi hakkında kanun\"\n",
    "\n",
    "getSimilarDocuments(INPUT_TEXT, topk=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}