NLP Quiz.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['walk', 'teh', 'start', 'cup', 'coffee', 'hand', 'feel', 'warm', 'sun', 'face', 'sun', 'hang', 'low', 'sky', 'cast', 'long', 'shadow', 'tranquil', 'meadow', 'gentle', 'breeze', 'whisper', 'tall', 'grass', 'cause', 'sway', 'rhythmically', 'like', 'sea', 'green', 'wave', 'air', 'fill', 'sweet', 'fragrance', 'wildflower', 'perform', 'surrounding', 'nature', 'delicate', 'scent', 'bird', 'charge', 'melodiously', 'branch', 'nearby', 'tree', 'cheerful', 'tune', 'add', 'symphony', 'afternoon', 'distance', 'river', 'render', 'family', 'landscape', 'crystal', 'clear', 'water', 'shimmer', 'golden', 'light', 'set', 'sun', 'family', 'deer', 'graze', 'peacefully', 'water', 'edge', 'graceful', 'movement', 'testament', 'tranquility', 'scene', 'world', 'slow', 'moment', 'stretch', 'eternity', 'time', 'pause', 'favor', 'beauty', 'moment', 'evening', 'approach', 'sky', 'blame', 'vibrant', 'hue', 'orange', 'pink', 'paint', 'breathtaking', 'masterpiece', 'heaven', 'star', 'essay', 'writing', 'kind', 'assignment', 'assign', 'student', 'school', 'college', 'piece', 'writing', 'freedom', 'express', 'opinion', 'form', 'writing', 'thing', 'word', 'essay', 'usually', 'give', 'student', 'form', 'essay', 'pupil', 'require', 'express', 'opinion', 'give', 'word', 'limit', 'word', 'student', 'doubt', 'possible', 'write', 'essay', 'word', 'limit', 'think', 'let', 'tell', 'possible', 'write', 'essay', 'word', 'follow', 'proper', 'essay', 'format', 'introduction', 'body', 'conclusion', 'think', 'crucial', 'point', 'care', 'write', 'include', 'proper', 'planning', 'research', 'execution', 'word', 'essay', 'common', 'form', 'essay', 'assign', 'student', 'help', 'quantify', 'research', 'skill', 'opinion', 'knowledge', 'essay', 'easy', 'case', 'essay', 'writing', 'get', 'tough', 'word', 'limit', 'know', 'essay', 'research', 'present', 'argument', 'evidence', 'short', 'essay', 'word', 'need', 'skill', 'present', 'thought', 'idea', 'condense', 'manner', 'student', 'present', 'opinion', 'limit', 'word', 'type', 'essay', 'require', 'proper', 'plan', 'deep', 'research', 'topic', 'write', 'vaguely', 'precision', 'essay', 'order', 'handle', 'large', 'text', 'file', 'efficiently', 'crucial', 'implement', 'strategy', 'minimize', 'memory', 'usage', 'sacrifice', 'integrity', 'datum', 'deal', 'text', 'processing', 'task', 'like', 'tokenization', 'lemmatization', 'bag', 'word', 'bw', 'representation', 'select', 'relevant', 'information', 'imperative', 'achieve', 'step', 'involve', 'break', 'text', 'manageable', 'chunk', 'usually', 'line', 'paragraph', 'load', 'entire', 'content', 'memory', 'read', 'file', 'line', 'line', 'processing', 'perform', 'iterative', 'reduce', 'strain', 'system', 'resource', 'text', 'segment', 'line', 'undergo', 'tokenization', 'lemmatization', 'tool', 'like', 'space', 'efficiently', 'parse', 'extract', 'linguistic', 'feature', 'process', 'token', 'accumulate', 'analyze', 'build', 'vocabulary', 'essentially', 'capture', 'unique', 'word', 'present', 'text', 'scenario', 'file', 'size', 'considerable', 'store', 'entire', 'vocabulary', 'feasible', 'practical', 'approach', 'identify', 'n', 'frequent', 'word', 'refer', 'high', 'frequency', 'word', 'stop', 'word', 'base', 'occurrence', 'text']\n",
      "('teh', 1)|('start', 1)|('cup', 1)|('coffee', 1)|('hand', 1)|('feel', 1)|('warm', 1)|('sun', 3)|('face', 1)|('hang', 1)|('low', 1)|('sky', 2)|('cast', 1)|('long', 1)|('shadow', 1)|('tranquil', 1)|('meadow', 1)|('gentle', 1)|('breeze', 1)|('whisper', 1)|('tall', 1)|('grass', 1)|('cause', 1)|('sway', 1)|('rhythmically', 1)|('like', 3)|('sea', 1)|('green', 1)|('wave', 1)|('air', 1)|('fill', 1)|('sweet', 1)|('fragrance', 1)|('wildflower', 1)|('perform', 2)|('surrounding', 1)|('nature', 1)|('delicate', 1)|('scent', 1)|('bird', 1)|('charge', 1)|('melodiously', 1)|('branch', 1)|('nearby', 1)|('tree', 1)|('cheerful', 1)|('tune', 1)|('add', 1)|('symphony', 1)|('afternoon', 1)|('distance', 1)|('river', 1)|('render', 1)|('family', 2)|('landscape', 1)|('crystal', 1)|('clear', 1)|('water', 2)|('shimmer', 1)|('golden', 1)|('light', 1)|('set', 1)|('deer', 1)|('graze', 1)|('peacefully', 1)|('edge', 1)|('graceful', 1)|('movement', 1)|('testament', 1)|('tranquility', 1)|('scene', 1)|('world', 1)|('slow', 1)|('moment', 2)|('stretch', 1)|('eternity', 1)|('time', 1)|('pause', 1)|('favor', 1)|('beauty', 1)|('evening', 1)|('approach', 2)|('blame', 1)|('vibrant', 1)|('hue', 1)|('orange', 1)|('pink', 1)|('paint', 1)|('breathtaking', 1)|('masterpiece', 1)|('heaven', 1)|('star', 1)|('essay', 14)|('writing', 4)|('kind', 1)|('assignment', 1)|('assign', 2)|('student', 5)|('school', 1)|('college', 1)|('piece', 1)|('freedom', 1)|('express', 2)|('opinion', 4)|('form', 3)|('thing', 1)|('word', 14)|('usually', 2)|('give', 2)|('pupil', 1)|('require', 2)|('limit', 4)|('doubt', 1)|('possible', 2)|('write', 4)|('think', 2)|('let', 1)|('tell', 1)|('follow', 1)|('proper', 3)|('format', 1)|('introduction', 1)|('body', 1)|('conclusion', 1)|('crucial', 2)|('point', 1)|('care', 1)|('include', 1)|('planning', 1)|('research', 4)|('execution', 1)|('common', 1)|('help', 1)|('quantify', 1)|('skill', 2)|('knowledge', 1)|('easy', 1)|('case', 1)|('get', 1)|('tough', 1)|('know', 1)|('present', 4)|('argument', 1)|('evidence', 1)|('short', 1)|('need', 1)|('thought', 1)|('idea', 1)|('condense', 1)|('manner', 1)|('type', 1)|('plan', 1)|('deep', 1)|('topic', 1)|('vaguely', 1)|('precision', 1)|('order', 1)|('handle', 1)|('large', 1)|('text', 6)|('file', 3)|('efficiently', 2)|('implement', 1)|('strategy', 1)|('minimize', 1)|('memory', 2)|('usage', 1)|('sacrifice', 1)|('integrity', 1)|('datum', 1)|('deal', 1)|('processing', 2)|('task', 1)|('tokenization', 2)|('lemmatization', 2)|('bag', 1)|('bw', 1)|('representation', 1)|('select', 1)|('relevant', 1)|('information', 1)|('imperative', 1)|('achieve', 1)|('step', 1)|('involve', 1)|('break', 1)|('manageable', 1)|('chunk', 1)|('line', 4)|('paragraph', 1)|('load', 1)|('entire', 2)|('content', 1)|('read', 1)|('iterative', 1)|('reduce', 1)|('strain', 1)|('system', 1)|('resource', 1)|('segment', 1)|('undergo', 1)|('tool', 1)|('space', 1)|('parse', 1)|('extract', 1)|('linguistic', 1)|('feature', 1)|('process', 1)|('token', 1)|('accumulate', 1)|('analyze', 1)|('build', 1)|('vocabulary', 2)|('essentially', 1)|('capture', 1)|('unique', 1)|('scenario', 1)|('size', 1)|('considerable', 1)|('store', 1)|('feasible', 1)|('practical', 1)|('identify', 1)|('n', 1)|('frequent', 1)|('refer', 1)|('high', 1)|('frequency', 1)|('stop', 1)|('base', 1)|('occurrence', 1)|{'essay': 14, 'word': 14, 'text': 6, 'student': 5, 'writing': 4, 'opinion': 4, 'limit': 4, 'write': 4, 'research': 4, 'present': 4, 'line': 4, 'sun': 3, 'like': 3, 'form': 3, 'proper': 3, 'file': 3, 'sky': 2, 'perform': 2, 'family': 2, 'water': 2, 'moment': 2, 'approach': 2, 'assign': 2, 'express': 2, 'usually': 2, 'give': 2, 'require': 2, 'possible': 2, 'think': 2, 'crucial': 2, 'skill': 2, 'efficiently': 2, 'memory': 2, 'processing': 2, 'tokenization': 2, 'lemmatization': 2, 'entire': 2, 'vocabulary': 2, 'walk': 1, 'teh': 1, 'start': 1, 'cup': 1, 'coffee': 1, 'hand': 1, 'feel': 1, 'warm': 1, 'face': 1, 'hang': 1, 'low': 1, 'cast': 1, 'long': 1, 'shadow': 1, 'tranquil': 1, 'meadow': 1, 'gentle': 1, 'breeze': 1, 'whisper': 1, 'tall': 1, 'grass': 1, 'cause': 1, 'sway': 1, 'rhythmically': 1, 'sea': 1, 'green': 1, 'wave': 1, 'air': 1, 'fill': 1, 'sweet': 1, 'fragrance': 1, 'wildflower': 1, 'surrounding': 1, 'nature': 1, 'delicate': 1, 'scent': 1, 'bird': 1, 'charge': 1, 'melodiously': 1, 'branch': 1, 'nearby': 1, 'tree': 1, 'cheerful': 1, 'tune': 1, 'add': 1, 'symphony': 1, 'afternoon': 1, 'distance': 1, 'river': 1, 'render': 1, 'landscape': 1, 'crystal': 1, 'clear': 1, 'shimmer': 1, 'golden': 1, 'light': 1, 'set': 1, 'deer': 1, 'graze': 1, 'peacefully': 1, 'edge': 1, 'graceful': 1, 'movement': 1, 'testament': 1, 'tranquility': 1, 'scene': 1, 'world': 1, 'slow': 1, 'stretch': 1, 'eternity': 1, 'time': 1, 'pause': 1, 'favor': 1, 'beauty': 1, 'evening': 1, 'blame': 1, 'vibrant': 1, 'hue': 1, 'orange': 1, 'pink': 1, 'paint': 1, 'breathtaking': 1, 'masterpiece': 1, 'heaven': 1, 'star': 1, 'kind': 1, 'assignment': 1, 'school': 1, 'college': 1, 'piece': 1, 'freedom': 1, 'thing': 1, 'pupil': 1, 'doubt': 1, 'let': 1, 'tell': 1, 'follow': 1, 'format': 1, 'introduction': 1, 'body': 1, 'conclusion': 1, 'point': 1, 'care': 1, 'include': 1, 'planning': 1, 'execution': 1, 'common': 1, 'help': 1, 'quantify': 1, 'knowledge': 1, 'easy': 1, 'case': 1, 'get': 1, 'tough': 1, 'know': 1, 'argument': 1, 'evidence': 1, 'short': 1, 'need': 1, 'thought': 1, 'idea': 1, 'condense': 1, 'manner': 1, 'type': 1, 'plan': 1, 'deep': 1, 'topic': 1, 'vaguely': 1, 'precision': 1, 'order': 1, 'handle': 1, 'large': 1, 'implement': 1, 'strategy': 1, 'minimize': 1, 'usage': 1, 'sacrifice': 1, 'integrity': 1, 'datum': 1, 'deal': 1, 'task': 1, 'bag': 1, 'bw': 1, 'representation': 1, 'select': 1, 'relevant': 1, 'information': 1, 'imperative': 1, 'achieve': 1, 'step': 1, 'involve': 1, 'break': 1, 'manageable': 1, 'chunk': 1, 'paragraph': 1, 'load': 1, 'content': 1, 'read': 1, 'iterative': 1, 'reduce': 1, 'strain': 1, 'system': 1, 'resource': 1, 'segment': 1, 'undergo': 1, 'tool': 1, 'space': 1, 'parse': 1, 'extract': 1, 'linguistic': 1, 'feature': 1, 'process': 1, 'token': 1, 'accumulate': 1, 'analyze': 1, 'build': 1, 'essentially': 1, 'capture': 1, 'unique': 1, 'scenario': 1, 'size': 1, 'considerable': 1, 'store': 1, 'feasible': 1, 'practical': 1, 'identify': 1, 'n': 1, 'frequent': 1, 'refer': 1, 'high': 1, 'frequency': 1, 'stop': 1, 'base': 1, 'occurrence': 1}\n",
      "50\n",
      "{'essay': 14, 'word': 14, 'text': 6, 'student': 5, 'writing': 4, 'opinion': 4, 'limit': 4, 'write': 4, 'research': 4, 'present': 4, 'line': 4, 'sun': 3, 'like': 3, 'form': 3, 'proper': 3, 'file': 3, 'sky': 2, 'perform': 2, 'family': 2, 'water': 2, 'moment': 2, 'approach': 2, 'assign': 2, 'express': 2, 'usually': 2, 'give': 2, 'require': 2, 'possible': 2, 'think': 2, 'crucial': 2, 'skill': 2, 'efficiently': 2, 'memory': 2, 'processing': 2, 'tokenization': 2, 'lemmatization': 2, 'entire': 2, 'vocabulary': 2, 'walk': 1, 'teh': 1, 'start': 1, 'cup': 1, 'coffee': 1, 'hand': 1, 'feel': 1, 'warm': 1, 'face': 1, 'hang': 1, 'low': 1, 'cast': 1}\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import spacy\n",
    "nlp=spacy.load(\"en_core_web_sm\")\n",
    "from autocorrect import Speller\n",
    "spell = Speller()\n",
    "import nltk\n",
    "from nltk import sent_tokenize\n",
    "path='D:\\\\University\\\\Semester 5\\\\NLP\\\\DP Learning\\\\Text_Files' \n",
    "filepath=os.listdir(path)\n",
    "\n",
    "D_F=[]\n",
    "def filetext (filename):\n",
    "    for file in filename:\n",
    "        text=open(path+\"\\\\\"+file,\"r\").read() \n",
    "        to_store_sentences=[]\n",
    "        sent_=sent_tokenize(text)\n",
    "        for sentence in sent_:\n",
    "            to_store_sentences.append(sentence)\n",
    "        index=[text,type(text),to_store_sentences] \n",
    "        D_F.append(index)\n",
    "\n",
    "filetext(filepath)\n",
    "\n",
    "\n",
    "D_S_P=[]\n",
    "for sentence in D_F:\n",
    "    D_S_P.append(sentence[2])\n",
    "\n",
    "def PreProcessingspacy(text):\n",
    "    text=spell(text)\n",
    "    token=[]\n",
    "    tokens=nlp(text)\n",
    "    token=[tokens]\n",
    "    pos=[]\n",
    "    for w in tokens:\n",
    "        pos.append(w.text)\n",
    "        pos.append(w.pos_)\n",
    "    token_without_stopwords = [w.text for w in tokens if not  w.is_stop]    \n",
    "    token_in_lowercase = [w.lower() for w in token_without_stopwords]\n",
    "    lemmas = [w.lemma_ for w in tokens if not (w.is_stop or not w.is_alpha)]\n",
    "    lemmas=[w.lower() for w in lemmas]\n",
    "    \n",
    "    return [token,pos,token_without_stopwords,token_in_lowercase,lemmas]\n",
    "\n",
    "i=0\n",
    "j=0\n",
    "for file in D_S_P:\n",
    "    for sentence in file:\n",
    "        data=PreProcessingspacy(sentence)\n",
    "        sentence=[sentence,data]   \n",
    "        D_S_P[i][j]=sentence\n",
    "        j=j+1\n",
    "    j=0\n",
    "    i=i+1\n",
    "\n",
    "\n",
    "import pickle\n",
    "save1=open(\"PreProcessed_Data.pickle\",\"wb\")\n",
    "pickle.dump(D_S_P,save1)\n",
    "save1.close()\n",
    "\n",
    "\n",
    "import pickle\n",
    "file3=open(\"PreProcessed_Data.pickle\",\"rb\")\n",
    "D_S_P=pickle.load(file3)\n",
    "\n",
    "BOW=[]\n",
    "for index in range(0,len(D_S_P)):\n",
    "    for index1 in range(0,len(D_S_P[index])):\n",
    "        lemmas=D_S_P[index][index1][-1][-1]\n",
    "        for l in lemmas:\n",
    "            BOW.append(l)\n",
    "\n",
    "print(BOW)\n",
    "\n",
    "Freuency={}\n",
    "for word in BOW:\n",
    "    if word in Freuency:\n",
    "        Freuency[word]=Freuency[word]+1\n",
    "    else:\n",
    "        Freuency[word]=1\n",
    "\n",
    "\n",
    "listed=list(Freuency.items())\n",
    "\n",
    "def insertionsort(Freuency):\n",
    "    for i in range(1, len(Freuency)):\n",
    "        valuetochange=Freuency[i]\n",
    "        print(valuetochange,end=\"|\")\n",
    "        a=i-1\n",
    "        while a >= 0 and Freuency[a][1] < valuetochange[1]:\n",
    "            Freuency[a+1]=Freuency[a]\n",
    "            a=a-1\n",
    "        Freuency[a+1]=valuetochange\n",
    "\n",
    "insertionsort(listed)\n",
    "sortedd = dict(listed)\n",
    "\n",
    "print(sortedd)\n",
    "top50=[listed[i] for i in range(0,50)]\n",
    "top50sortedd=dict(top50)\n",
    "print(len(top50sortedd))\n",
    "print(top50sortedd)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}