Parser.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# This code was used to parse multiple datasets in the Text Age project -\n",
    "\n",
    "The process is split into multiple centuries and a few other opensource datasets are added into creating the final code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['1508_Richard-rolle_The-remedy-against-the-troubles-of-temptations_book.xml', '1508_William-Dunbar_The-ballade-of-ane-right-noble-victorius_book.xml', '1511_Henry-watson_The-noble-history-of-King-Ponthus_book.xml', '1530_Edward-Gosynhyll_dialogue-betwene-the-comen-secretary-and-Ielowsy-touchynge-the-vnstablenes-of-harlottes_letter.xml', '1530_Richard-ullerston_A-compendious-olde-treatyse_book.xml', '1531_Elyyot-Thomas_The-boke-named-The-Gouernour_book.xml', '1532_Hervet-gentian_Xenophons-treatise-of-ousholde_treaty.xml', '1535_George-joye_A-compendyouse-somme-of-the-very-Christen_book.xml', '1535_Joye-George_An-apolgye-to-w.-Tindale_letter.xml', '1542_Heinrich-cornelius_A-treatise-of-the-nobilitie-and-excellencye-of-vvoman-kynde_treaty.xml', '1549_William-Baldwin_The-canticles-or-balades-of-Salomon_book.xml', '1550_Richard-sherry_A-treatise-of-schemes-&-tropes_book.xml', '1550_unknown_Pyers-plowmans-exhortation_book.xml', '1551_Henry-wingfield_A-compendious-or-short-treatise_treaty.xml', '1551_thomas-more_Utopia_book.xml', '1552_William-Elderton_A-decree-betwene-Churchyarde-and-Camell-quod-W-Jlderton_book.xml', '1553_Thomas-wilson_The-arte-of-rhetorique_book.xml', '1561_james-pilkington_The-true-report-of-the-burnyng-of-the-steple-and-church-of-Poules-in-London_report.xml', '1565_Giovanni-boccaccio_A-pleasant-and-delightfull-history-of-Galesus-Cymon-and-Iphigenia _book.xml', '1565_Titus-maccius-platus_A-new-enterlued-for-chyldren-to-playe_book.xml', '1571_Edward-grant-plutarch_A-president-for-parentes_book.xml', '1571_Giovanni-boccaccio_Thirtene-most-plesant-and-delectable-questions_book.xml', '1574_Samuel-william_A-new-balade-or-songe-of-the-Lambes-feast_book.xml', '1575_unknown_Gammer_Gvrtone-nedle_book.xml', '1577_John-pits_A-prayer-for-queen-Elizabeth_prayer.xml', '1577_Richard-robinson_selected-histories-for-christian-recreation_song.xml', '1578_John-rogers_The-summe-of-Christianitie-reduced-vnto-eight-propositions_book.xml', '1579_john-rogers_An-answere-vnto-a-wicked-&-infamous-libel-made-by-Christopher-Vitel_book.xml', '1579_thomas-Plutarch_The-lives-of-the-noble-grecians_book.xml', '1582_unknown_bible_book.xml', '1584_Hakluyt richard_Discourse-of-Western-Planting_book.xml', '1584_John-Rainolds_A-sermon-vpon-part-of-the-prophesie-of-Obadiah_book.xml', \"1587_Marlowe-chrishtopher_Ovid's-elegies_book.xml\", '1589_George-puttenham_The-arte-of-English-poesie_book.xml', '1589_Leonard-wright_A-display-of-dutie-dect-vvith-sage-sayings_book.xml', '1591_Michael-drayton_The-harmonie-of-the-church_songs.xml', '1592_Kyd Thomas_The-Spanish-tragedie_book.xml', '1592_Robert-parsons_aduertisement-written-to-secretarie-L.-Treasurers-of-Ingland_book.xml', '1592_Unknown_Arden-of-Feversham_book.xml', '1594_Michel de-renichon_The-confession-of-Michael-Renichon-of-Templeu_book.xml', '1594_Richard-Barnfield_The-affectionate-shepheard-Containing-the-complaint-of-Daphnis-for-the-loue-of-Ganymede_book.xml', '1595_Thomas-edwards_Cephalus-&-Procris-Narcissus_book.xml', '1596_Sabie_francis_Adams-complaint-The-olde-vvorldes-tragedie-Dauid-and-Bathsheba_book.xml', '1596_unknown_The-raigne-of-King-Edvvard-the-Third_book.xml', '1596_unknown_The-Spanish-tragedie_book.xml', '1597_Nicholas-yonge_Musica-transalpina-Cantus_book.xml', '1598_Richard-stapleton_Phillis-and-Flora-The-sweete_book.xml', '1598_Samuel-rowlands_The-betraying-of-Christ-Iudas-in-despaire_books.xml', '1599_Henry-chettle_The-tragedy-of-Hoffman_book.xml', \"1599_Thomas-dekker_The-shoemakers'-holiday_book.xml\", '1601_Robert-yarington_two-lamentable-tragedies_report.xml', '1603_Thomas-p_the-power-of_sermon.xml', '1604_Thomas-dekker_the-honest-whore_dialogue.xml', '1605_Ben-jonson_Sejanus_Drama.xml', '1605_Thomas-p_sick-mans-couch_sermon.xml', '1606_Ben-Jonson_Hymenaei_report.xml', '1606_Cyril-Toruneur_The-revengers-tragedy_novel.xml', '1609_John-skelton_Pimlyco_report.xml', '1609_Thomas-Dekker_the-guls-horne-brooke_novel.xml', '1611_Thomas-draxe_the-christian-armorie_religious.xml', '1613_George-w_Epithalamia_poem.xml', '1617_willbald_the-praise-of-the-gout_novel.xml', '1618_Thomas-dekker_the-owels_report.xml', '1621_Unknown_the-true-medium-of-the-monies_report.xml', '1623_John-webster_The-duchess-of-mafia_novel.xml', '1623_William-shakespeare_henry-the-fourth_drama.xml', '1623_William-shakespeare_the-taming-of-the-shrew_drama.xml', '1623_William-shakespeare_The-tragedie_Drama.xml', '1637_John-milton_Comus_drama.xml', '1639_Samuel-pick_Festum-voluptatis_poems.xml', '1641_John-williams_an-abstract-of-answers_report.xml', '1642_John-hales_a-tract_report.xml', '1642_Lazarus-haward_a-continvation-of-the-divrnal_report.xml', '1642_lionel-copley_a-letter-from-a-gentleman_letter.xml', '1642_Parliament_the-state-of-case_history.xml', '1644_John-milton_Areopagitica_speech.xml', '1645_John-Win_a-declaration-of-former-passages_report.xml', '1645_William-Prynne_hidden-works_novel.xml', '1647_Jasper-mayne_sermon-against-false-prophets_sermon.xml', '1649_John-cook_king-charles-case_report.xml', '1651_Augustine_the-profit_novel.xml', '1652_Jasper-M_a-sermon_sermon.xml', '1652_John-Aufley_liberties-of-people_report.xml', '1658_Thomas_Browne_hydriotaphia-Urn-birial_letter.xml', '1660_Edward-cooke_concernment-in-ireland_report.xml', '1661_HF_a-plain-record_report.xml', '1661_Unknown_Catalogue-of-the-prelates_report.xml', \"1668_John-Dryden_the-feign'd-innocence_drama.xml\", '1674_John-milton_Paradise-lost_poem.xml', '1675_Rice-Vaughan_discourse-of-coin_discourse.xml', '1675_William-Wycherley_the-country-wife_comedy.xml', '1676_Augustine_digitus-dei_novel.xml', '1677_John-Dryden_the-state-of-innocence_drama.xml', '1678_Edward-cooke_Love-triumph_Drama.xml', '1680_Aulonoy_novels-of-elizabeth_novel.xml', '1680_John-d_three-poems_poems.xml', '1682_George-b_the-tryal_report.xml', '1682_John-Bunyan_The-holy-war_history.xml', '1683_William-atwood_three-letters_letter.xml', '1684_John-a_aarraignment-of-rebellion_letter.xml', '1684_Robert-F_third-diallogue_drame.xml', '1685_John-dryden_Sylvae_poem.xml', '1685_Shadrach-Cooke_preached-at-islaington_sermon.xml', '1687_Edmund-elys_an-epistle-to-the-truly-religious_letter.xml', '1687_James-II_A-proclamation_report.xml', '1687_Willian-penn_Advice-to-freeholders_letter.xml', '1688_chaise_letter-from-father_letter.xml', '1688_James-II_by-the-king_report.xml', '1690_gentleman-in-dort_letter-to-commons_letter.xml', '1690_Gentlemen-in-the-city_the-trade-in-africa_report.xml', '1690_Thomas-Browne_Letter-to-a-friend_letter.xml', '1690_Unknown_the-controversie-between-robin_poem.xml', '1694_Francis-Atterbury_the-christian-religion_sermon.xml', '1695_Edmund-Elys_a-letter-to-author_letter.xml', '1695_William-atwood_supplying-want-of-coin_report.xml', '1696_edmund-elys_a-letter-sir-robert_letter.xml', '1696_shadrach-crook_Christian-sermon_sermon.xml', '1698_John-Cary_An-answer-to_letter.xml', '1698_John-hopkins_the-victory-of-death_poem.xml', '1699_Robert-atkyns_a-treatise-of-jurisdiction_report.xml', '1700_aesop-benjamin-harris_the-fables-of-young-aesop-with-their-morals-with-a-moral-history-of-his-life-and-death-illustrated-with-forty-curious-cuts-applicable-to-each-fable_fables.xml', '1700_daniel-defoe_the-pacificator-a-poem_poem.xml', \"1700_john-cosin_bishop-cozens's-argument-proving-that-adultery-woks-a-dissolution-of-the-marriage-being-the-substance-of-several-of-bishob-cozens-his-speeches-in-the-house-of-lords-upon-the-debate-of-lord-ross's-case-taken-from-original-papers_speech.xml\", '1700_r-b_the-english_history.xml', '1700_siculus-diodorus-george-booth-henride-valois-lorenz-rhodoman_the-historical-library-of-diodorus-the-sicilian-in-first-five-contain-the-antiquities-of-egypt-asia-africa-greece-the-islands-and-europe-the-last-ten-an-historical-account-of_history.xml', \"1700_thomas-tryon_tryon's-letters-upon-several-occassions-by-tho-tryon_letter.xml\", '1702_john-danforth_the-right-christian-temper-in-every-condition-endeavoured-as-the-lord-vouchsafed-to-assist-to-be-set-forth-and-recommended-and-the-due-debt-of-a-suitable-contentment-with-our-outward-lot-humbly-demanded-on-the-behalf-of-the_poem.xml', '1702_unknown-author_the-loyal-address-of-the-clergy-of-virginia_poem.xml', '1703_cotton-mather_agreeable-admonitions-for-old-&-young-or-aged-piety-honoured-and-early-piety-quickened-in-a-brief-disclosure-delivered-april-25-1703_poem.xml', '1704_cotton-mather_eureka-the-vertuous-woman-found-a-short-essay-on-the-memory-of-mrs-mary-brown-late-consort-of-benjamin-brown-esq-in-salem-three-lines-from-ruth-iii_poems.xml', '1706_daniel-defoe_caledonia-&c-a-poem-in-honour-of-scotland-and-the-scots-nation-in-three-parts_poem.xml', '1707_cotton-mather_ornamental-piety_poem.xml', '1707_nicholas-noyes_may-28th-1706-to-my-worthy-friend-mr-james-bayley-living-if-living-in-roxbury-a-poem_poem.xml', \"1712_peter-thacher-john-danforth_christ's-forgiveness-of-true-christians-is-a-preceptive-patern-of-christian-fraternal-forgiveness-a-sermon-on-col-iii-13-preached-upon-a-solemn-occasion-by-peter-thacher-a-m-pastor-of-the-church-of-christ-in_poem.xml\", '1713_samuel-sewall_proposals-touching-the-accomplishment-of-prophecies-humbly-offered-by-samuel-sewall-m-a-and-sometime-fellow-of-harvard-college-at-cambridge-in-new-england-eight-lines-of-quotations_poem.xml', '1714_john-barnard_two-sermons-the-christians-behaviour-under-severe-and-repeated-bereavements-and-the-fatal-consequence-of-a-peoples-persisting-in-sin_sermon.xml', '1714_robert-hunter_androboros-a-bographical-sic-farce-in-three-acts-viz-the-senate-the-consistory-and-the-apotheosis_play.xml', '1715_nicholas-rowe_the-tragedy-of-the-lady-jane-gray-as-it-is-acted-at-the-theatre-royal-in-drury-lane-by-n-rowe-esq_play.xml', '1721_daniel-defoe_the-history-and-misfortunes-of-the-famous-moll-flanders-etc_fiction.xml', '1738_mather-byles-jonathon-belcher-dedicatee_on-the-death-of-the-queen-a-poem-inscribed-to-his-excellency-governour-belcher-by-the-reverned-mr-byles_poem.xml', '1742_henry-fielding_the-history-of-the-adventures-of-joseph-andrews-and-his-friend-mr-abraham-adams_novel.xml', '1746_thomas-prince_the-salvations-of-god-in-1746.-in-part-set-forth-in-a-sermon-at-the-south-church-in-boston,-nov.27.1746.-being-the-day-of-the-anniversary-thanksgiving-in-the-province-of-the-massachusetts-bay-in-n.e.-wherein-the-most-remark_history.xml', '1747_jonathon-edwards_true-saints,-when-absent-from-the-body,-are-present-with-the-lord_sermon.xml', '1748_w-william-kernick_the-town-a-satire-by-w-kernick_verses.xml', '1749_henry-fielding_the-history-of-tom-jones-a-foundling_novel.xml', '1749_john-cleland_memoirs-of-a-woman-of-pleasure_fiction.xml', '1749_thomas-warton_the-triump-of-isis-a-poem-occasioned-by-isis-an-elegy_poem.xml', '1750_jonathan-mayhew_a-disclosure-concerning-unlimited-submission-and-non-resistance-to-the-higher-powers_sermon.xml', '1762_anthony-benezet_a-short-account-of-that-part-of-africa--habited-by-the-negroes-with-respect-to-the-fertility-of-the-country-the-good-disposition-of-many-of-the-natives-and-the-manner-by-which-the-slave-trade-is-carried-on_travel.xml', \"1762_college-of-new-jersey_the-military-glory-of-great-britain-an-entertainment-given-by-the-late-candidates-for-bachelor's-degree-at-the-close-of-the-anniversary-commencement-held-in-nassau-hall-new-jersey-september-29th-1762_play.xml\", \"1767_benjamin-franklin_father-abraham's-speech-to-a-great-number-of-people-at-a-vendue-of-merchant-goods-introduced-to-the-public-by-poor-richard-a-famous-conjurer-and-almanack-maker-in-answer-to-the-following-questions-pray-father-abraham_speech.xml\", '1768_john-dickinson_a-copy-of-a-letter-from-a-gentleman-in-virginia-to-a-merchant-in-philadelphia_letter.xml', '1768_john-dickinson_a0-copy-of-a-letter-from-a-gentleman-in virginia,-to-a-merchant-in-philadelphia-_letter.xml', '1768_john-dickinson_letters-from-a-farmer-in-pennsylvania,-to-the-inhabitants-of the-british-colonies_letter.xml', '1768_naphtali-daggett_the-great-importance-of-speaking-in-the-most-intelligible-manner-in-the-christian-church-a-sermon-preached-at-the-installatiion-of-the-rev-nathaniel-sherman-over-the-church-of-the-christ-in-mount-carmel-new-heaven_sermon.xml', '1768_sterne-laurence_a-sentimental-journey-through-france-and-italy_novel.xml', '1768_thomas-barnard_the-power-of-god-the-proof-of-christianity-a-disclosure-delivered-at-the-dudleian-lecture-in-the-chapel-of-harvard-college-cambridge-n-e-may-11th-1768-by-thomas-barnard-a-m-pastor-of-the-first-church-in-salem_sermon.xml', '1772_richard-cumberland-william-shakespear_timon-of-athens-altered-from-shakespear-a-tragedy-as-it-is-acted-at-the-theatre-royal-in-druny-lane_play.xml', '1775_mercy-otis-warren_the-group-a-farce-as-lately-acted-and-to-be-re-acted-to-the-wonder-of-all-superior-intelligences-nigh-head-quarters-at-amboyne-in-two-acts_play.xml', \"1776_thomas-prince_extract-of-a-sermon-preach'd-at-the-south-church-in-boston-november-27-1746-by-the-rev-mr-thomas-prince-occasioned-by-the-surprizing-appearance-of-divine-providence-for-north-america,-in-the-destruction-of-the-french-fleet_history.xml\", \"1782_thomas-warton_verses-on-sir-joshua-reynolds's-painted-window-at-new-college-oxford_verses.xml\", '1784_peter-markoe_the-patriot-chief-a-tragedy-five-lines-from-thomson_play.xml', '1786_charles-hutton_tracts-mathematical-and-philosophical-by-charles-hutton-vol-1_fiction.xml', '1787_stephen-crisp_scripture-truth-demonstrated-in-thirty-two-sermons-or-declaration-of-stephen-crisp-late-of-colchester-in-essex-deceased-carefully-taken-in-short-hand-as-they-are-delivered-by-him-at-the-public-meeting-houses-of-the-people_gospel.xml', '1792_willium-rufus-chetwood-daniel-defoe-benjamin-victor-sir-d-william-yonge_the-voyages-and-adventures-of-captain-robert-boyle-in-several-parts-of-the-world-intermixed-with-the-story-of-mrs-villars-an-english-lady-with-whom-he-made-his_travel.xml', '1793_american-lady_the-hapless-orphan-or-innocent-victim-of-revenge-a-novel-founded-on-incidents-in-real-life-in-a-series-of-letters-from-caroline-francis-to-maria-b-in-two-volumes_novel.xml', '1793_mrs-rowson-cockburne-lady-dedicatee_the-inquisitor-or-invisible-rambler-in-three-volumes_novel.xml', '1794_mrs-rowson_charlotte-a-tale-of-truth-by-mrs-rowson-of-the-new-theatre-philadelphia-author-of-victoria-the-inquisitor-fille-de-chambre-&c_novel.xml', \"1794_mrs-rowson_mentoria-or-the-young-lady's-friend-in-two-volumes-by-mrs-rowson-of-the-new-theatre-philadelphia-author-of-the-inquisior-file-de-chambre-victoria-charlotte-&c-&c_novel.xml\", '1795_james-hardie_the-american-remembrance-and-universal-tablet-of-memory-containing-a-list-of-the-most-eminent-men-whether-in-ancient-or-modern-times-with-the-achievements-for-which-they-have-been-particularly-distinguished-as-also-the-most_history.xml', '1795_richard-lee_the-melancholy-end-of-ungrateful-children-exemplified-in-the-dreadful-fate-of-the-son-and-daughter-of-a-wealthy-farmer-who-after-receiving-and-dividing-the-wealth-of-their-parents-refused-them-in-their-old-age-the-shhelter-of_chapbook.xml', '1796_edmund-burke_thoughts-on-the-prospect-of-a-regicide-peace-in-a-series-of-letters_letter.xml', '1796_hannah-more_the-way-to-plenty-or-the-second-part-of-tom-white_chapbook.xml', '1796_jeremy-belknap-john-arbuthnot_the-foresters-an-american-tale-being-a-sequel-to-the-history-of-john-bull-the-clothier-in-a-series-of-letters-to-a-friend-published-according-to-act-of-congress_novel.xml', '1797_dannie-joseph-rogers-ransford_a-collection-of-essays-on-a-variety-of-subjects-in-prose-and-verse_essay.xml', '1797_hannah-webster-foster_the-coquette-or-the-history-of-eliza-wharton-a-novel-founded-on-fact-by-a-lady-of-massachusetts_novel.xml', '1797_samuel-relf-ann-louisa-bingham-dedicatee_infidelity-or-the-victims-of-sentiment-a-novel-in-a-series-of-letters_novel.xml', '1798_charles-holt_a-short-account-of-the-yellow-fever-as-it-appeared-in-new-london-in-august-september-and-october-1798-with-an-accurate-list-of-who-died-of-the-disease-the-donations_nonfiction.xml', '1798_john-mariner-willock-thomas-engraver-clarke_the-voyages-and-adventures-of-john-willock-mariner-interspersed-with-remarks-on-different-countries-in-europe-africa-and-america-with-the-customs-and-manners-of-the-inhabitants-and-a-number_travel.xml', '1798_thomas-bellamy_the-history-of-capt-thomas-parismas-containing-a-particular-account-of-the-cruel-and-barbarous-treatment-of-a-young-lady-who-was-the-wife-of-mr-james-negotio-an-english-merchant-in-the-east-indies-the-history-of-this_story.xml']\n"
     ]
    }
   ],
   "source": [
    "# Text age parser\n",
    "from bs4 import BeautifulSoup\n",
    "#import datasets\n",
    "from datetime import datetime\n",
    "import os\n",
    "\n",
    "dt_format = \"%Y\"\n",
    "base_path = r\"C:\\Users\\rhira\\Documents\\TextAge\\17001800\\dataset\"\n",
    "files = os.listdir(base_path)\n",
    "print(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame(columns=[\"text\",\"category\",\"date\",\"head\",\"title\",\"author\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\rhira\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Downloading data: 100%|██████████| 3.06M/3.06M [00:01<00:00, 2.39MB/s]\n",
      "Generating train split: 100%|██████████| 120/120 [00:00<00:00, 611.85 examples/s]\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "dataset = load_dataset(\"biglam/lampeter_corpus\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'text', 'date', 'genre', 'head', 'title'],\n",
       "        num_rows: 120\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>text</th>\n",
       "      <th>date</th>\n",
       "      <th>genre</th>\n",
       "      <th>head</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>EcA1641</td>\n",
       "      <td>To contend with men that deny their owne publi...</td>\n",
       "      <td>1641-01-01</td>\n",
       "      <td>Economy</td>\n",
       "      <td>A Reply to a most untrue Relation made, and se...</td>\n",
       "      <td>A reply to a most untrue relation [...] by cer...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>EcA1652</td>\n",
       "      <td>IT hath been a thing for many years generally ...</td>\n",
       "      <td>1652-01-01</td>\n",
       "      <td>Economy</td>\n",
       "      <td>THE ADVOCATE: OR, A NARRATIVE Of the state and...</td>\n",
       "      <td>The Advocate.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>EcA1668</td>\n",
       "      <td>I Presume, there are few Englishmen so disloya...</td>\n",
       "      <td>1668-01-01</td>\n",
       "      <td>Economy</td>\n",
       "      <td>\\nPropos. 1.\\n\\nIt will supply his Majesties p...</td>\n",
       "      <td>A discourse ,shewing the many advantages which...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>EcA1676</td>\n",
       "      <td>HAving lately seen a Pamphlet under the Title ...</td>\n",
       "      <td>1676-01-01</td>\n",
       "      <td>Economy</td>\n",
       "      <td>An Enquiry into the Grounds of a late Pamphlet...</td>\n",
       "      <td>Is not the hand of Joab in all this? Or An enq...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>EcA1681</td>\n",
       "      <td>FOrasmuch as there is a very great Complaint i...</td>\n",
       "      <td>1681-01-01</td>\n",
       "      <td>Economy</td>\n",
       "      <td>SECT. I. \\nThe Introduction.</td>\n",
       "      <td>The Trade of England Revived: And the Abuses t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                               text        date  \\\n",
       "0  EcA1641  To contend with men that deny their owne publi...  1641-01-01   \n",
       "1  EcA1652  IT hath been a thing for many years generally ...  1652-01-01   \n",
       "2  EcA1668  I Presume, there are few Englishmen so disloya...  1668-01-01   \n",
       "3  EcA1676  HAving lately seen a Pamphlet under the Title ...  1676-01-01   \n",
       "4  EcA1681  FOrasmuch as there is a very great Complaint i...  1681-01-01   \n",
       "\n",
       "     genre                                               head  \\\n",
       "0  Economy  A Reply to a most untrue Relation made, and se...   \n",
       "1  Economy  THE ADVOCATE: OR, A NARRATIVE Of the state and...   \n",
       "2  Economy  \\nPropos. 1.\\n\\nIt will supply his Majesties p...   \n",
       "3  Economy  An Enquiry into the Grounds of a late Pamphlet...   \n",
       "4  Economy                       SECT. I. \\nThe Introduction.   \n",
       "\n",
       "                                               title  \n",
       "0  A reply to a most untrue relation [...] by cer...  \n",
       "1                                      The Advocate.  \n",
       "2  A discourse ,shewing the many advantages which...  \n",
       "3  Is not the hand of Joab in all this? Or An enq...  \n",
       "4  The Trade of England Revived: And the Abuses t...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_pandas = dataset[\"train\"].to_pandas()\n",
    "df_pandas.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_pandas.to_csv(\"lampenter.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import xml.etree.ElementTree as ET\n",
    "import pandas as pd\n",
    "\n",
    "def extract_data(xml_file):\n",
    "    tree = ET.parse(xml_file)\n",
    "    root = tree.getroot()\n",
    "\n",
    "    title = root.find(\".//title\").text\n",
    "    author = root.find(\".//author\").text\n",
    "\n",
    "    # Extract 4-digit year from speechPubDate within the range 1500 to 1900\n",
    "    raw_speech_pub_date = root.find(\".//speechPubDate\").text\n",
    "    match = re.search(r'\\b(15\\d{2}|16\\d{2}|17\\d{2}|18\\d{2}|19\\d{2})\\b', raw_speech_pub_date)\n",
    "    speech_pub_date = match.group() if match else None\n",
    "\n",
    "    dialogue_elements = root.findall(\".//dialogueText//dialogue\")\n",
    "    non_speech_elements = root.findall(\".//dialogueText//nonSpeech\")\n",
    "\n",
    "    dialogue_content = ' '.join([''.join(dialogue.itertext()).strip() for dialogue in dialogue_elements])\n",
    "    non_speech_content = ' '.join([''.join(non_speech.itertext()).strip() for non_speech in non_speech_elements])\n",
    "\n",
    "    combined_content = dialogue_content + ' ' + non_speech_content\n",
    "\n",
    "    data = {'Title': [title], 'Author': [author], 'SpeechPubDate': [speech_pub_date], 'Content': [combined_content]}\n",
    "    df = pd.DataFrame(data)\n",
    "    return df\n",
    "\n",
    "def process_folder(folder_path):\n",
    "    all_dfs = []\n",
    "\n",
    "    for filename in os.listdir(folder_path):\n",
    "        if filename.endswith(\".xml\"):\n",
    "            xml_file_path = os.path.join(folder_path, filename)\n",
    "            df = extract_data(xml_file_path)\n",
    "            all_dfs.append(df)\n",
    "\n",
    "    if all_dfs:\n",
    "        combined_df = pd.concat(all_dfs, ignore_index=True)\n",
    "        return combined_df\n",
    "    else:\n",
    "        print(\"No XML files found in the folder.\")\n",
    "        return None\n",
    "\n",
    "# Replace 'your_folder_path' with the actual path to your folder containing XML files\n",
    "folder_path = 'english_dialogues'\n",
    "combined_dataframe = process_folder(folder_path)\n",
    "\n",
    "# Display the combined DataFrame\n",
    "if combined_dataframe is not None:\n",
    "    pd.set_option('display.max_colwidth', None)\n",
    "    print(combined_dataframe)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      1599\n",
       "1      1594\n",
       "2      1584\n",
       "3      1595\n",
       "4      1595\n",
       "       ... \n",
       "172    1752\n",
       "173    1729\n",
       "174    1744\n",
       "175    1726\n",
       "176    1725\n",
       "Name: year, Length: 177, dtype: object"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined_dataframe[\"year\"] = combined_dataframe[\"SpeechPubDate\"]\n",
    "combined_dataframe[\"year\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#files = [\"1570_John-Drout_The-pityfull-histori-of-two-louing-Italians_book_book.xml\"]\n",
    "for filepath in files:\n",
    "    with open(base_path+\"/\"+filepath, encoding=\"utf-8\") as f:\n",
    "            \n",
    "            split = filepath.split(\"_\")\n",
    "            date = split[0]\n",
    "            author = split[1]\n",
    "            title = split[2]\n",
    "            category = split[3]\n",
    "\n",
    "            soup = BeautifulSoup(f, features=\"xml\")\n",
    "            for entry in soup.find_all(\"TEI\"):\n",
    "                text_parts = []\n",
    "                #title_with_id = entry.teiHeader.fileDesc.titleStmt.title.text\n",
    "                #date = entry.teiHeader.fileDesc.edition.date\n",
    "                #print(title_with_id, date)\n",
    "                #id, title = title_with_id.split(\":\", maxsplit=1)\n",
    "                #id = id.strip()\n",
    "                #title = title.strip()\n",
    "                #date = datetime.strptime(id[-4:], dt_format)\n",
    "                content = entry.find(\"text\")\n",
    "                #print(content)\n",
    "                head = content.find(\"body\").find(\"head\")\n",
    "                \n",
    "                if head:\n",
    "                    head = head.text\n",
    "                else:\n",
    "                    head = \"\"\n",
    "                body_parts = content.find(\"body\").find_all(\"p\")\n",
    "                if len(str(body_parts)) < 100:\n",
    "                     body_parts = content.find(\"body\").find_all(\"l\")\n",
    "                for body_part in body_parts:\n",
    "                    text_parts.append(body_part.text)\n",
    "                full_text = \" \".join(text_parts)\n",
    "                #print(filepath, len(full_text))\n",
    "\n",
    "                #genre = _CLASS_MAP[id[0]]\n",
    "                data_point = {\n",
    "                    \"text\": full_text,\n",
    "                    \"category\": category,\n",
    "                    \"date\": date,\n",
    "                    \"head\": head,\n",
    "                    \"title\": title,\n",
    "                    \"author\":author\n",
    "                }\n",
    "                df = df.append(data_point, ignore_index=True)\n",
    "                #print(data_point)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "book.xml                                                          38\n",
       "report.xml                                                        21\n",
       "novel.xml                                                         18\n",
       "letter.xml                                                        18\n",
       "poem.xml                                                          16\n",
       "sermon.xml                                                        12\n",
       "history.xml                                                        7\n",
       "play.xml                                                           6\n",
       "drama.xml                                                          5\n",
       "travel.xml                                                         3\n",
       "fiction.xml                                                        3\n",
       "speech.xml                                                         3\n",
       "poems.xml                                                          3\n",
       "Drama.xml                                                          3\n",
       "treaty.xml                                                         3\n",
       "chapbook.xml                                                       2\n",
       "verses.xml                                                         2\n",
       "fables.xml                                                         1\n",
       "nonfiction.xml                                                     1\n",
       "essay.xml                                                          1\n",
       "gospel.xml                                                         1\n",
       "Gvrtone-nedle                                                      1\n",
       "prayer.xml                                                         1\n",
       "drame.xml                                                          1\n",
       "religious.xml                                                      1\n",
       "comedy.xml                                                         1\n",
       "discourse.xml                                                      1\n",
       "hydriotaphia-Urn-birial                                            1\n",
       "song.xml                                                           1\n",
       "songs.xml                                                          1\n",
       "Adams-complaint-The-olde-vvorldes-tragedie-Dauid-and-Bathsheba     1\n",
       "books.xml                                                          1\n",
       "dialogue.xml                                                       1\n",
       "story.xml                                                          1\n",
       "Name: category, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.category.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 180 entries, 0 to 179\n",
      "Data columns (total 6 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   text      180 non-null    object\n",
      " 1   category  180 non-null    object\n",
      " 2   date      180 non-null    object\n",
      " 3   head      180 non-null    object\n",
      " 4   title     180 non-null    object\n",
      " 5   author    180 non-null    object\n",
      "dtypes: object(6)\n",
      "memory usage: 8.6+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lampenter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1641-01-01\n",
       "1    1652-01-01\n",
       "2    1668-01-01\n",
       "3    1676-01-01\n",
       "4    1681-01-01\n",
       "Name: date, dtype: object"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = pd.read_csv(\"lampenter.csv\")\n",
    "x[\"date\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'id', 'text', 'date', 'genre', 'head', 'title'], dtype='object')"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "lst = x[\"date\"].to_list()\n",
    "yr = [int(x.split(\"-\")[0]) for x in lst if x]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "x[\"year\"] = yr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "lampenter_1700 = x[x[\"year\"]>1700]\n",
    "lampenter_1700.to_csv(\"17001800.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Gefa"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'author_tag', 'title', 'subjects', 'subjectcount', 'pd',\n",
       "       'dob', 'dod', 'edp', 'decade', 'century', 'text'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gefa = pd.read_csv(\"GEFA_18001900.csv\")\n",
    "gefa.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>author_tag</th>\n",
       "      <th>title</th>\n",
       "      <th>subjects</th>\n",
       "      <th>subjectcount</th>\n",
       "      <th>pd</th>\n",
       "      <th>dob</th>\n",
       "      <th>dod</th>\n",
       "      <th>edp</th>\n",
       "      <th>decade</th>\n",
       "      <th>century</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>11</td>\n",
       "      <td>MÃ¼hlbachLLuise</td>\n",
       "      <td>Napoleon and BlÃ¼cher: An Historical Novel</td>\n",
       "      <td>Napoleon I, Emperor of the French, 1769-1821 -- Fiction; BlÃ¼cher, Gebhard Leberecht von, 1742-1819 -- Fiction; Germany -- History -- 1806-1815 -- Fiction</td>\n",
       "      <td>3</td>\n",
       "      <td>1814</td>\n",
       "      <td>1814</td>\n",
       "      <td>1873</td>\n",
       "      <td>1843</td>\n",
       "      <td>1840</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13</td>\n",
       "      <td>MÃ¼hlbachLLuise</td>\n",
       "      <td>Louisa of Prussia and Her Times: A Historical Novel</td>\n",
       "      <td>Napoleon I, Emperor of the French, 1769-1821 -- Fiction; Luise, Queen, consort of Frederick William III, King of Prussia, 1776-1810 -- Fiction; Prussia (Germany) -- History -- Frederick William III, 1797-1840 -- Fiction; Napoleonic Wars, 1800-1815 -- Campaigns -- Germany -- Prussia -- Fiction</td>\n",
       "      <td>4</td>\n",
       "      <td>1867</td>\n",
       "      <td>1814</td>\n",
       "      <td>1873</td>\n",
       "      <td>1843</td>\n",
       "      <td>1840</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18</td>\n",
       "      <td>MÃ¼hlbachLLuise</td>\n",
       "      <td>Frederick the Great and His Court</td>\n",
       "      <td>Frederick II, King of Prussia, 1712-1786 -- Fiction; Prussia (Germany) -- History -- Frederick II, 1740-1786 -- Fiction; Courts and courtiers -- Fiction</td>\n",
       "      <td>3</td>\n",
       "      <td>1866</td>\n",
       "      <td>1814</td>\n",
       "      <td>1873</td>\n",
       "      <td>1843</td>\n",
       "      <td>1840</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>93</td>\n",
       "      <td>MÃ¼hlbachLLuise</td>\n",
       "      <td>Andreas Hofer: An Historical Novel</td>\n",
       "      <td>Hofer, Andreas, 1767-1810 -- Fiction; Tyrol (Austria) -- History -- Uprising of 1809 -- Fiction</td>\n",
       "      <td>2</td>\n",
       "      <td>1868</td>\n",
       "      <td>1814</td>\n",
       "      <td>1873</td>\n",
       "      <td>1843</td>\n",
       "      <td>1840</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>125</td>\n",
       "      <td>MÃ¼hlbachLLuise</td>\n",
       "      <td>Marie Antoinette and Her Son</td>\n",
       "      <td>Historical fiction; Marie Antoinette, Queen, consort of Louis XVI, King of France, 1755-1793 -- Fiction; Louis XVII, of France, 1785-1795 -- Fiction</td>\n",
       "      <td>3</td>\n",
       "      <td>1866</td>\n",
       "      <td>1814</td>\n",
       "      <td>1873</td>\n",
       "      <td>1843</td>\n",
       "      <td>1840</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8545</th>\n",
       "      <td>12865</td>\n",
       "      <td>AlexanderMrs</td>\n",
       "      <td>A Crooked Path: A Novel</td>\n",
       "      <td>Inheritance and succession -- Fiction; England -- Social life and customs -- Fiction</td>\n",
       "      <td>2</td>\n",
       "      <td>1889</td>\n",
       "      <td>1825</td>\n",
       "      <td>1902</td>\n",
       "      <td>1863</td>\n",
       "      <td>1860</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8546</th>\n",
       "      <td>12874</td>\n",
       "      <td>AlexanderMrs</td>\n",
       "      <td>Ralph Wilton's weird</td>\n",
       "      <td>Inheritance and succession -- Fiction; Mate selection -- Fiction; Courtship -- Fiction; Social classes -- England -- Fiction</td>\n",
       "      <td>4</td>\n",
       "      <td>1875</td>\n",
       "      <td>1825</td>\n",
       "      <td>1902</td>\n",
       "      <td>1863</td>\n",
       "      <td>1860</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8547</th>\n",
       "      <td>12883</td>\n",
       "      <td>AlexanderMrs</td>\n",
       "      <td>At Bay</td>\n",
       "      <td>Fiction</td>\n",
       "      <td>1</td>\n",
       "      <td>1854</td>\n",
       "      <td>1825</td>\n",
       "      <td>1902</td>\n",
       "      <td>1863</td>\n",
       "      <td>1860</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8548</th>\n",
       "      <td>12888</td>\n",
       "      <td>AlexanderMrs</td>\n",
       "      <td>Women Novelists of Queen Victoria's Reign: A Book of Appreciations</td>\n",
       "      <td>Novelists, English -- 19th century -- Biography; Women and literature -- Great Britain -- History -- 19th century; English fiction -- 19th century -- History and criticism; English fiction -- Women authors -- History and criticism; Women novelists, English -- Biography</td>\n",
       "      <td>5</td>\n",
       "      <td>1897</td>\n",
       "      <td>1825</td>\n",
       "      <td>1902</td>\n",
       "      <td>1863</td>\n",
       "      <td>1860</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8549</th>\n",
       "      <td>13263</td>\n",
       "      <td>AlexanderMrs</td>\n",
       "      <td>Kate Vernon: A Tale. Vol. 2 (of 3)</td>\n",
       "      <td>Fiction</td>\n",
       "      <td>1</td>\n",
       "      <td>1854</td>\n",
       "      <td>1825</td>\n",
       "      <td>1902</td>\n",
       "      <td>1863</td>\n",
       "      <td>1860</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8550 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Unnamed: 0       author_tag  \\\n",
       "0             11  MÃ¼hlbachLLuise   \n",
       "1             13  MÃ¼hlbachLLuise   \n",
       "2             18  MÃ¼hlbachLLuise   \n",
       "3             93  MÃ¼hlbachLLuise   \n",
       "4            125  MÃ¼hlbachLLuise   \n",
       "...          ...              ...   \n",
       "8545       12865     AlexanderMrs   \n",
       "8546       12874     AlexanderMrs   \n",
       "8547       12883     AlexanderMrs   \n",
       "8548       12888     AlexanderMrs   \n",
       "8549       13263     AlexanderMrs   \n",
       "\n",
       "                                                                   title  \\\n",
       "0                             Napoleon and BlÃ¼cher: An Historical Novel   \n",
       "1                    Louisa of Prussia and Her Times: A Historical Novel   \n",
       "2                                      Frederick the Great and His Court   \n",
       "3                                     Andreas Hofer: An Historical Novel   \n",
       "4                                           Marie Antoinette and Her Son   \n",
       "...                                                                  ...   \n",
       "8545                                             A Crooked Path: A Novel   \n",
       "8546                                                Ralph Wilton's weird   \n",
       "8547                                                              At Bay   \n",
       "8548  Women Novelists of Queen Victoria's Reign: A Book of Appreciations   \n",
       "8549                                  Kate Vernon: A Tale. Vol. 2 (of 3)   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                   subjects  \\\n",
       "0                                                                                                                                                Napoleon I, Emperor of the French, 1769-1821 -- Fiction; BlÃ¼cher, Gebhard Leberecht von, 1742-1819 -- Fiction; Germany -- History -- 1806-1815 -- Fiction   \n",
       "1     Napoleon I, Emperor of the French, 1769-1821 -- Fiction; Luise, Queen, consort of Frederick William III, King of Prussia, 1776-1810 -- Fiction; Prussia (Germany) -- History -- Frederick William III, 1797-1840 -- Fiction; Napoleonic Wars, 1800-1815 -- Campaigns -- Germany -- Prussia -- Fiction   \n",
       "2                                                                                                                                                  Frederick II, King of Prussia, 1712-1786 -- Fiction; Prussia (Germany) -- History -- Frederick II, 1740-1786 -- Fiction; Courts and courtiers -- Fiction   \n",
       "3                                                                                                                                                                                                           Hofer, Andreas, 1767-1810 -- Fiction; Tyrol (Austria) -- History -- Uprising of 1809 -- Fiction   \n",
       "4                                                                                                                                                      Historical fiction; Marie Antoinette, Queen, consort of Louis XVI, King of France, 1755-1793 -- Fiction; Louis XVII, of France, 1785-1795 -- Fiction   \n",
       "...                                                                                                                                                                                                                                                                                                     ...   \n",
       "8545                                                                                                                                                                                                                   Inheritance and succession -- Fiction; England -- Social life and customs -- Fiction   \n",
       "8546                                                                                                                                                                           Inheritance and succession -- Fiction; Mate selection -- Fiction; Courtship -- Fiction; Social classes -- England -- Fiction   \n",
       "8547                                                                                                                                                                                                                                                                                                Fiction   \n",
       "8548                          Novelists, English -- 19th century -- Biography; Women and literature -- Great Britain -- History -- 19th century; English fiction -- 19th century -- History and criticism; English fiction -- Women authors -- History and criticism; Women novelists, English -- Biography   \n",
       "8549                                                                                                                                                                                                                                                                                                Fiction   \n",
       "\n",
       "      subjectcount    pd   dob   dod   edp  decade  century  \n",
       "0                3  1814  1814  1873  1843    1840       19  \n",
       "1                4  1867  1814  1873  1843    1840       19  \n",
       "2                3  1866  1814  1873  1843    1840       19  \n",
       "3                2  1868  1814  1873  1843    1840       19  \n",
       "4                3  1866  1814  1873  1843    1840       19  \n",
       "...            ...   ...   ...   ...   ...     ...      ...  \n",
       "8545             2  1889  1825  1902  1863    1860       19  \n",
       "8546             4  1875  1825  1902  1863    1860       19  \n",
       "8547             1  1854  1825  1902  1863    1860       19  \n",
       "8548             5  1897  1825  1902  1863    1860       19  \n",
       "8549             1  1854  1825  1902  1863    1860       19  \n",
       "\n",
       "[8550 rows x 11 columns]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gefa.drop(\"text\",axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "egefa = gefa[gefa[\"pd\"]<1899]\n",
    "ngefa = gefa[gefa[\"pd\"]>1899]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "ngefa.to_csv(\"19002000.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dividing lampenter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1641-01-01\n",
       "Name: date, dtype: object"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lampenter = pd.read_csv(\"lampenter.csv\")\n",
    "lampenter.date.head(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Century wise cleaning and the combining the files\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1500 - 1600"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "      <th>date</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[The remedy against the troubles of temptation...</td>\n",
       "      <td>book.xml</td>\n",
       "      <td>1508</td>\n",
       "      <td>The-remedy-against-the-troubles-of-temptations</td>\n",
       "      <td>Richard-rolle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The ballade of ane right noble victorius &amp; myg...</td>\n",
       "      <td>book.xml</td>\n",
       "      <td>1508</td>\n",
       "      <td>The-ballade-of-ane-right-noble-victorius</td>\n",
       "      <td>William-Dunbar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[The noble history of King Ponthus.]Ponthus et...</td>\n",
       "      <td>book.xml</td>\n",
       "      <td>1511</td>\n",
       "      <td>The-noble-history-of-King-Ponthus</td>\n",
       "      <td>Henry-watson</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Here begynneth a dialogue betwene the comen se...</td>\n",
       "      <td>letter.xml</td>\n",
       "      <td>1530</td>\n",
       "      <td>dialogue-betwene-the-comen-secretary-and-Ielow...</td>\n",
       "      <td>Edward-Gosynhyll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A compendious olde treatyse, shewynge howe tha...</td>\n",
       "      <td>book.xml</td>\n",
       "      <td>1530</td>\n",
       "      <td>A-compendious-olde-treatyse</td>\n",
       "      <td>Richard-ullerston</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text    category  date  \\\n",
       "0  [The remedy against the troubles of temptation...    book.xml  1508   \n",
       "1  The ballade of ane right noble victorius & myg...    book.xml  1508   \n",
       "2  [The noble history of King Ponthus.]Ponthus et...    book.xml  1511   \n",
       "3  Here begynneth a dialogue betwene the comen se...  letter.xml  1530   \n",
       "4  A compendious olde treatyse, shewynge howe tha...    book.xml  1530   \n",
       "\n",
       "                                               title             author  \n",
       "0     The-remedy-against-the-troubles-of-temptations      Richard-rolle  \n",
       "1           The-ballade-of-ane-right-noble-victorius     William-Dunbar  \n",
       "2                  The-noble-history-of-King-Ponthus       Henry-watson  \n",
       "3  dialogue-betwene-the-comen-secretary-and-Ielow...   Edward-Gosynhyll  \n",
       "4                        A-compendious-olde-treatyse  Richard-ullerston  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import xml.etree.ElementTree as ET\n",
    "\n",
    "df_1500 = pd.DataFrame(columns=[\"text\",\"category\",\"date\",\"title\",\"author\"])\n",
    "\n",
    "lst = os.listdir(r\"C:\\Users\\rhira\\Documents\\TextAge\\15001600\")\n",
    "for nam in lst:\n",
    "    yr, aut, tit, typ = nam.split(\"_\")\n",
    "    #print(yr, aut, tit, type)\n",
    "    \n",
    "    xml_file_path = r'C:\\Users\\rhira\\Documents\\TextAge\\15001600'+\"\\\\\"+nam\n",
    "    tree = ET.parse(xml_file_path)\n",
    "    root = tree.getroot()\n",
    "\n",
    "    # Define a namespace dictionary for convenience\n",
    "    ns = {'tei': 'http://www.tei-c.org/ns/1.0'}\n",
    "\n",
    "    # Function to safely extract text content from an element\n",
    "    def get_element_text(element, xpath, namespaces=ns):\n",
    "        result = element.find(xpath, namespaces=namespaces)\n",
    "        return result.text.strip() if result is not None and result.text is not None else None\n",
    "\n",
    "    # Function to recursively concatenate text content from an element and its children\n",
    "    def get_all_text(element, namespaces=ns, exclude_tags=None):\n",
    "        result = ''\n",
    "        if element.text and (exclude_tags is None or all(tag not in element.tag for tag in exclude_tags)):\n",
    "            result += element.text.strip()\n",
    "        for child in element:\n",
    "            result += get_all_text(child, exclude_tags=exclude_tags)\n",
    "            if child.tail:\n",
    "                result += child.tail.strip()\n",
    "        return result\n",
    "    \n",
    "    exclude_tags = {'titleStmt', 'licence', 'author', 'respStmt'}\n",
    "    \n",
    "    all_text = get_all_text(root, exclude_tags=exclude_tags)\n",
    "    \n",
    "    # text\",\"category\",\"date\",\"head\",\"title\",\"author\"\n",
    "    dct = {\"text\":all_text,\n",
    "           \"date\": yr, \n",
    "           \"author\":aut, \n",
    "           \"title\":tit, \n",
    "           \"category\":typ}\n",
    "    \n",
    "    df_dictionary = pd.DataFrame([dct])\n",
    "    df_1500 = pd.concat([df_1500, df_dictionary], ignore_index=True)\n",
    "\n",
    "\n",
    "# Exclude specific elements from being included in the te\n",
    "\n",
    "df_1500.head()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 50 entries, 0 to 49\n",
      "Data columns (total 5 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   text      50 non-null     object\n",
      " 1   category  50 non-null     object\n",
      " 2   date      50 non-null     object\n",
      " 3   title     50 non-null     object\n",
      " 4   author    50 non-null     object\n",
      "dtypes: object(5)\n",
      "memory usage: 2.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df_1500.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1600-1700"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import xml.etree.ElementTree as ET\n",
    "\n",
    "df_1600 = pd.DataFrame(columns=[\"text\",\"category\",\"date\",\"title\",\"author\"])\n",
    "\n",
    "lst = os.listdir(r\"C:\\Users\\rhira\\Documents\\TextAge\\16001700\")\n",
    "\n",
    "lst.remove(\"16001700.csv\")\n",
    "for nam in lst:\n",
    "    #print(nam)\n",
    "    yr, aut, tit, typ = nam.split(\"_\")\n",
    "    #print(yr, aut, tit, type)\n",
    "    \n",
    "    xml_file_path = r'C:\\Users\\rhira\\Documents\\TextAge\\16001700'+\"\\\\\"+nam\n",
    "    tree = ET.parse(xml_file_path)\n",
    "    root = tree.getroot()\n",
    "\n",
    "    # Define a namespace dictionary for convenience\n",
    "    ns = {'tei': 'http://www.tei-c.org/ns/1.0'}\n",
    "\n",
    "    # Function to safely extract text content from an element\n",
    "    def get_element_text(element, xpath, namespaces=ns):\n",
    "        result = element.find(xpath, namespaces=namespaces)\n",
    "        return result.text.strip() if result is not None and result.text is not None else None\n",
    "\n",
    "    # Function to recursively concatenate text content from an element and its children\n",
    "    def get_all_text(element, namespaces=ns, exclude_tags=None):\n",
    "        result = ''\n",
    "        if element.text and (exclude_tags is None or all(tag not in element.tag for tag in exclude_tags)):\n",
    "            result += element.text.strip()\n",
    "        for child in element:\n",
    "            result += get_all_text(child, exclude_tags=exclude_tags)\n",
    "            if child.tail:\n",
    "                result += child.tail.strip()\n",
    "        return result\n",
    "    \n",
    "    exclude_tags = {'titleStmt', 'licence', 'author', 'respStmt'}\n",
    "    \n",
    "    all_text = get_all_text(root, exclude_tags=exclude_tags)\n",
    "    \n",
    "    # text\",\"category\",\"date\",\"head\",\"title\",\"author\"\n",
    "    dct = {\"text\":all_text,\n",
    "           \"date\": yr, \n",
    "           \"author\":aut, \n",
    "           \"title\":tit, \n",
    "           \"category\":typ}\n",
    "    \n",
    "    df_dictionary = pd.DataFrame([dct])\n",
    "    df_1600 = pd.concat([df_1600, df_dictionary], ignore_index=True)\n",
    "\n",
    "\n",
    "# Exclude specific elements from being included in the te\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 70 entries, 0 to 69\n",
      "Data columns (total 5 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   text      70 non-null     object\n",
      " 1   category  70 non-null     object\n",
      " 2   date      70 non-null     object\n",
      " 3   title     70 non-null     object\n",
      " 4   author    70 non-null     object\n",
      "dtypes: object(5)\n",
      "memory usage: 2.9+ KB\n"
     ]
    }
   ],
   "source": [
    "df_1600.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>To contend with men that deny their owne publi...</td>\n",
       "      <td>Economy</td>\n",
       "      <td>A Reply to a most untrue Relation made, and se...</td>\n",
       "      <td>A reply to a most untrue relation [...] by cer...</td>\n",
       "      <td>1641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>IT hath been a thing for many years generally ...</td>\n",
       "      <td>Economy</td>\n",
       "      <td>THE ADVOCATE: OR, A NARRATIVE Of the state and...</td>\n",
       "      <td>The Advocate.</td>\n",
       "      <td>1652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I Presume, there are few Englishmen so disloya...</td>\n",
       "      <td>Economy</td>\n",
       "      <td>\\nPropos. 1.\\n\\nIt will supply his Majesties p...</td>\n",
       "      <td>A discourse ,shewing the many advantages which...</td>\n",
       "      <td>1668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HAving lately seen a Pamphlet under the Title ...</td>\n",
       "      <td>Economy</td>\n",
       "      <td>An Enquiry into the Grounds of a late Pamphlet...</td>\n",
       "      <td>Is not the hand of Joab in all this? Or An enq...</td>\n",
       "      <td>1676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FOrasmuch as there is a very great Complaint i...</td>\n",
       "      <td>Economy</td>\n",
       "      <td>SECT. I. \\nThe Introduction.</td>\n",
       "      <td>The Trade of England Revived: And the Abuses t...</td>\n",
       "      <td>1681</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text category  \\\n",
       "0  To contend with men that deny their owne publi...  Economy   \n",
       "1  IT hath been a thing for many years generally ...  Economy   \n",
       "2  I Presume, there are few Englishmen so disloya...  Economy   \n",
       "3  HAving lately seen a Pamphlet under the Title ...  Economy   \n",
       "4  FOrasmuch as there is a very great Complaint i...  Economy   \n",
       "\n",
       "                                              author  \\\n",
       "0  A Reply to a most untrue Relation made, and se...   \n",
       "1  THE ADVOCATE: OR, A NARRATIVE Of the state and...   \n",
       "2  \\nPropos. 1.\\n\\nIt will supply his Majesties p...   \n",
       "3  An Enquiry into the Grounds of a late Pamphlet...   \n",
       "4                       SECT. I. \\nThe Introduction.   \n",
       "\n",
       "                                               title  date  \n",
       "0  A reply to a most untrue relation [...] by cer...  1641  \n",
       "1                                      The Advocate.  1652  \n",
       "2  A discourse ,shewing the many advantages which...  1668  \n",
       "3  Is not the hand of Joab in all this? Or An enq...  1676  \n",
       "4  The Trade of England Revived: And the Abuses t...  1681  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df_1600_extra = pd.read_csv(r\"C:\\Users\\rhira\\Documents\\TextAge\\16001700\\16001700.csv\")\n",
    "df_1600_extra.drop([\"Unnamed: 0.1\",\"Unnamed: 0\",\"id\",\"date\"],axis=1, inplace=True)\n",
    "df_1600_extra.rename(columns={\"genre\": \"category\", \"year\": \"date\",\"head\": \"author\"}, inplace=True)\n",
    "df_1600_extra.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 142 entries, 0 to 141\n",
      "Data columns (total 5 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   text      142 non-null    object\n",
      " 1   category  142 non-null    object\n",
      " 2   date      142 non-null    object\n",
      " 3   title     142 non-null    object\n",
      " 4   author    136 non-null    object\n",
      "dtypes: object(5)\n",
      "memory usage: 5.7+ KB\n"
     ]
    }
   ],
   "source": [
    "df_1600 = pd.concat([df_1600, df_1600_extra], ignore_index=True)\n",
    "df_1600.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1700 1800"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import xml.etree.ElementTree as ET\n",
    "\n",
    "df_1700 = pd.DataFrame(columns=[\"text\",\"category\",\"date\",\"title\",\"author\"])\n",
    "\n",
    "lst = os.listdir(r\"C:\\Users\\rhira\\Documents\\TextAge\\17001800\\dataset\")\n",
    "\n",
    "lst.remove(\"17001800.csv\")\n",
    "for nam in lst:\n",
    "    #print(nam)\n",
    "    yr, aut, tit, typ = nam.split(\"_\")\n",
    "    #print(yr, aut, tit, type)\n",
    "    \n",
    "    xml_file_path = r'C:\\Users\\rhira\\Documents\\TextAge\\17001800\\dataset'+\"\\\\\"+nam\n",
    "    tree = ET.parse(xml_file_path)\n",
    "    root = tree.getroot()\n",
    "\n",
    "    # Define a namespace dictionary for convenience\n",
    "    ns = {'tei': 'http://www.tei-c.org/ns/1.0'}\n",
    "\n",
    "    # Function to safely extract text content from an element\n",
    "    def get_element_text(element, xpath, namespaces=ns):\n",
    "        result = element.find(xpath, namespaces=namespaces)\n",
    "        return result.text.strip() if result is not None and result.text is not None else None\n",
    "\n",
    "    # Function to recursively concatenate text content from an element and its children\n",
    "    def get_all_text(element, namespaces=ns, exclude_tags=None):\n",
    "        result = ''\n",
    "        if element.text and (exclude_tags is None or all(tag not in element.tag for tag in exclude_tags)):\n",
    "            result += element.text.strip()\n",
    "        for child in element:\n",
    "            result += get_all_text(child, exclude_tags=exclude_tags)\n",
    "            if child.tail:\n",
    "                result += child.tail.strip()\n",
    "        return result\n",
    "    \n",
    "    exclude_tags = {'titleStmt', 'licence', 'author', 'respStmt'}\n",
    "    \n",
    "    all_text = get_all_text(root, exclude_tags=exclude_tags)\n",
    "    \n",
    "    # text\",\"category\",\"date\",\"head\",\"title\",\"author\"\n",
    "    dct = {\"text\":all_text,\n",
    "           \"date\": yr, \n",
    "           \"author\":aut, \n",
    "           \"title\":tit, \n",
    "           \"category\":typ}\n",
    "    \n",
    "    df_dictionary = pd.DataFrame([dct])\n",
    "    df_1700 = pd.concat([df_1700, df_dictionary], ignore_index=True)\n",
    "\n",
    "\n",
    "# Exclude specific elements from being included in the te\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "      <th>author</th>\n",
       "      <th>title</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>\\n\\nThe Influence of the Bank considerable. NO...</td>\n",
       "      <td>Economy</td>\n",
       "      <td>THE Introduction.</td>\n",
       "      <td>Remarks upon the Bank of England [...]</td>\n",
       "      <td>1705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>THERE has been a Paper already offered to the ...</td>\n",
       "      <td>Economy</td>\n",
       "      <td>SOME OBSERVATIONS, SHEWING The DANGER of Losin...</td>\n",
       "      <td>Some observations, shewing the danger of losin...</td>\n",
       "      <td>1714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>THE Miscarriage of the Naval StoreBill last Se...</td>\n",
       "      <td>Economy</td>\n",
       "      <td>A LETTER TO A Member of Parliament.</td>\n",
       "      <td>A letter to a member of parliament, concerning...</td>\n",
       "      <td>1720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>IT is not possible that the Island of Barbadoe...</td>\n",
       "      <td>Economy</td>\n",
       "      <td>\\nREMARKS Upon a BOOK, Entituled, The Present ...</td>\n",
       "      <td>Remarks upon a book, entituled, the present st...</td>\n",
       "      <td>1731</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>WHEN I see a Proposal for raising of Money, wi...</td>\n",
       "      <td>Economy</td>\n",
       "      <td>THE Injustice, Dishonour AND Ill-Policy OF Bre...</td>\n",
       "      <td>Wednesday club-law: or, The injustice, dishono...</td>\n",
       "      <td>1717</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text category  \\\n",
       "0  \\n\\nThe Influence of the Bank considerable. NO...  Economy   \n",
       "1  THERE has been a Paper already offered to the ...  Economy   \n",
       "2  THE Miscarriage of the Naval StoreBill last Se...  Economy   \n",
       "3  IT is not possible that the Island of Barbadoe...  Economy   \n",
       "4  WHEN I see a Proposal for raising of Money, wi...  Economy   \n",
       "\n",
       "                                              author  \\\n",
       "0                                  THE Introduction.   \n",
       "1  SOME OBSERVATIONS, SHEWING The DANGER of Losin...   \n",
       "2                A LETTER TO A Member of Parliament.   \n",
       "3  \\nREMARKS Upon a BOOK, Entituled, The Present ...   \n",
       "4  THE Injustice, Dishonour AND Ill-Policy OF Bre...   \n",
       "\n",
       "                                               title  date  \n",
       "0             Remarks upon the Bank of England [...]  1705  \n",
       "1  Some observations, shewing the danger of losin...  1714  \n",
       "2  A letter to a member of parliament, concerning...  1720  \n",
       "3  Remarks upon a book, entituled, the present st...  1731  \n",
       "4  Wednesday club-law: or, The injustice, dishono...  1717  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df_1700_extra = pd.read_csv(r\"C:\\Users\\rhira\\Documents\\TextAge\\17001800\\dataset\\17001800.csv\")\n",
    "df_1700_extra.drop([\"Unnamed: 0.1\",\"Unnamed: 0\",\"id\",\"date\"],axis=1, inplace=True)\n",
    "df_1700_extra.rename(columns={\"genre\": \"category\", \"year\": \"date\",\"head\": \"author\"}, inplace=True)\n",
    "df_1700_extra.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 104 entries, 0 to 103\n",
      "Data columns (total 5 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   text      104 non-null    object\n",
      " 1   category  104 non-null    object\n",
      " 2   date      104 non-null    object\n",
      " 3   title     104 non-null    object\n",
      " 4   author    96 non-null     object\n",
      "dtypes: object(5)\n",
      "memory usage: 4.2+ KB\n"
     ]
    }
   ],
   "source": [
    "df_1700 = pd.concat([df_1700, df_1700_extra], ignore_index=True)\n",
    "df_1700.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1800 1900"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "      <th>date</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The author is indebted to one of the novels of...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1817</td>\n",
       "      <td>8550.txt</td>\n",
       "      <td>The author is indebted to one of the novel</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Dramatis Person . Alexius , Emperour of Greece...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1827</td>\n",
       "      <td>8950.txt</td>\n",
       "      <td>Dramatis Person . Alexius , Emperour of Gr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>\" Oh Love ! what is it in this world of ours ....</td>\n",
       "      <td>fic</td>\n",
       "      <td>1830</td>\n",
       "      <td>8750.txt</td>\n",
       "      <td>\" Oh Love ! what is it in this world of ou</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TO MRS . HAMBLIN , THIS DRAMATIC TRIFLE . ( Th...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1831</td>\n",
       "      <td>8850.txt</td>\n",
       "      <td>TO MRS . HAMBLIN , THIS DRAMATIC TRIFLE .</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>THE PARTISAN LEADER ; A TALE OF THE FUTURE . B...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1836</td>\n",
       "      <td>7350.txt</td>\n",
       "      <td>THE PARTISAN LEADER ; A TALE OF THE FUTURE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>PROLOGUE . DELIVERED BY MRS . PRESTON . This n...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1836</td>\n",
       "      <td>8650.txt</td>\n",
       "      <td>PROLOGUE . DELIVERED BY MRS . PRESTON . Th</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>CHARCOAL SKETCHES ; OR , SCENES IN A METROPOLI...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1838</td>\n",
       "      <td>7250.txt</td>\n",
       "      <td>CHARCOAL SKETCHES ; OR , SCENES IN A METRO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Chivers , T. H. ( Thomas Holley ) , 1809-1858 ...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1845</td>\n",
       "      <td>7450.txt</td>\n",
       "      <td>Chivers , T. H. ( Thomas Holley ) , 1809-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Hoffman , Charles Fenno , 1806-1884 : AFTERTHO...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1847</td>\n",
       "      <td>7650.txt</td>\n",
       "      <td>Hoffman , Charles Fenno , 1806-1884 : AFTE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>CAST OF CHARACTERS . Uncle Jeff , a mischief-m...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1868</td>\n",
       "      <td>9550.txt</td>\n",
       "      <td>CAST OF CHARACTERS . Uncle Jeff , a mischi</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text category  date     title  \\\n",
       "0  The author is indebted to one of the novels of...      fic  1817  8550.txt   \n",
       "1  Dramatis Person . Alexius , Emperour of Greece...      fic  1827  8950.txt   \n",
       "2  \" Oh Love ! what is it in this world of ours ....      fic  1830  8750.txt   \n",
       "3  TO MRS . HAMBLIN , THIS DRAMATIC TRIFLE . ( Th...      fic  1831  8850.txt   \n",
       "4  THE PARTISAN LEADER ; A TALE OF THE FUTURE . B...      fic  1836  7350.txt   \n",
       "5  PROLOGUE . DELIVERED BY MRS . PRESTON . This n...      fic  1836  8650.txt   \n",
       "6  CHARCOAL SKETCHES ; OR , SCENES IN A METROPOLI...      fic  1838  7250.txt   \n",
       "7  Chivers , T. H. ( Thomas Holley ) , 1809-1858 ...      fic  1845  7450.txt   \n",
       "8  Hoffman , Charles Fenno , 1806-1884 : AFTERTHO...      fic  1847  7650.txt   \n",
       "9  CAST OF CHARACTERS . Uncle Jeff , a mischief-m...      fic  1868  9550.txt   \n",
       "\n",
       "                                       author  \n",
       "0  The author is indebted to one of the novel  \n",
       "1  Dramatis Person . Alexius , Emperour of Gr  \n",
       "2  \" Oh Love ! what is it in this world of ou  \n",
       "3  TO MRS . HAMBLIN , THIS DRAMATIC TRIFLE .   \n",
       "4  THE PARTISAN LEADER ; A TALE OF THE FUTURE  \n",
       "5  PROLOGUE . DELIVERED BY MRS . PRESTON . Th  \n",
       "6  CHARCOAL SKETCHES ; OR , SCENES IN A METRO  \n",
       "7  Chivers , T. H. ( Thomas Holley ) , 1809-1  \n",
       "8  Hoffman , Charles Fenno , 1806-1884 : AFTE  \n",
       "9  CAST OF CHARACTERS . Uncle Jeff , a mischi  "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_1800 = pd.DataFrame(columns=[\"text\",\"category\",\"date\",\"title\",\"author\"])\n",
    "\n",
    "lst = os.listdir(r\"C:\\Users\\rhira\\Documents\\TextAge\\18001900\")\n",
    "\n",
    "lst.remove(\"18001900.csv\")\n",
    "\n",
    "for nam in lst:\n",
    "    #print(nam)\n",
    "    #yr, aut, tit, typ = nam.split(\"_\")\n",
    "    typ, yr, fid = nam.split(\"_\")\n",
    "    f = open(r\"C:\\Users\\rhira\\Documents\\TextAge\\18001900\"+\"\\\\\"+nam, \"r\")\n",
    "    txt = f.read()\n",
    "\n",
    "    dct = {\"text\":txt[8:],\n",
    "           \"date\": yr, \n",
    "           \"author\":txt[8:50], \n",
    "           \"title\":fid, \n",
    "           \"category\":typ}\n",
    "    \n",
    "    df_dictionary = pd.DataFrame([dct])\n",
    "    df_1800 = pd.concat([df_1800, df_dictionary], ignore_index=True)\n",
    "df_1800.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4449 entries, 0 to 4448\n",
      "Data columns (total 5 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   text      4449 non-null   object\n",
      " 1   category  4449 non-null   object\n",
      " 2   date      4449 non-null   object\n",
      " 3   title     4449 non-null   object\n",
      " 4   author    4449 non-null   object\n",
      "dtypes: object(5)\n",
      "memory usage: 173.9+ KB\n"
     ]
    }
   ],
   "source": [
    "df_1800_extra = pd.read_csv(r\"C:\\Users\\rhira\\Documents\\TextAge\\18001900\\18001900.csv\")\n",
    "df_1800_extra.drop([\"Unnamed: 0.1\",\"Unnamed: 0\",\"subjectcount\",\"dob\",\"dod\",\"edp\",\"century\",\"decade\"],axis=1, inplace=True)\n",
    "# text\tcategory\tdate\ttitle\tauthor\n",
    "df_1800_extra.rename(columns={\"subjects\": \"category\", \"pd\": \"date\",\"author_tag\": \"author\"}, inplace=True)\n",
    "\n",
    "df_1800 = pd.concat([df_1800, df_1800_extra], ignore_index=True)\n",
    "df_1800.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1900 2000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "      <th>date</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>( This file was produced from scans of public ...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1900</td>\n",
       "      <td>3850.txt</td>\n",
       "      <td>( This file was produced from scans of pub</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\\nA kitchen . At the right an ironing-board , ...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1910</td>\n",
       "      <td>13150.txt</td>\n",
       "      <td>\\nA kitchen . At the right an ironing-board</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>JIMSY *** Produced by Annie McGuire Jimsy The ...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1915</td>\n",
       "      <td>5250.txt</td>\n",
       "      <td>JIMSY *** Produced by Annie McGuire Jimsy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>THE VALLEY OF VISION A BOOK OF ROMANCE AND SOM...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1919</td>\n",
       "      <td>1550.txt</td>\n",
       "      <td>THE VALLEY OF VISION A BOOK OF ROMANCE AND</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HE DRUMS OF JEOPARDY By Harold MacGrath CHAPTE...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1920</td>\n",
       "      <td>550.txt</td>\n",
       "      <td>HE DRUMS OF JEOPARDY By Harold MacGrath CH</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>\\nVIOLETTE A STORY BY VINCENT SHEEAN T HE pass...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1930</td>\n",
       "      <td>23150.txt</td>\n",
       "      <td>\\nVIOLETTE A STORY BY VINCENT SHEEAN T HE p</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>\\nVr ) 411 , - THE MAN WHO KNEW EVERYBODY A ST...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1933</td>\n",
       "      <td>23250.txt</td>\n",
       "      <td>\\nVr ) 411 , - THE MAN WHO KNEW EVERYBODY A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>A Sequel to \" Seed of the Arctic Ice \" Under A...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1933</td>\n",
       "      <td>5650.txt</td>\n",
       "      <td>A Sequel to \" Seed of the Arctic Ice \" Und</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>\\nLADY IN TROUBLE A STORY BY MARGERY SHARP ry'...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1937</td>\n",
       "      <td>23350.txt</td>\n",
       "      <td>\\nLADY IN TROUBLE A STORY BY MARGERY SHARP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>\\nWILD LINES : SWIMMING HOLE SCENE View page i...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1938</td>\n",
       "      <td>16350.txt</td>\n",
       "      <td>\\nWILD LINES : SWIMMING HOLE SCENE View pag</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text category  date  \\\n",
       "0  ( This file was produced from scans of public ...      fic  1900   \n",
       "1  \\nA kitchen . At the right an ironing-board , ...      fic  1910   \n",
       "2  JIMSY *** Produced by Annie McGuire Jimsy The ...      fic  1915   \n",
       "3  THE VALLEY OF VISION A BOOK OF ROMANCE AND SOM...      fic  1919   \n",
       "4  HE DRUMS OF JEOPARDY By Harold MacGrath CHAPTE...      fic  1920   \n",
       "5  \\nVIOLETTE A STORY BY VINCENT SHEEAN T HE pass...      fic  1930   \n",
       "6  \\nVr ) 411 , - THE MAN WHO KNEW EVERYBODY A ST...      fic  1933   \n",
       "7  A Sequel to \" Seed of the Arctic Ice \" Under A...      fic  1933   \n",
       "8  \\nLADY IN TROUBLE A STORY BY MARGERY SHARP ry'...      fic  1937   \n",
       "9  \\nWILD LINES : SWIMMING HOLE SCENE View page i...      fic  1938   \n",
       "\n",
       "       title                                       author  \n",
       "0   3850.txt   ( This file was produced from scans of pub  \n",
       "1  13150.txt  \\nA kitchen . At the right an ironing-board  \n",
       "2   5250.txt   JIMSY *** Produced by Annie McGuire Jimsy   \n",
       "3   1550.txt   THE VALLEY OF VISION A BOOK OF ROMANCE AND  \n",
       "4    550.txt   HE DRUMS OF JEOPARDY By Harold MacGrath CH  \n",
       "5  23150.txt  \\nVIOLETTE A STORY BY VINCENT SHEEAN T HE p  \n",
       "6  23250.txt  \\nVr ) 411 , - THE MAN WHO KNEW EVERYBODY A  \n",
       "7   5650.txt   A Sequel to \" Seed of the Arctic Ice \" Und  \n",
       "8  23350.txt  \\nLADY IN TROUBLE A STORY BY MARGERY SHARP   \n",
       "9  16350.txt  \\nWILD LINES : SWIMMING HOLE SCENE View pag  "
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_1900 = pd.DataFrame(columns=[\"text\",\"category\",\"date\",\"title\",\"author\"])\n",
    "\n",
    "lst = os.listdir(r\"C:\\Users\\rhira\\Documents\\TextAge\\19002000\")\n",
    "\n",
    "lst.remove(\"19002000.csv\")\n",
    "\n",
    "for nam in lst:\n",
    "    #print(nam)\n",
    "    #yr, aut, tit, typ = nam.split(\"_\")\n",
    "    typ, yr, fid = nam.split(\"_\")\n",
    "    f = open(r\"C:\\Users\\rhira\\Documents\\TextAge\\19002000\"+\"\\\\\"+nam, \"r\")\n",
    "    txt = f.read()\n",
    "\n",
    "    dct = {\"text\":txt[8:],\n",
    "           \"date\": yr, \n",
    "           \"author\":txt[8:50], \n",
    "           \"title\":fid, \n",
    "           \"category\":typ}\n",
    "    \n",
    "    df_dictionary = pd.DataFrame([dct])\n",
    "    df_1900 = pd.concat([df_1900, df_dictionary], ignore_index=True)\n",
    "df_1900.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 4555 entries, 0 to 4554\n",
      "Data columns (total 5 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   text      3121 non-null   object\n",
      " 1   category  3121 non-null   object\n",
      " 2   date      3121 non-null   object\n",
      " 3   title     3121 non-null   object\n",
      " 4   author    3121 non-null   object\n",
      "dtypes: object(5)\n",
      "memory usage: 178.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df_1900_extra = pd.read_csv(r\"C:\\Users\\rhira\\Documents\\TextAge\\19002000\\19002000.csv\")\n",
    "df_1900_extra.drop([\"Unnamed: 0.1\",\"Unnamed: 0\",\"subjectcount\",\"dob\",\"dod\",\"edp\",\"century\",\"decade\"],axis=1, inplace=True)\n",
    "# text\tcategory\tdate\ttitle\tauthor\n",
    "df_1900_extra.rename(columns={\"subjects\": \"category\", \"pd\": \"date\",\"author_tag\": \"author\"}, inplace=True)\n",
    "\n",
    "df_1900 = pd.concat([df_1900, df_1900_extra], ignore_index=True)\n",
    "df_1900.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "      <th>date</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>( This file was produced from scans of public ...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1900</td>\n",
       "      <td>3850.txt</td>\n",
       "      <td>( This file was produced from scans of pub</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\\nA kitchen . At the right an ironing-board , ...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1910</td>\n",
       "      <td>13150.txt</td>\n",
       "      <td>\\nA kitchen . At the right an ironing-board</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>JIMSY *** Produced by Annie McGuire Jimsy The ...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1915</td>\n",
       "      <td>5250.txt</td>\n",
       "      <td>JIMSY *** Produced by Annie McGuire Jimsy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>THE VALLEY OF VISION A BOOK OF ROMANCE AND SOM...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1919</td>\n",
       "      <td>1550.txt</td>\n",
       "      <td>THE VALLEY OF VISION A BOOK OF ROMANCE AND</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HE DRUMS OF JEOPARDY By Harold MacGrath CHAPTE...</td>\n",
       "      <td>fic</td>\n",
       "      <td>1920</td>\n",
       "      <td>550.txt</td>\n",
       "      <td>HE DRUMS OF JEOPARDY By Harold MacGrath CH</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text category  date  \\\n",
       "0  ( This file was produced from scans of public ...      fic  1900   \n",
       "1  \\nA kitchen . At the right an ironing-board , ...      fic  1910   \n",
       "2  JIMSY *** Produced by Annie McGuire Jimsy The ...      fic  1915   \n",
       "3  THE VALLEY OF VISION A BOOK OF ROMANCE AND SOM...      fic  1919   \n",
       "4  HE DRUMS OF JEOPARDY By Harold MacGrath CHAPTE...      fic  1920   \n",
       "\n",
       "       title                                       author  \n",
       "0   3850.txt   ( This file was produced from scans of pub  \n",
       "1  13150.txt  \\nA kitchen . At the right an ironing-board  \n",
       "2   5250.txt   JIMSY *** Produced by Annie McGuire Jimsy   \n",
       "3   1550.txt   THE VALLEY OF VISION A BOOK OF ROMANCE AND  \n",
       "4    550.txt   HE DRUMS OF JEOPARDY By Harold MacGrath CH  "
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_1900.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 9300 entries, 0 to 9299\n",
      "Data columns (total 5 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   text      7866 non-null   object\n",
      " 1   category  7866 non-null   object\n",
      " 2   date      7866 non-null   object\n",
      " 3   title     7866 non-null   object\n",
      " 4   author    7852 non-null   object\n",
      "dtypes: object(5)\n",
      "memory usage: 363.4+ KB\n"
     ]
    }
   ],
   "source": [
    "fin_df = pd.concat([df_1500, df_1600,df_1700,df_1800,df_1900], ignore_index=True)\n",
    "fin_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "fin_df.to_csv(\"fin_df.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parsing the clmmet dataset to add to corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "fl = r'C:\\Users\\rhira\\Documents\\TextAge\\plain\\CLMET3_1_3_317.txt'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                text           category  date  \\\n",
      "0  A TREATISE Concerning the PRINCIPLES OF Human ...           Treatise  1710   \n",
      "1  SERMON I. UPON HUMAN NATURE .\\nROMANS xii .\\n4...           Treatise  1726   \n",
      "2  THE London Merchant : OR , THE HISTORY OF GEOR...              Drama  1731   \n",
      "3  MARINA : A PLAY OF THREE ACTS .\\nAs it is Acte...              Drama  1738   \n",
      "4  THE FORTUNATE FOUNDLINGS .\\nCHAP .\\nI .\\nConta...  Narrative fiction  1744   \n",
      "\n",
      "                                               title                 author  \n",
      "0  A treatise concerning the principles of human ...       Berkeley, George  \n",
      "1                     Human nature and other sermons         Butler, Joseph  \n",
      "2                                The London merchant          Lillo, George  \n",
      "3                                             Marina          Lillo, George  \n",
      "4                           The fortunate foundlings  Haywood, Eliza Fowler  \n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "# Function to extract information from a file\n",
    "def extract_info(file_path):\n",
    "    with open(file_path, 'r',encoding=\"utf-8\") as file:\n",
    "        content = file.read()\n",
    "\n",
    "    soup = BeautifulSoup(content, 'html.parser')\n",
    "\n",
    "    id_tag = soup.find('id')\n",
    "    genre_tag = soup.find('genre')\n",
    "    year_tag = soup.find('year')\n",
    "    title_tag = soup.find('title')\n",
    "    author_tag = soup.find('author')\n",
    "\n",
    "    id_value = id_tag.text if id_tag else None\n",
    "    genre_value = genre_tag.text if genre_tag else None\n",
    "    year_value = year_tag.text if year_tag else None\n",
    "    title_value = title_tag.text if title_tag else None\n",
    "    author_value = author_tag.text if author_tag else None\n",
    "\n",
    "    paragraphs = soup.find_all('p')\n",
    "    p_content = '\\n'.join(paragraph.text.strip() for paragraph in paragraphs)\n",
    "\n",
    "    output_dict = {\n",
    "        'text': p_content,\n",
    "        'category': genre_value,\n",
    "        'date': year_value,\n",
    "        'title': title_value,\n",
    "        'author': author_value\n",
    "    }\n",
    "\n",
    "    return output_dict\n",
    "\n",
    "# Folder containing the HTML files\n",
    "folder_path = r'C:\\Users\\rhira\\Documents\\TextAge\\plain'\n",
    "\n",
    "# List to store dictionaries for each file\n",
    "data_list = []\n",
    "\n",
    "# Process each file in the folder\n",
    "for filename in os.listdir(folder_path):\n",
    "    if filename.endswith(\".txt\"):  # Assuming the files have an HTML extension\n",
    "        file_path = os.path.join(folder_path, filename)\n",
    "        #print(file_path)\n",
    "        try:\n",
    "            file_data = extract_info(file_path)\n",
    "            data_list.append(file_data)\n",
    "        except:\n",
    "            print(file_path)\n",
    "\n",
    "# Convert the list of dictionaries to a Pandas DataFrame\n",
    "df = pd.DataFrame(data_list)\n",
    "\n",
    "# Print or use the DataFrame as needed\n",
    "print(df.head())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 333 entries, 0 to 332\n",
      "Data columns (total 5 columns):\n",
      " #   Column    Non-Null Count  Dtype \n",
      "---  ------    --------------  ----- \n",
      " 0   text      333 non-null    object\n",
      " 1   category  333 non-null    object\n",
      " 2   date      333 non-null    object\n",
      " 3   title     333 non-null    object\n",
      " 4   author    333 non-null    object\n",
      "dtypes: object(5)\n",
      "memory usage: 13.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "      <th>date</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A TREATISE Concerning the PRINCIPLES OF Human ...</td>\n",
       "      <td>Treatise</td>\n",
       "      <td>1710</td>\n",
       "      <td>A treatise concerning the principles of human ...</td>\n",
       "      <td>Berkeley, George</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SERMON I. UPON HUMAN NATURE .\\nROMANS xii .\\n4...</td>\n",
       "      <td>Treatise</td>\n",
       "      <td>1726</td>\n",
       "      <td>Human nature and other sermons</td>\n",
       "      <td>Butler, Joseph</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>THE London Merchant : OR , THE HISTORY OF GEOR...</td>\n",
       "      <td>Drama</td>\n",
       "      <td>1731</td>\n",
       "      <td>The London merchant</td>\n",
       "      <td>Lillo, George</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>MARINA : A PLAY OF THREE ACTS .\\nAs it is Acte...</td>\n",
       "      <td>Drama</td>\n",
       "      <td>1738</td>\n",
       "      <td>Marina</td>\n",
       "      <td>Lillo, George</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>THE FORTUNATE FOUNDLINGS .\\nCHAP .\\nI .\\nConta...</td>\n",
       "      <td>Narrative fiction</td>\n",
       "      <td>1744</td>\n",
       "      <td>The fortunate foundlings</td>\n",
       "      <td>Haywood, Eliza Fowler</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text           category  date  \\\n",
       "0  A TREATISE Concerning the PRINCIPLES OF Human ...           Treatise  1710   \n",
       "1  SERMON I. UPON HUMAN NATURE .\\nROMANS xii .\\n4...           Treatise  1726   \n",
       "2  THE London Merchant : OR , THE HISTORY OF GEOR...              Drama  1731   \n",
       "3  MARINA : A PLAY OF THREE ACTS .\\nAs it is Acte...              Drama  1738   \n",
       "4  THE FORTUNATE FOUNDLINGS .\\nCHAP .\\nI .\\nConta...  Narrative fiction  1744   \n",
       "\n",
       "                                               title                 author  \n",
       "0  A treatise concerning the principles of human ...       Berkeley, George  \n",
       "1                     Human nature and other sermons         Butler, Joseph  \n",
       "2                                The London merchant          Lillo, George  \n",
       "3                                             Marina          Lillo, George  \n",
       "4                           The fortunate foundlings  Haywood, Eliza Fowler  "
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>text</th>\n",
       "      <th>category</th>\n",
       "      <th>date</th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>[The remedy against the troubles of temptation...</td>\n",
       "      <td>book.xml</td>\n",
       "      <td>1508.0</td>\n",
       "      <td>The-remedy-against-the-troubles-of-temptations</td>\n",
       "      <td>Richard-rolle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>The ballade of ane right noble victorius &amp; myg...</td>\n",
       "      <td>book.xml</td>\n",
       "      <td>1508.0</td>\n",
       "      <td>The-ballade-of-ane-right-noble-victorius</td>\n",
       "      <td>William-Dunbar</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>[The noble history of King Ponthus.]Ponthus et...</td>\n",
       "      <td>book.xml</td>\n",
       "      <td>1511.0</td>\n",
       "      <td>The-noble-history-of-King-Ponthus</td>\n",
       "      <td>Henry-watson</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Here begynneth a dialogue betwene the comen se...</td>\n",
       "      <td>letter.xml</td>\n",
       "      <td>1530.0</td>\n",
       "      <td>dialogue-betwene-the-comen-secretary-and-Ielow...</td>\n",
       "      <td>Edward-Gosynhyll</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>A compendious olde treatyse, shewynge howe tha...</td>\n",
       "      <td>book.xml</td>\n",
       "      <td>1530.0</td>\n",
       "      <td>A-compendious-olde-treatyse</td>\n",
       "      <td>Richard-ullerston</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                               text    category  \\\n",
       "0           0  [The remedy against the troubles of temptation...    book.xml   \n",
       "1           1  The ballade of ane right noble victorius & myg...    book.xml   \n",
       "2           2  [The noble history of King Ponthus.]Ponthus et...    book.xml   \n",
       "3           3  Here begynneth a dialogue betwene the comen se...  letter.xml   \n",
       "4           4  A compendious olde treatyse, shewynge howe tha...    book.xml   \n",
       "\n",
       "     date                                              title  \\\n",
       "0  1508.0     The-remedy-against-the-troubles-of-temptations   \n",
       "1  1508.0           The-ballade-of-ane-right-noble-victorius   \n",
       "2  1511.0                  The-noble-history-of-King-Ponthus   \n",
       "3  1530.0  dialogue-betwene-the-comen-secretary-and-Ielow...   \n",
       "4  1530.0                        A-compendious-olde-treatyse   \n",
       "\n",
       "              author  \n",
       "0      Richard-rolle  \n",
       "1     William-Dunbar  \n",
       "2       Henry-watson  \n",
       "3   Edward-Gosynhyll  \n",
       "4  Richard-ullerston  "
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_old.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_old = pd.read_csv(\"fin_df.csv\")\n",
    "\n",
    "df_fin2  = pd.concat([df_old, df], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_fin2.to_csv(\"fin2.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 9633 entries, 0 to 9632\n",
      "Data columns (total 6 columns):\n",
      " #   Column      Non-Null Count  Dtype  \n",
      "---  ------      --------------  -----  \n",
      " 0   Unnamed: 0  9300 non-null   float64\n",
      " 1   text        8199 non-null   object \n",
      " 2   category    8199 non-null   object \n",
      " 3   date        8199 non-null   object \n",
      " 4   title       8199 non-null   object \n",
      " 5   author      8185 non-null   object \n",
      "dtypes: float64(1), object(5)\n",
      "memory usage: 451.7+ KB\n"
     ]
    }
   ],
   "source": [
    "df_fin2.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Court Case dataset - Using this was cancled as there were too many spelling errors in the short hand typing\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4 Julii 1569 super libello ex parte Jane Clerck con Richard Churcher dat\n",
      "\n",
      "Johannes Wilkins de Sora Winton glasier ubi moram fecit a nativitate &c. libere &c. testis &c. etatis xxv annos dicit quod dictam Clerck per viii annos et dict Churcher et uxor sua tam diu bene novit\n",
      "\n",
      "Ad 1 articlum libelli refert se ad iura\n",
      "\n",
      "Ad 2 deponit et dicit that abowt Ester last upon a working day (ut recolit) this deponent dwelling with Gillam glasier was requested by the saide Jane clerck to help her Mayde to the water with a buck. And when they came right over the saide Churchers dore with ther buck of clothes the saide churchers wiff had left wasshing. And was in her howse where upon James Godslens wiff his contest was standing upon a borde by the water redy to wash the clothes which this deponent & she brought forth. And then Churchers wiff came owte agayne from her howse with ii kettles And sayde to Mres Clerck then being ther / dost thow thrust my mayde from the wasshing place & imediatly went to the saide Godslens wiff & thrust her into the water. And then Mres Clerck demanded of the sayde Churchers wiff why she did soe. / And she sware that she wolde thurst her in allso, and which imediatly she did as sone as the saide Mrs Clerck sett her fote uppon the borde & thrust her twise back And sayde to the saide Mres Clerck, Thow arte no honest woman & with that the said Richard Churcher her husband came owte & saide to this deponent what have you to do with this / And sayde unto the saide Mres Clerck then being all wett, thow arte an unhonest woman And she asked what dishonesty know you by me And then this deponent wente his way et aliter nescit deponere\n",
      "\n",
      "Ad 3 nescit deponere.\n",
      "\n",
      "Ad 4 verum est.\n",
      "\n",
      "Ad 5 credit articulum.\n",
      "\n",
      "Ad 6 predeposita per eum sunt vera &c.\n",
      "\n",
      "Idem super Interrogtoriis\n",
      "\n",
      "Ad 1 satisfact est.\n",
      "\n",
      "Ad 2 respondet that he is owte of everie bodies debte & otherwese very litle worth.\n",
      "\n",
      "Ad 3 nescit aliter respondere qui predeposunt\n",
      "\n",
      "Ad 4 respondet quod prous dicit et audivit.\n",
      "\n",
      "Ad 5 venit rogatu Georgii Clerck mariti dicte Clerck\n",
      "\n",
      "Ad 6 satisfact est aliter nescit respondere\n",
      "\n",
      "Ad 7 respondet quod indifferens\n",
      "\n",
      "Ad 8 9 respondet negative\n",
      "\n",
      "Ad 10 respondet that once L. Magdalene hill this respondent did weare his armor et aliter respondet negative\n",
      "\n",
      "Ad 11 respondet quod credit articulum ad deponend veritatem.\n",
      "\n",
      "Ad 12 respondet that he dwelleth with Gillam Glasier.\n",
      "\n",
      "Ad 13 satisfact est\n",
      "\n",
      "his mark\n",
      "\n",
      "Deposition 2\n",
      "4 Julii 1569 super libello ex parte Jane Clerck con Richard Churcher dat\n",
      "\n",
      "Rabadge Godsland uxor Jacobi Godsland de Civitate Winton etatis xxviii annos libere &c. testis &c. dicit quod dict Clerck per iiies annos et dict Chercher et uxor sua per x. annos bene novit\n",
      "\n",
      "Ad 1 articulum libelli refert se ad iura\n",
      "\n",
      "Ad 2 deponitet dicit that abowt a fortnight afore Ester last ut recolit this deponent & the saide John Wilkens her contest at Mres Clercks request carried a bucke of Clothes to the wasshing place right over the saide Churcher dore And when they came thither the sayde Churchers mayde had done wasshing of her Clothes And then Mres Clerck layde downe a borde to wash upon And this deponent stode upon the borde and immediatly the saide Churchers wiff came owte of her howse with a cople of kettles in her hand to the wasshing place, and being greved with the saide Mrs Clerck for wasshing ther she thrust this deponent in to the water And then with Mres Clerck asked her why she did soe, And the saide Churchers wiff awnswered & swore that she wold thrust her in allso & so she ded, and sayde she the saide Mres Clerck was no honest woman, And then the saide Richard Churcher came owte of his howse And taking his wiffes parte in conclusion saide the saide Mres Clerck was no honest woman. Why quod Mres Clerck what unhonesty do you know by me. et aliter nescit deponere.\n",
      "\n",
      "Ad 3 nescit deponere.\n",
      "\n",
      "Ad 4 verum est.\n",
      "\n",
      "Ad 5 refert se ad Iura.\n",
      "\n",
      "Ad ult predeposita per eum sunt vera &c.\n",
      "\n",
      "Eadem super Interrogtoriis\n",
      "\n",
      "Ad 1 satisfact est\n",
      "\n",
      "Ad 2 non concernit eam.\n",
      "\n",
      "Ad 3 4 satisfact est\n",
      "\n",
      "Ad 5 venit rogatu Mr Clerck marite dicte Clerck.\n",
      "\n",
      "Ad 6 satisfact est\n",
      "\n",
      "Ad 7 est indifferens\n",
      "\n",
      "Ad 8 respondet that she is ther at Mr Clerck once in a weke to helpe wash a bucke et aliter respondet negative\n",
      "\n",
      "Ad 9 respondet negative.\n",
      "\n",
      "Ad 10 11 12 non concernit eam.\n",
      "\n",
      "Adult satisfact est.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "lst = os.listdir(r\"C:\\Users\\rhira\\Documents\\TextAge\\courtcase\")\n",
    "for fl in lst:\n",
    "    splt = fl.split(\"_\")\n",
    "    yr = splt[0]\n",
    "    aut = splt[1]\n",
    "    title = splt[2]\n",
    "    file_path = r\"C:\\Users\\rhira\\Documents\\TextAge\\courtcase\" + \"\\\\\" + fl\n",
    "    with open(file_path, 'r',encoding=\"utf-8\") as file:\n",
    "        content = file.read()\n",
    "        print(content)\n",
    "        break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}