ModMurray.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "134048\t5852\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "12"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#!/usr/bin/python\n",
    "  \n",
    "import sys\n",
    "import re\n",
    "\n",
    "ethtable = {}\n",
    "curlemma = ''\n",
    "modwords = 0\n",
    "basewords = 0\n",
    "\n",
    "simpsub = {}\n",
    "simpsub['raiment'] = \"clothing\"\n",
    "simpsub['maiden'] = \"young woman\"\n",
    "simpsub['maidens'] = \"young women\"\n",
    "simpsub['abide'] = \"remain\"\n",
    "simpsub['adread'] = \"afraid\"\n",
    "simpsub['aforetime'] = \"in the past\"\n",
    "simpsub['afrighted'] = \"terrified\"\n",
    "simpsub['aftertime'] = \"future\"\n",
    "simpsub['Albeit'] = \"Although\"\n",
    "simpsub['albeit'] = \"although\"\n",
    "simpsub['amid'] = \"among\"\n",
    "simpsub['anigh'] = \"near\"\n",
    "simpsub['anywise'] = \"in any way\"\n",
    "simpsub['aright'] = \"correctly\"\n",
    "simpsub['asunder'] = \"apart\"\n",
    "simpsub['athrob'] = \"throbbing\"\n",
    "simpsub['aught'] = \"anything\"\n",
    "simpsub['bade'] = \"commanded\"\n",
    "simpsub['badest'] = \"commanded\"\n",
    "simpsub['bane'] = \"harm\"\n",
    "simpsub['baneful'] = \"harmful\"\n",
    "simpsub['bare'] = \"bore\"\n",
    "simpsub['besought'] = \"begged\"\n",
    "simpsub['begat'] = \"fathered\"\n",
    "simpsub['bid'] = \"command\"\n",
    "simpsub['bidden'] = \"commanded\"\n",
    "simpsub['bids'] = \"commands\"\n",
    "simpsub['brake'] = \"broke\"\n",
    "simpsub['builded'] = \"built\"\n",
    "simpsub['car'] = \"chariot\"\n",
    "simpsub['chine'] = \"back-meat\"\n",
    "simpsub['clove'] = \"cut\"\n",
    "simpsub['cloven'] = \"cut\"\n",
    "simpsub['convoy'] = \"passage\"\n",
    "simpsub['couch'] = \"bed\"\n",
    "simpsub['coursing'] = \"racing\"\n",
    "simpsub['dainties'] = \"fine foods\"\n",
    "simpsub['deem'] = \"think\"\n",
    "simpsub['deemed'] = \"thought\"\n",
    "simpsub['didst'] = \"did\"\n",
    "simpsub['dight'] = \"adorned\"\n",
    "simpsub['doth'] = \"does\"\n",
    "simpsub['dost'] = \"do\"\n",
    "simpsub['drave'] = \"drove\"\n",
    "simpsub['dravest'] = \"drove\"\n",
    "simpsub['dusky'] = \"dark\"\n",
    "simpsub['dwelt'] = \"dwelled\" # keep this rather than \"lived\" as \"dwelled\" is less ambiguous\n",
    "simpsub['dykes'] = \"bridges\"\n",
    "simpsub['ere'] = \"before\"\n",
    "simpsub['essayed'] = \"tried\"\n",
    "simpsub['everywhither'] = \"everywhere\"\n",
    "simpsub['foeman'] = \"enemy\"\n",
    "simpsub['foemen'] = \"enemies\"\n",
    "simpsub['folk'] = \"people\"\n",
    "simpsub['fordone'] = \"done in\"\n",
    "simpsub['therefrom'] = \"done in\"\n",
    "simpsub['froward'] = \"willful\"\n",
    "simpsub['gainsay'] = \"deny\"\n",
    "simpsub['gat'] = \"got\"\n",
    "simpsub['hadst'] = \"had\"\n",
    "simpsub['hapless'] = \"unhappy\"\n",
    "simpsub['hath'] = \"has\"\n",
    "simpsub['hast'] = \"have\"\n",
    "simpsub['hasted'] = \"hastened\"\n",
    "simpsub['hearken'] = \"listen\"\n",
    "simpsub['hearkened'] = \"listened\"\n",
    "simpsub['helm'] = \"helmet\"\n",
    "simpsub['hereafter'] = \"afterwards\"\n",
    "simpsub['heretofore'] = \"until now\"\n",
    "simpsub['Heretofore'] = \"Until now\"\n",
    "simpsub['hither'] = \"here\" #NB: hither is more preccise.\n",
    "simpsub['house-dame'] = 'household-manager'\n",
    "simpsub['housewife'] = \"woman in charge of the house\" #housewife always corresponds to tamih -- i lose the one-word to one-word equivalence by chosing this phrase\n",
    "simpsub['Howbeit'] = \"However\"\n",
    "simpsub['howbeit'] = \"however\"\n",
    "simpsub['kine'] = \"cattle\"\n",
    "simpsub['quaff'] = \"drink\"\n",
    "simpsub['quaffs'] = \"drinks\"\n",
    "simpsub['straightly'] = \"firmly\"\n",
    "simpsub['thee'] = \"you\"\n",
    "simpsub['thou'] = \"you\"\n",
    "simpsub['thy'] = \"your\"\n",
    "simpsub['thyself'] = \"yourself\"\n",
    "simpsub['ye'] = \"you\"\n",
    "simpsub['haply'] = \"by chance\"\n",
    "simpsub['kinsfolk'] = \"family\"\n",
    "simpsub['leech'] = \"doctor\"\n",
    "#simpsub['lo'] = \"indeed\"\n",
    "simpsub['loathly'] = \"disgusting\"\n",
    "simpsub['meaner'] = \"inferior\"\n",
    "simpsub['meseems'] = \"I think\"\n",
    "simpsub['methinks'] = \"I think\"\n",
    "simpsub['naught'] = \"nothing\"\n",
    "simpsub['nigh'] = \"near\"\n",
    "simpsub['nowise'] = \"in no way\"\n",
    "simpsub['oft'] = \"often\"\n",
    "simpsub['perchance'] = \"by chance\"\n",
    "simpsub['perforce'] = \"of necessity\"\n",
    "simpsub['reck'] = \"care\"\n",
    "simpsub['recks'] = \"cares\"\n",
    "simpsub['rede'] = \"counsel\"\n",
    "simpsub['saidst'] = \"said\"\n",
    "simpsub['knave'] = \"scoundrel\"\n",
    "simpsub['shalt'] = \"shall\"\n",
    "simpsub['shew'] = \"show\"\n",
    "simpsub['shewed'] = \"showed\"\n",
    "simpsub['shouldst'] = \"should\"\n",
    "simpsub['slay'] = \"kill\"\n",
    "simpsub['slew'] = \"killed\"\n",
    "simpsub['smite'] = \"strike\"\n",
    "simpsub['smitest'] = \"strike\"\n",
    "simpsub['smiting'] = \"striking\"\n",
    "simpsub['smitten'] = \"struck\"\n",
    "simpsub['smote'] = \"struck\"\n",
    "simpsub['somewhat'] = \"something\"\n",
    "simpsub['sooth'] = \"truth\"\n",
    "simpsub['spake'] = \"spoke\"\n",
    "simpsub['stinting'] = \"lack\"\n",
    "simpsub['stranger-folk'] = \"strangers\"\n",
    "simpsub['succour'] = \"help\"\n",
    "simpsub['surfeit'] = \"all too much\"\n",
    "simpsub['suspecting'] = \"suspicious\"\n",
    "simpsub['tarried'] = \"waited\"\n",
    "simpsub['tarry'] = \"wait\"\n",
    "simpsub['thine'] = \"your\"\n",
    "simpsub['thither'] = \"there\" #thither is more precise\n",
    "simpsub['Thrice'] = \"Three times\"\n",
    "simpsub['thrice'] = \"three times\"\n",
    "simpsub['travailing'] = \"laboring\"\n",
    "simpsub['Troy-land'] = \"Troy\"\n",
    "simpsub['twain'] = \"two\"\n",
    "simpsub['ungentle'] = \"harsh\"\n",
    "simpsub['unto'] = \"to\"\n",
    "simpsub['unwashn'] = \"unwashed\"\n",
    "simpsub['uprose'] = \"rose up\"\n",
    "simpsub['Verily'] = \"Truly\"\n",
    "simpsub['verily'] = \"truly\"\n",
    "simpsub['vex'] = \"anger\"\n",
    "simpsub['vexed'] = \"angered\"\n",
    "simpsub['vouchsafe'] = \"promise\"\n",
    "simpsub['vouchsafed'] = \"promised\"\n",
    "simpsub['vouchsafes'] = \"promises\"\n",
    "simpsub['wallet'] = \"leather sack\"\n",
    "simpsub['ware'] = \"aware\"\n",
    "simpsub['wast'] = \"were\"\n",
    "simpsub['waxed'] = \"became\"\n",
    "simpsub['waxes'] = \"grows\"\n",
    "simpsub['ween'] = \"believe\"\n",
    "simpsub['whatsoe\\'er'] = \"whatever\"\n",
    "simpsub['whenso'] = \"whenever\"\n",
    "simpsub['wherefore'] = \"for which reason\"\n",
    "simpsub['Wherefrom'] = \"From which\"\n",
    "simpsub['wherefrom'] = \"from which\"\n",
    "simpsub['Wherewith'] = \"with which\"\n",
    "simpsub['wherein'] = \"in which\"\n",
    "simpsub['Wherein'] = \"In which\"\n",
    "simpsub['whereon'] = \"on which\"\n",
    "simpsub['Whereon'] = \"On which\"\n",
    "simpsub['Whither'] = \"Where\"\n",
    "simpsub['whither'] = \"where\"\n",
    "simpsub['whenso'] = \"whenever\"\n",
    "simpsub['whosoever'] = \"whoever\"\n",
    "simpsub['whomsoever'] = \"whoever\"\n",
    "simpsub['wilt'] = \"will\"\n",
    "simpsub['wist'] = \"knew\"\n",
    "simpsub['wooer'] = \"suitor\"\n",
    "simpsub['wooers'] = \"suitors\"\n",
    "simpsub['wont'] = \"accustomed\"\n",
    "simpsub['would\\'st'] = \"accustomed\"\n",
    "simpsub['wroth'] = \"angry\"\n",
    "simpsub['Ye'] = \"You\"\n",
    "simpsub['ye'] = \"you\"\n",
    "\n",
    "bigrams = {}\n",
    "bigrams['a foeman'] = \"an enemy\"\n",
    "bigrams['am come'] = \"have come\"\n",
    "bigrams['any wise'] = \"any way\"\n",
    "bigrams['arrayed him'] = \"arrayed himself\"\n",
    "bigrams['art come'] = \"have come\"\n",
    "bigrams['bathe him'] = 'bathe himself'\n",
    "bigrams['in sunder'] = \"apart\"\n",
    "bigrams['baser fellow'] = \"lesser man\"\n",
    "bigrams['base fellow'] = \"lesser man\"\n",
    "bigrams['be minded'] = 'wish'\n",
    "bigrams['bethought him'] = \"thought\"\n",
    "bigrams['bethought them'] = \"thought\"\n",
    "bigrams['bethought thee'] = \"thought\"\n",
    "bigrams['clad him'] = \"clad himself\"\n",
    "bigrams['did on'] = \"put on\"\n",
    "bigrams['do thou'] = \"\"\n",
    "bigrams['do ye'] = \"\"\n",
    "bigrams['do you'] = \"you\"\n",
    "bigrams['evil case'] = \"bad situation\"\n",
    "bigrams['fear me'] = \"fear\"\n",
    "bigrams['from without'] = \"from outside\"\n",
    "bigrams['go thou'] = \"go\"\n",
    "bigrams['his wont'] = \"his habit\"\n",
    "bigrams['insatiate fellow'] = \"insatiable man\"\n",
    "bigrams['is come'] = \"has come\"\n",
    "bigrams['maid unwed'] = \"unmarried girl\"\n",
    "bigrams['unwedded maid'] = \"unmarried girl\"\n",
    "bigrams['made answer'] = \"answered\"\n",
    "bigrams['other fellow'] = \"other man\"\n",
    "bigrams['rouse thee'] = \"rouse yourself\"\n",
    "bigrams['sat him'] = \"sat himself\"\n",
    "bigrams['sit thou'] = \"sit yourself\"\n",
    "bigrams['such wise'] = \"such a way\"\n",
    "bigrams['this fellow'] = \"this man\"\n",
    "bigrams['to surfeit'] = \"to a bellyful\"\n",
    "bigrams['travail sore'] = \"suffer badly\"\n",
    "bigrams['wax wroth'] = \"become angry\"\n",
    "bigrams['what time'] = \"when\"\n",
    "bigrams['young maiden'] = \"young girl\"\n",
    "bigrams['cares not'] = 'does not care'\n",
    "bigrams['concerns not'] = 'does not concern'\n",
    "bigrams['lives not'] = 'does not live'\n",
    "bigrams['perishes not'] = 'does not perish'\n",
    "bigrams['bridles not'] = 'does not bridle'\n",
    "bigrams['deigns not'] = 'does not deign'\n",
    "bigrams['comes not'] = 'does not come'\n",
    "bigrams['recks not'] = 'does not think'\n",
    "bigrams['like to'] = 'like'\n",
    "bigrams['lives not'] = 'does not live'\n",
    "bigrams['escapes not'] = 'does not escape'\n",
    "bigrams['knows not'] = 'does not know'\n",
    "bigrams['leaves not'] = 'does not leave'\n",
    "bigrams['beseems not'] = 'it is not fitting'\n",
    "ngramsub = {}\n",
    "bigrams[\"a meaner\"] = \"an inferior\"\n",
    "bigrams[\"be not\"] = \"do not be\"\n",
    "bigrams[\"Be thou\"] = \"Be\"\n",
    "bigrams[\"for that\"] = \"for\"\n",
    "bigrams[\"from out\"] = \"out of\"\n",
    "bigrams[\"ful many\"] = \"many\"\n",
    "bigrams[\"gave ear\"] = \"listened\"\n",
    "bigrams[\"give ear\"] = \"listen\"\n",
    "bigrams[\"good cheer\"] = \"good food\"\n",
    "bigrams[\"hard by\"] = \"nearby\"\n",
    "bigrams[\"holden of\"] = \"held by\"\n",
    "bigrams[\"is fain\"] = \"wants\"\n",
    "bigrams[\"like unto\"] = \"like\"\n",
    "bigrams[\"make essay\"] = \"attempt\"\n",
    "bigrams[\"make prayer\"] = \"pray\"\n",
    "bigrams[\"mine eyes\"] = \"my eyes\"\n",
    "bigrams[\"mine own\"] = \"my own\"\n",
    "bigrams[\"must needs\"] = \"must\"\n",
    "bigrams[\"near to\"] = \"near\"\n",
    "bigrams[\"no wise\"] = \"no way\"\n",
    "bigrams[\"sate him\"] = \"sat\"\n",
    "bigrams[\"was fain\"] = \"wanted\"\n",
    "bigrams[\"this wise\"] = \"this way\"\n",
    "bigrams[\"thou longest\"] = \"you long\"\n",
    "bigrams[\"to wax\"] = \"to grow\"\n",
    "bigrams['didst promise'] = 'promised'\n",
    "bigrams['didst leave'] = 'left'\n",
    "bigrams['didst hear'] = 'heard'\n",
    "bigrams['didst come'] = 'came'\n",
    "bigrams['didst blind'] = 'blinded'\n",
    "bigrams['didst take'] = 'took'\n",
    "bigrams['didst slay'] = 'killed'\n",
    "bigrams['didst send'] = 'sent'\n",
    "bigrams['didst name'] = 'named'\n",
    "bigrams['didst lie'] = 'lied'\n",
    "bigrams['didst lay'] = 'lay'\n",
    "bigrams['didst bid'] = 'commanded'\n",
    "bigrams['didst bear'] = 'bore'\n",
    "bigrams['didst yield'] = 'yielded'\n",
    "bigrams['didst wish'] = ' wished'\n",
    "bigrams['didst win'] = 'won'\n",
    "bigrams['didst watch'] = ' watched'\n",
    "bigrams['didst wander'] = ' wandered'\n",
    "bigrams['didst vanquish'] = 'vanquished'\n",
    "bigrams['didst speak'] = 'spoke'\n",
    "bigrams['didst sit'] = 'sat'\n",
    "bigrams['didst set'] = 'set'\n",
    "bigrams['didst save'] = 'saved'\n",
    "bigrams['didst sail'] = ' sailed'\n",
    "bigrams['didst rouse'] = 'roused'\n",
    "bigrams['didst rend'] = 'rent'\n",
    "bigrams['didst rear'] = ' reared'\n",
    "bigrams['didst reach'] = ' reached'\n",
    "bigrams['didst rage'] = 'raged'\n",
    "bigrams['didst pray'] = ' prayed'\n",
    "bigrams['didst moor'] = ' moored'\n",
    "bigrams['didst make'] = 'made'\n",
    "bigrams['didst long'] = ' longed'\n",
    "bigrams['didst lead'] = 'led'\n",
    "bigrams['didst honour'] = ' honoured'\n",
    "bigrams['didst honor'] = ' honored'\n",
    "bigrams['didst herd'] = ' herded'\n",
    "bigrams['didst hearken'] = ' hearkened'\n",
    "bigrams['didst hang'] = 'hung'\n",
    "bigrams['didst guard'] = ' guarded'\n",
    "bigrams['didst grasp'] = ' grasped'\n",
    "bigrams['didst follow'] = ' followed'\n",
    "bigrams['didst entertain'] = 'entertained'\n",
    "bigrams['didst endure'] = 'endured'\n",
    "bigrams['didst devise'] = 'devised'\n",
    "bigrams['didst cheer'] = ' cheered'\n",
    "bigrams['didst call'] = ' called'\n",
    "bigrams['didst bring'] = 'brought'\n",
    "bigrams['didst boast'] = ' boasted'\n",
    "bigrams['didst beguile'] = 'beguiled'\n",
    "bigrams['was come'] = \"had come\"\n",
    "bigrams['what time'] = \"when\"\n",
    "compsub = {}\n",
    "compsub['abode'] = 'a'\n",
    "compsub['aye'] = 'a'\n",
    "compsub['baneful'] = 'a'\n",
    "compsub['bethink'] = 'a'\n",
    "compsub['dread'] = 'a'\n",
    "compsub['demesne'] = 'τέμενος' #separate land, of gods and of kinds\n",
    "compsub['essay'] = 'a'\n",
    "compsub['fellow'] = 'a'\n",
    "compsub['forsooth'] = 'a'\n",
    "compsub['Forthwith'] = 'a'\n",
    "compsub['forthwith'] = 'a'\n",
    "compsub['hence'] = 'a'\n",
    "compsub['hie'] = 'a'\n",
    "compsub['laden'] = 'a'\n",
    "compsub['Lo'] = 'a'\n",
    "compsub['lo'] = 'a'\n",
    "compsub['meed'] = 'a'\n",
    "compsub['Nay'] = 'a'\n",
    "compsub['nay'] = 'a'\n",
    "compsub['pratest'] = 'a'\n",
    "compsub['sate'] = 'a'\n",
    "compsub['sore'] = 'a'\n",
    "compsub['Thereat'] = 'a'\n",
    "compsub['thereat'] = 'a'\n",
    "compsub['therein'] = 'a'\n",
    "compsub['thereof'] = 'a'\n",
    "compsub['Thereto'] = 'a'\n",
    "compsub['thereto'] = 'a'\n",
    "compsub['therewithal'] = 'a'\n",
    "compsub['whence'] = 'a'\n",
    "compsub['wherein'] = 'a'\n",
    "compsub['whereof'] = 'a'\n",
    "compsub['withal'] = 'a'\n",
    "compsub['wonderous'] = 'a'\n",
    "compsub['wonderously'] = 'a'\n",
    "compsub['wrought'] = 'a'\n",
    "compsub['Yea'] = 'a'\n",
    "compsub['yea'] = 'a'\n",
    "compsub['yon'] = 'a'\n",
    "compsub['yonder'] = 'a'\n",
    "\n",
    "\n",
    "def do_counts(s):\n",
    "  global modwords\n",
    "  global basewords\n",
    "  works = s\n",
    "  while(re.search('<mod',works)):\n",
    "    modwords = modwords + 1\n",
    "    works = re.sub('<mod[^>]+>',' ',works,1)\n",
    "  works = s\n",
    "  works = re.sub('<[^>]+>',' ',works)\n",
    "  while(re.search('[A-Za-z\\']+',works)):\n",
    "    basewords = basewords + 1\n",
    "    works = re.sub('[A-Za-z\\']+',' ',works,1)\n",
    "\n",
    "f = open(\"eth-3rd-pers.txt\",\"r\")\n",
    "for l in f:\n",
    "  l = re.sub(\":[^\\t]+\",\"\",l)\n",
    "  args = l.split('\\t')\n",
    "  ethtable[args[0]] = args[1]\n",
    "  #print(args[0],ethtable[args[0]])\n",
    "f.close()\n",
    "\n",
    "f = open(\"est-2nd-pers.txt\",\"r\")\n",
    "for l in f:\n",
    "  l = re.sub(\":[^\\t]+\",\"\",l)\n",
    "  args = l.split('\\t')\n",
    "  ethtable[args[0]] = args[1]\n",
    "  #print(args[0],ethtable[args[0]])\n",
    "f.close()\n",
    "\n",
    "V_excepts = {}\n",
    "V_excepts[\"fled not\"] = 'did not flee'\n",
    "V_excepts[\"found not\"] = 'did not find'\n",
    "\n",
    "V_s_excepts = {}\n",
    "V_s_excepts['is not'] = 1\n",
    "V_s_excepts['does not'] = 1\n",
    "V_s_excepts['was not'] = 1\n",
    "V_s_excepts['has not'] = 1\n",
    "V_s_excepts['as not'] = 1\n",
    "V_s_excepts['us not'] = 1\n",
    "def V_not(s):\n",
    "\n",
    "  works = s\n",
    "  while( re.search('\\\\b([a-z]+s[ ]not)\\\\b',works)):\n",
    "   m = re.search('\\\\b([a-z]+s[ ]not)\\\\b',works)\n",
    "   if( m and not m[1] in V_s_excepts):\n",
    "    sys.stderr.write(m[1]+'\\n')\n",
    "   works = re.sub('\\\\b([a-z]+s[ ]not)\\\\b',' ',works,1)\n",
    "  for foo in V_excepts:\n",
    "    if( re.search('\\\\b' + foo + '\\\\b',s)):\n",
    "     s = re.sub('\\\\b' + foo + '\\\\b','<mod n=\"V_not:'+foo+'\">'+V_excepts[foo]+\"</mod>\",s)\n",
    "     return(s)\n",
    "  works = s\n",
    "  works = re.sub('<mod[^>]+>[^<]+',' ',works)\n",
    "  while(re.search('\\\\b([a-z]+)ed[ ]+not\\\\b',works)):\n",
    "     m = re.search('\\\\b([a-z]+)(ed)[ ]+not\\\\b',works)\n",
    "     sval = m[1] + m[2] + '[ ]+not\\\\b'\n",
    "     subv = '<mod n=\"V_not:' + foo + '\">did not ' + m[1] + '</mod>'\n",
    "     s = re.sub('\\\\b' + sval + '\\\\b',subv,s)\n",
    "     works = re.sub('\\\\b' + sval + '\\\\b',' ',works)\n",
    "  return(s)\n",
    "\n",
    "def do_eth(s):\n",
    "   t = re.sub(\"<[^>]+>\",\" \",s)\n",
    "   t = re.sub('[—\\'\\.,;:\"!,\\s\\?]+',\" \",t)\n",
    "   t = re.sub('<mod[^>]+>[^<]+<\\/mod>',' ',t)\n",
    "   checkedwords = {}\n",
    "   args = t.split(\" \")\n",
    "   for a in args:\n",
    "     if(a in checkedwords):\n",
    "      continue\n",
    "     if(a in ethtable):\n",
    "  #     print(\"eth-s\",a,ethtable[a])\n",
    "       s = re.sub('\\\\b' + a + \"\\\\b\",\"<mod n='\" + a + \"'>\" + ethtable[a] + \"</mod>\",s)\n",
    "     if(a in compsub):\n",
    "       s = re.sub('\\\\b' + a + \"\\\\b\",\"<sic>\" + a + \"</sic>\",s)\n",
    "     elif(re.search('^[A-Z]',a) and a.lower() in compsub):\n",
    "       s = re.sub('\\\\b' + a + \"\\\\b\",\"<sic>\" + a + \"</sic>\",s)\n",
    "     if(a in simpsub and a  not in checkedwords):\n",
    "       s = re.sub('\\\\b' + a + \"\\\\b\",'<mod n=\"' + a + ':simpsub\">' +  simpsub[a] + \"</mod>\",s)\n",
    "     elif(re.search('^[A-Z]',a) and a.lower() in simpsub and a  not in checkedwords):\n",
    "       sub = simpsub[a.lower()].capitalize()\n",
    "       s = re.sub('\\\\b' + a + \"\\\\b\",'<mod n=\"' + a + ':simpsub\">' +  sub + \"</mod>\",s)\n",
    "     checkedwords[a] = 1\n",
    "\n",
    "   return(s)\n",
    "\n",
    "trigrams = {}\n",
    "trigrams['am I come'] = 'have I come'\n",
    "trigrams['bethink you of'] = 'call to mind'\n",
    "trigrams['day by day'] = 'each and every day'\n",
    "trigrams['do thou thus'] = 'do as follows'\n",
    "trigrams['fear me lest'] = 'fear lest'\n",
    "trigrams[\"from out of\"] = \"out of\"\n",
    "\n",
    "trigrams['in evil case'] = 'in great stress'\n",
    "trigrams['in despite of'] = 'in spite of'\n",
    "\n",
    "trigrams['on the morrow'] = 'tomorrow'\n",
    "trigrams['on a time'] = 'once'\n",
    "trigrams['on this wise'] = 'in this way'\n",
    "trigrams['of good cheer'] = 'confident'\n",
    "trigrams[\"of a surety\"] = \"surely\"\n",
    "trigrams[\"of a truth\"] = \"truly\"\n",
    "trigrams[\"out on it\"] = \"curse it\"\n",
    "trigrams[\"put on him\"] = \"put on\"\n",
    "trigrams[\"thou ever longest\"] = \"you always long\"\n",
    "trigrams[\"So said he\"] = \"So he spoke\"\n",
    "trigrams[\"were come near\"] = \"were near\"\n",
    "trigrams[\"without the courtyard\"] = \"outside the courtyard\"\n",
    "\n",
    "def do_bigrams(s):\n",
    "   works = s\n",
    "   works = re.sub('<mod[^>]+>[^<]+<\\/mod>',' ',works)\n",
    "   works = re.sub('<[^>]+>', ' ' ,works)\n",
    "\n",
    "   for foo in trigrams:\n",
    "     if( re.search('\\\\b' + foo + '\\\\b',works)):\n",
    "       s = re.sub('\\\\b' + foo + '\\\\b','<mod n=\"trigram:' + foo.upper() + '\">' + trigrams[foo] + '</mod>',s)\n",
    "       works = re.sub('\\\\b' + foo + '\\\\b',' ',works)\n",
    "     if( re.search('\\\\b' + foo.capitalize() + '\\\\b',works)):\n",
    "       s = re.sub('\\\\b' + foo.capitalize() + '\\\\b','<mod n=\"' + foo + '\">' + trigrams[foo].capitalize() + '</mod>',s)\n",
    "       works = re.sub('\\\\b' + foo + '\\\\b',' ',works)\n",
    "   \n",
    "   while(re.search('\\\\b[A-Za-z]+[ ]+[a-z]+',works)):\n",
    "      m = re.search('\\\\b([A-Za-z]+[ ]+[a-z]+)',works)\n",
    "      if(m and m[1] in bigrams):\n",
    "        s = re.sub('\\\\b' + m[1], '<mod n=\"bigram:'+m[1].upper()+'\">'+bigrams[m[1]] + '</mod>',s)\n",
    "      if(m and m[1].lower() in bigrams):\n",
    "        subv =  bigrams[m[1].lower()]\n",
    "        subv = subv.capitalize()\n",
    "        s = re.sub('\\\\b' + m[1], '<mod n=\"bigram:'+m[1].upper()+'\">'+subv + '</mod>',s)\n",
    "      works = re.sub('\\\\b[A-Za-z]+[ ]+([a-z]+)',' \\g<1>',works,1)\n",
    "   return(s)\n",
    "\n",
    "odysseybase = 'atmurray.odyssey-sentalign'\n",
    "odysseyfname = odysseybase + '.xml'\n",
    "odysseyoutfname = odysseybase + '-mod.xml'\n",
    "\n",
    "iliadbase = 'atmurray.iliad-sentalign'\n",
    "iliadinfname = iliadbase + '.xml'\n",
    "iliadoutfname = iliadbase + '-mod.xml'\n",
    "\n",
    "fname = iliadinfname\n",
    "outfname = iliadoutfname\n",
    "\n",
    "fname = odysseyfname\n",
    "outfname = odysseyoutfname\n",
    "\n",
    "f = open(fname)\n",
    "\n",
    "#iliadoutfname = 'atmurray.iliad-sentalign-mod.xml'\n",
    "outf = open(outfname,'w')\n",
    "intext = 0\n",
    "for w in f:\n",
    "    if( re.search(\"<text\",w) ):\n",
    "        intext = 1\n",
    "    if( intext ):\n",
    "        w = re.sub(\"goodly\\\\s+([A-Z])\",\"<mod n=\\\"goodly\\\">brilliant</mod> \\g<1>\",w)\n",
    "        w = re.sub(\"goodly\\\\s+([a-z])\",\"<mod n=\\\"goodly\\\">brilliant</mod> \\g<1>\",w)\n",
    "        w = do_bigrams(w)\n",
    "        w = V_not(w)\n",
    "        w = re.sub(\"<s\\\\b\",\"\\n\\n<s\",w)\n",
    "        w = re.sub(\"^[ \\t]+\",\"\",w)\n",
    "        w = re.sub(\"\\\\be'en\\\\b\",\"even\",w) # I hate this form and I am not even leving a trace of it.\n",
    "     #for a in ngramsub:\n",
    "       #w = re.sub(\"\\\\b\" + a + \"\\\\b\",\"<corr n=\\\"\" + a + \"\\\">\" + ngramsub[a]+'</corr>',w)\n",
    "        w = re.sub(\"was\\\\s+(abated)\",\"<mod n=\\\"was\\\">had</mod> \\g<1>\",w)\n",
    "    w = do_eth(w)\n",
    "    w = re.sub('<mod n=\"thou:simpsub\">you</mod>[\\s]+art\\\\b',\"<corr n=\\\"thou art\\\">you are</corr>\",w)\n",
    "    w = re.sub('\\\\bart[\\s]+<mod n=\"thou:simpsub\">you</mod>',\"<corr n=\\\"art thou\\\">are you</corr>\",w)\n",
    "\n",
    "    workw = w\n",
    "    while( re.search('gram:([A-Z][A-Z][^\"]+)',workw)):\n",
    "      m = re.search('gram:([A-Z][A-Z][^\"]+)',workw)\n",
    "      v1 = m[1]\n",
    "      v2 = m[1].lower()\n",
    "      w = re.sub('gram:' + v1 ,'gram:' + v2,w,1)\n",
    "      workw  = re.sub('gram:[A-Z][A-Z][^\"]+',' ',workw,1)\n",
    "    do_counts(w)\n",
    "    print(w,end='',file=outf)\n",
    "\n",
    "f.close()\n",
    "outf.close()\n",
    "sys.stderr.write(str(basewords) + '\\t' + str(modwords) + '\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "iliadbase\r\n"
     ]
    }
   ],
   "source": [
    "!echo iliadbase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}