textpro.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
# Comparable text miner

# Description 
Comparable document miner: Arabic-English morphological analysis, text processing, n-gram features extraction, POS tagging, dictionary translation, documents alignment, corpus information, text classification, tf-idf computation, text similarity computation, HTML documents cleaning, and others. 

This code is implemented by Motaz SAAD (motaz.saad@gmail.com) during his PhD work. The PhD thesis is available at: https://sites.google.com/site/motazsite/Home/publications/saad_phd.pdf

Motaz Saad. Mining Documents and Sentiments in Cross-lingual Context. PhD thesis, Université de Lorraine, January 2015.

This code processes Arabic and English text. To use this software, load it as follows:

import imp
tp = imp.load_source('textpro', 'textpro.py')

Then, you can use functions as follows:

clean_text = process_text(text)

# Dependencies
This software depends on the following python packages scipy, numpy, nltk, sklearn, bs4. Please make sure that they are installed before using this software. 

# References
This software uses the following resources:
- Arabic stopwords: http://www.ranks.nl/stopwords/arabic 
- Open Multilingual WordNet (OMW) dictionaries http://compling.hss.ntu.edu.sg/omw/ The references of OMW are listed below:
	- Francis Bond and Kyonghee Paik (2012), A survey of wordnets and their licenses In Proceedings of the 6th Global WordNet Conference (GWC 2012). Matsue. 64–71.
	- Francis Bond and Ryan Foster (2013), Linking and extending an open multilingual wordnet. In 51st Annual Meeting of the Association for Computational Linguistics: ACL-2013. Sofia. 1352–1362. 

- ISRI Arabic Stemmer, which is a rooting algorithm for Arabic text. The reference of ISRI Arabic Stemmer is below:
	- Taghva, K., Elkoury, R., and Coombs, J. 2005. Arabic Stemming without a root dictionary. Information Science Research Institute. University of Nevada, Las Vegas, USA.
 

- This software modifies the ISRI Arabic Stemmer to perform light stemming for Arabic words. 

'''

import sys
import os.path
import string
import collections

import nltk
from nltk.corpus import stopwords
from nltk.stem.wordnet import WordNetLemmatizer
from nltk import word_tokenize, pos_tag
from nltk.util import ngrams
from nltk.corpus import wordnet as omw # open multilingual wordnet
from nltk.stem.isri import ISRIStemmer

from gensim import corpora, models, similarities, matutils
from joblib import Parallel, delayed
#Parallel(n_jobs=4)(delayed(func_name)(arg1, arg2, ...) for i in range(n))

import pyprind

import sqlite3


import sklearn
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score

from random import shuffle
from scipy.spatial import distance
import math

from bs4 import BeautifulSoup

import logging
logging.basicConfig(format='%(levelname)s : %(asctime)s : %(message)s', level=logging.INFO)


import re
whiteSpace = re.compile(r'\s+')

#import imp
#tp = imp.load_source('textpro', 'textpro.py')


x_seperator = '\nXXXXXXX\n' # define document separator (7 Xs). This separator is used when all the docs are in one file (a corpus file)

##################################################################

# Arabic diacritics
arabic_punct = ''' ` ÷ × ؛ < > _ ( ) * & ^ % ] [ ـ ، / : " ؟ . , ' { } ~ ¦ + | !  ”  …  “ –   ـ  '''
arabic_diacritics = '''  َ     ُ       ِ      ّ       ً      ٌ       ٍ      ْ     '''

arabic_punctUnicode = arabic_punct.decode('utf-8')
arabic_punct = arabic_punct.split()
arabic_punctUnicode = arabic_punctUnicode.split()

arabic_diacritics_unicode = arabic_diacritics.decode('utf-8')
arabic_diacritics = arabic_diacritics.split()
arabic_diacritics_unicode = arabic_diacritics_unicode.split()

english_punt = list(string.punctuation)
english_puntUnicode = list(string.punctuation.decode('utf-8'))

# Arabic punctuations and dicritis + English and Arabic
punctuations = set( english_punt + english_puntUnicode + arabic_punct + arabic_punctUnicode + arabic_diacritics + arabic_diacritics_unicode)

englishStopWords = stopwords.words('english')
englishStopWords_unicode = ' '.join(englishStopWords).decode('utf-8').split()

# Arabic stopwords. This list are obtained from http://www.ranks.nl/stopwords/arabic 

asw = open('stopwords.txt').read()

aswUinicode = asw.decode('utf-8')
arabicStopWords =  asw.split() + aswUinicode.split()

# Arabic stopwords. This list are obtained from https://code.google.com/p/stop-words/

asw2 = ''

# Arabic and English stopwords
all_stopwords = set(englishStopWords + englishStopWords_unicode + arabicStopWords)



###################################################################################
###################################################################################

# remove punctcutions
def remove_punct(word):
	for c in word: return ''.join(ch for ch in word if not ch in punctuations) # remove punctuation
###################################################################################

# takes a string of text and returns the word list (tonkized words)
# processing includes: removing diacritics and punctcutions, removing stopwords, and tokenizing
def process_text(text, removePunct=True, removeSW=True, removeNum=False):
	text = remove_diacritics(text)# remove arabic diacritics
	word_list = nltk.tokenize.wordpunct_tokenize(text.lower())
	if removePunct:
		word_list = [ w for w in word_list if not w in punctuations ]
		word_list = [ remove_punct(w) for w in word_list ]
	if removeSW: word_list = [ w for w in word_list if not w in all_stopwords ]
	if removeNum: word_list = [ w for w in word_list if not w.isdigit() ]
	word_list = [ w for w in word_list if w]# remove empty words

	return word_list
###################################################################################

# remove arabic diacritics
def remove_diacritics(text):
	arstemmer = ISRIStemmer()
	result = arstemmer.norm(text, num=1) #  remove diacritics which representing Arabic short vowels
	return result
###################################################################################

"""
ISRI Arabic Stemmer

The algorithm for this stemmer is described in:

Taghva, K., Elkoury, R., and Coombs, J. 2005. Arabic Stemming without a root dictionary.
Information Science Research Institute. University of Nevada, Las Vegas, USA.

The Information Science Research Institute’s (ISRI) Arabic stemmer shares many features
with the Khoja stemmer. However, the main difference is that ISRI stemmer does not use root
dictionary. Also, if a root is not found, ISRI stemmer returned normalized form, rather than
returning the original unmodified word.

Additional adjustments were made to improve the algorithm:

1- Adding 60 stop words.
2- Adding the pattern (تفاعيل) to ISRI pattern set.
3- The step 2 in the original algorithm was normalizing all hamza. This step is discarded because it
increases the word ambiguities and changes the original root.

"""


# takes a word list and returns the root for each Arabic words
def getRootAr(word_list):
	result = []
	arstemmer = ISRIStemmer()
	for word in word_list: result.append(arstemmer.stem(word))
	return ' '.join(result)
###################################################################################

# Arabic light stemming for Arabic text
# takes a word list and perform light stemming for each Arabic words
def lightStemAr(word_list):
	result = []
	arstemmer = ISRIStemmer()
	for word in word_list:
		word = arstemmer.norm(word, num=1)  #  remove diacritics which representing Arabic short vowels  
		if not word in arstemmer.stop_words:   # exclude stop words from being processed
			word = arstemmer.pre32(word)        # remove length three and length two prefixes in this order
			word = arstemmer.suf32(word)        # remove length three and length two suffixes in this order
			word = arstemmer.waw(word)          # remove connective ‘و’ if it precedes a word beginning with ‘و’
			word = arstemmer.norm(word, num=2)       # normalize initial hamza to bare alif
		result.append(word)
	return ' '.join(result)

###################################################################################

# combine rooting and light stemming: if light stemming alogrithm manage to reduce word form, then the light stem is returned, else, the root is returned
def arMorph(text_list):
	result = []
	for word in word_list:
		sol = None
		root = getRootAr(word)
		lightStem = lightStemAr(word)
		if t == lightStem: sol = root
		else: sol = lightStem
		result.append(sol)
	return ' '.join(result)

###################################################################################

# execlude stopwords from a list of words
def exclude_stopwords(word_list):
	return [ w for w in word_list if not w in all_stopwords ]

###################################################################################

# return lemma for english text
def getLemma(text, contextFlag=False):
	lemmatizer = WordNetLemmatizer()
	#'NN':wordnet.NOUN,'JJ':wordnet.ADJ,'VB':wordnet.VERB,'RB':wordnet.ADV
	wordnet_tag ={'NN':'n','JJ':'a','VB':'v','RB':'r'}
	result = None
	if text.split() == 1: # on word
		tokenized = word_tokenize(t)
		tagged = pos_tag(tokenized)[0]
		lemma = ''
		try: lemma = lemmatizer.lemmatize(tagged[0],wordnet_tag[tagged[1][:2]])
		except: lemma = lemmatizer.lemmatize(tagged[0])
		result = lemma
	elif text.split() > 1 and contextFlag == True: # mutiple words i.e. text and without considering the context
		resultList = []
		for t in text.split():
			tokenized = word_tokenize(t)
			tagged = pos_tag(tokenized)[0]
			lemma = ''
			try: lemma = lemmatizer.lemmatize(tagged[0],wordnet_tag[tagged[1][:2]])
			except: lemma = lemmatizer.lemmatize(tagged[0])
			resultList.append(lemma)
		result = ' '.join(resultList)
	else: # mutiple words i.e. text and consider the context
		resultList = []
		tokens = word_tokenize(text)
		tagged = pos_tag(tokens)
		for t in tagged:
			try: resultList.append(lemmatizer.lemmatize(t[0],wordnet_tag[t[1][:2]]))
			except: resultList.append(lemmatizer.lemmatize(t[0]))
		result = ' '.join(resultList)
	return result
###################################################################################

# Given a Naive Bayes classifier, classify a text with a given certaintaity
def classify_text(text, classifier, certainity, g, unicodeFlag):
	#1. process text
	if unicodeFlag: text = text.decode('utf-8')
	word_list = process_text(text, removePunct=True, removeSW=False, removeNum=False)

	#2. generate ngrams
	mygrams = generate_ngrams(word_list, g)

	#3. generate features from ngrams
	feats = generate_features(mygrams)

	#4. classify
	probs = classifier.prob_classify(feats)
	label = probs.max()
	if probs.prob(label) >= certainity: return label, probs.prob(label)
	else: return 'none', probs.prob(label)

###################################################################################
# generates n-gram (g = num of grams)
# for example, if g=3, then the fuction will generate unigrams, bigrams, and tri-grams from the text.
def generate_ngrams(word_list, g):
	mygrams = []
	unigrams = [word for word in word_list]
	mygrams += unigrams
	for i in range(2,g+1): mygrams += ngrams(word_list, i)
	return mygrams
###################################################################################

# generate n-gram features in the form (n-gram, True), i.e., binary feature. In other words, the n-gram exists
def generate_features(mygrams):
	feats = dict([(word, True) for word in mygrams])
	return feats
###################################################################################

# generate features for a doc from selected features grams (selected from a corpus)
# taks 2 parameters:
# 1. document feature grams
# 2. corpus selected feature grams
def build_features(doc_feat_grams, corpus_feat_grams):
	doc_grams = set(doc_feat_grams)
	feats = dict([(word, True) for word in doc_grams if word in corpus_feat_grams])
	return feats
###################################################################################

# evaluate predicted results using true values.
# evaluation metrics are acccuracy, precicion, recall and f-measure.
def evaluate(trueValues, predicted, decimals, note):
	print note
	label = 1
	avg = 'weighted'
	a = accuracy_score(trueValues, predicted)
	p = precision_score(trueValues, predicted, pos_label=label, average=avg)
	r = recall_score(trueValues, predicted, pos_label=label, average=avg)
	avg_f1 = f1_score(trueValues, predicted, pos_label=label, average=avg)
	fclasses = f1_score(trueValues, predicted, average=None)
	f1c1 = fclasses[0]; f1c2 = fclasses[1]
	fw = (f1c1 + f1c2)/2.0

	print 'accuracy:\t', str(round(a,decimals))
	print 'precision:\t', str(round(p,decimals))
	print 'recall:\t', str(round(r,decimals))
	print 'avg f1:\t', str(round(avg_f1,decimals))
	print 'c1 f1:\t', str(round(f1c1,decimals))
	print 'c2 f1:\t', str(round(f1c2,decimals))
	print 'avg(c1,c2):\t', str(round(fw,decimals))
	print '------------'

###################################################################################


# split a parallel or comparable corpus into two parts
def split_corpus(source_corpus, target_corpus, percentage):
	print 'len(source_corpus) == len(target_corpus)', len(source_corpus), '==' , len(target_corpus) , len(source_corpus) == len(target_corpus)
	if len(source_corpus) != len(target_corpus): print 'FAILED: the corpus is not aligned correclty'; return None

	size = len(source_corpus)
	p1 = int (len(source_corpus) * percentage )
	p2 = len(source_corpus) - p1
	print 'size, p1, p2: ', size, p1, p2

	udoc = []

	for e,a in zip(source_corpus,target_corpus): udoc.append( (e,a) )

	shuffle(udoc)

	source_p1 = [] ; source_p2 = []
	target_p1 = [] ; target_p2 = []

	for d in udoc[:p1]: source_p1.append( d[0] )


	for d in udoc[:p1]: target_p1.append( d[1] )


	for d in udoc[p1:]: source_p2.append( d[0] )


	for d in udoc[p1:]: target_p2.append( d[1] )

	return source_p1, target_p1, source_p2, target_p2

##################################################################################
##################################################################################
##################################################################################



# load WordNet (WN) dictionaries
# Dictionaries are obtained from Open Multilingual WordNet website: http://compling.hss.ntu.edu.sg/omw/

# To cite these dictionaries:
# Francis Bond and Kyonghee Paik (2012), A survey of wordnets and their licenses In Proceedings of the 6th Global WordNet Conference (GWC 2012). Matsue. 64–71.
# Francis Bond and Ryan Foster (2013), Linking and extending an open multilingual wordnet. In 51st Annual Meeting of the Association for Computational Linguistics: ACL-2013. Sofia. 1352–1362. 

eng_dict_file = 'wordnet/wn-data-eng.tab'
arb_dict_file = 'wordnet/wn-data-arb.tab'

eng_dict_lines = open(eng_dict_file).readlines()
arb_dict_lines = open(arb_dict_file).readlines()

eng_dict_key = []; eng_dict_word = [];
arb_dict_key = []; arb_dict_word = [];

for l in eng_dict_lines:
	tokens = l.split('\t')
	key = tokens[0][:-2].strip()
	eng_dict_key.append(key)
	word = tokens[2].strip().decode('utf-8')
	eng_dict_word.append(word)

for l in arb_dict_lines:
	tokens = l.split('\t')
	key = tokens[0][:-2].strip()
	arb_dict_key.append(key)
	word = tokens[2].strip().decode('utf-8')
	arb_dict_word.append(word)

###################################################################################
# translation functions using WN bilingual dictionaries
def translate_en2ar(word):
	translations = []
	keys = []
	for i in range(len(eng_dict_word)):
		if word == eng_dict_word[i]: keys.append(eng_dict_key[i])

	for i in range(len(arb_dict_key)):
		for j in range(len(keys)):
			if keys[j] == arb_dict_key[i]: translations.append(arb_dict_word[i])

	return set(translations)
###################################################################################
def translate_ar2en(word):
	translations = []
	keys = []

	for i in range(len(arb_dict_word)):
		if word == arb_dict_word[i]: keys.append(arb_dict_key[i])

	for i in range(len(eng_dict_key)):
		for j in range(len(keys)):
			if keys[j] == eng_dict_key[i]: translations.append(eng_dict_word[i])

	return set(translations)
##################################################################################
##################################################################################
##################################################################################

# binary similarity between two binary vectors
def sim_bin(s_vector,t_vector): return 1 - distance.jaccard(s_vector, t_vector)

# cosine similarity between two wieghted vectors
def sim_cosine(s_vector,t_vector): return 1 - distance.cosine(s_vector, t_vector)

##################################################################################
##################################################################################
##################################################################################

# computes tfidf wieghts for words in a given document. The function needs the corpus to compute idf
def tf_idf(word, document, corpus):
	base = 10
	corpus_size = float(len(corpus))
	tf =  document.count(word)
	doc_freq = float ( sum(1 for doc in corpus if word in doc) )
	idf = math.log( (corpus_size /  doc_freq ), base )
	tf_idf = tf * idf

	return tf_idf
##################################################################################


##################################################################################
# Compute average number of sentences per document for a corpus collectection

def avgSenPerArticle(corpus):
	avg = 0.0
	for d in corpus:
		n = d.splitlines()
		avg += n
	avg /= len(corpus)
	return avg

##################################################################################

##################################################################################
# Compute average number of words per document for a corpus collectection

def avgWordsPerArticle(corpus):
	avg = 0.0
	for d in corpus:
		n = len(d.split())
		avg += n
	avg /= len(corpus)
	return avg

##################################################################################

##################################################################################
# Compute vocabulary size for a text
def vocab(text):
	tok = text.split()
	v = set(tok)
	return len(v)

##################################################################################

##################################################################################

# remove empty lines and white spaces (remove empty lines and keep '\n' in the text)
def pretty_print(text):
	lines = text.splitlines()
	filtered1 = filter(lambda x: not re.match(r'^\s*$', x), lines)
	filtered2 = [whiteSpace.sub(' ', l).strip() for l in filtered1]
	cleantext = '\n'.join(filtered2)
	return cleantext
##################################################################################

# clean html tages from a text
def strip_html_tags(text):
    soup = BeautifulSoup(text)
    doc = pretty_print(soup.get_text())

    return doc
##################################################################################

# find text between two substrings
def find_between(text , first, last ):
	try:
		start = text.index( first ) + len( first )
		end = text.index( last, start )
		return text[start:end]
	except ValueError:
		return None
##################################################################################

def merge_source_target_docs(source_corpus, target_corpus):
	merged_corpus = []
	for source_doc, target_doc in zip(source_corpus, target_corpus):
		merged_corpus.append(source_doc + target_doc)
	
	return merged_corpus

##################################################################################

def load_corpus(corpus_file, corpus_type):
	corpus = None
	global doc_separator
	if corpus_type == 'comparable':
		corpus = open(corpus_file).read().decode('utf-8').split(doc_separator); del corpus[-1]
	if corpus_type == 'parallel':
		corpus = open(corpus_file).read().decode('utf-8').splitlines()
	if not corpus: 
		print 'corpus type is not supported... The corpus should be parallel or comparable'
	return corpus
##################################################################################

def prepare_gensim_corpus(corpus_name, corpus, output_path, min_freq=5):
	if not output_path.endswith('/'): output_path = output_path + '/'
	check_dir(output_path) # if directory does not exist, then create
	
	logging.info( 'building gensim corpus and dictionary for %s corpus', corpus_name )
	logging.info( 'loading corpus' )
	texts = [[word for word in process_text(document, removePunct=True, removeSW=True, removeNum=True)] for document in corpus]
	logging.info( 'tokenizing' )
	all_tokens = [item for sublist in texts for item in sublist]
	logging.info( 'mark tokens which have frequency less than %d', min_freq )
	tokens_once = set([k for k, v in collections.Counter(all_tokens).iteritems() if v < min_freq ])
	logging.info( '|D|=%d' , len(texts) )
	logging.info( 'filter low frequency tokens' )
	texts = [[word for word in text if word not in tokens_once] for text in texts]
	logging.info( '|D|=%d' , len(texts) )
	logging.info( 'building dictionary' )
	dictionary = corpora.Dictionary(texts)
	logging.info( 'saving dictionary' )
	dictFile = output_path + corpus_name + '.dict'
	dictionary.save(dictFile) 
	logging.info( 'building corpus in  mm format' )
	corpus = [dictionary.doc2bow(text) for text in texts]
	logging.info( 'saving corpus' )
	gensim_corpus_file = output_path + corpus_name + '.mm'
	corpora.MmCorpus.serialize(gensim_corpus_file, corpus)
	logging.info( 'computing tfidf' )
	tfidf = models.TfidfModel(corpus) # tfidf model 
	corpus_tfidf = tfidf[corpus] # tfidf corpus 
	logging.info( 'saving tfidf corpus' )
	corpus_tfidf_file = output_path + corpus_name + '.tfidf.mm'
	corpora.MmCorpus.serialize(corpus_tfidf_file, corpus_tfidf)
	logging.info( 'gensim corpus is ready' )
##################################################################################
	
def build_lsi_model(corpus_name, corpus_path, topics=300):
	logging.info( 'building lsi model for %s corpus', corpus_name )
	dictFile = corpus_path + corpus_name + '.dict'
	corpus_tfidf_file = corpus_path + corpus_name + '.tfidf.mm'
	
	logging.info( 'loading dictionary ...' )
	dictionary = corpora.Dictionary.load(dictFile)
	logging.info( 'loading tfidf corpus ...' )
	corpus_tfidf = corpora.MmCorpus(corpus_tfidf_file)
	logging.info( 'building lsi model' )
	lsi = models.LsiModel(corpus_tfidf, id2word=dictionary, num_topics=topics)
	logging.info( 'saving lsi' )
	lsiFile = corpus_path + corpus_name + '.lsi'
	lsi.save(lsiFile)
	logging.info( 'lsi model is ready' )
##################################################################################

def align_documents_lsi(source_test_corpus, target_test_corpus, model_path, model_name, output_path, top_n=20, doc_separator=x_seperator):
	logging.info( 'aligning source and target documents using LSI model' )
	
	dictionaryFile = model_path +  model_name + '.dict'
	lsiFile = model_path +  model_name + '.lsi'
	
	dictionary = corpora.Dictionary.load(dictionaryFile) ; logging.info(  'dictionary loaded' )
	lsi = models.LsiModel.load(lsiFile) ; logging.info(  'lsi model loaded' )
	
	logging.info( '# of source docs %d \t# of target docs %d', len(source_test_corpus),  len(target_test_corpus) )
	
	source_lsi_corpus = generateLSIvectors(source_test_corpus, dictionary, lsi)
	logging.info( 'projects source corpus into LSI space' )
	target_lsi_corpus = generateLSIvectors(target_test_corpus, dictionary, lsi)
	logging.info( 'projects target corpus into LSI space' )
	
	allSims = [] ; doc_tuple = [] ; source_index = 0 
	
	for d in source_lsi_corpus:
		target_index, sim = getComparable(d, target_lsi_corpus)
		allSims.append(sim)
		source_doc = source_test_corpus[source_index] ; target_doc = target_test_corpus[target_index]
		del target_lsi_corpus[target_index] ; 
		del target_test_corpus[target_index] # remove the already aligned document from the target corpus
		doc_tuple.append((source_index,target_index, source_doc, target_doc))
		if not target_lsi_corpus: break # all target docs are aligned
		source_index+=1
		
	sortedAllSims = sorted(enumerate(allSims), key=lambda item: -item[1])
	topNList = sortedAllSims[:top_n]
	out = open (output_path + 'results.txt', 'w')
	count = 0
	print '\n#, src, target, sim'
	for e in topNList:
		i, sim = e
		srcIndx = doc_tuple[i][0] ; targetIndx = doc_tuple[i][1] ; sdoc = doc_tuple[i][2] ; tdoc = doc_tuple[i][3]
		print count, srcIndx, targetIndx, '%0.2f' % sim
		print>>out, count, srcIndx, targetIndx, '%0.2f' % sim
		source_out = open(output_path + str(count) + '.source.txt', 'w')
		target_out = open(output_path + str(count) + '.target.txt' , 'w')
		print>>source_out, sdoc.encode('utf-8')
		print>>target_out, tdoc.encode('utf-8')
		source_out.close(); target_out.close(); count+=1	
	out.close();
	logging.info( 'aligning source and target documents using LSI model is done!' )
##################################################################################

def align_sentences_lsi(source_sentences, target_sentences, model_path, model_name):
	logging.info( 'Sentence level alignment using LSI' )
	
	dictionaryFile = model_path +  model_name + '.dict'
	lsiFile = model_path +  model_name + '.lsi'
	
	dictionary = corpora.Dictionary.load(dictionaryFile) ; logging.info( 'dictionary loaded' )
	lsi = models.LsiModel.load(lsiFile) ; logging.info( 'lsi model loaded' )
	
	source_lsi_sentences = generateLSIvectors(source_sentences, dictionary, lsi); 
	logging.info( 'projects source sentences into LSI space')
	target_lsi_sentences = generateLSIvectors(target_sentences, dictionary, lsi); 
	logging.info( 'projects target sentences into LSI space' )
	
	source_index = 0 	
	new_source_doc = [] ; new_target_doc = []
	
	for d in source_lsi_sentences:
		target_index, sim = getComparable(d, target_lsi_sentences)
		source_sent = source_sentences[source_index] ; target_sent = target_sentences[target_index]
		del target_lsi_sentences[target_index] ; 
		del target_sentences[target_index] # remove the already aligned sentences from the target document
		new_source_doc.append(source_sent) 
		new_target_doc.append(target_sent)
		if not target_lsi_sentences: break # all target sentences are aligned
		source_index+=1
		
	return new_source_doc, new_target_doc
##################################################################################
# projecting a corpus into LSI space
def generateLSIvectors(corpus, dictionary, lsi):
	LSIcorpus = []
	for d in corpus:		
		vec_bow = dictionary.doc2bow(process_text(d))
		vec_lsi = lsi[vec_bow] 
		LSIcorpus.append(vec_lsi)
	return LSIcorpus
##################################################################################
# given a source doc, get the most comparable document from the target corpus
# returns the index of the target document in the the target corpus
def getComparable(source_lsi_doc, target_lsi_corpus):
	sims = []
	for i in range(len(target_lsi_corpus)):
		sims.append( matutils.cossim(source_lsi_doc, target_lsi_corpus[i]) )
	sortedSims = sorted(enumerate(sims), key=lambda item: -item[1])
	topIndex = sortedSims[0][0]
	topSim = sortedSims[0][1]
	return sortedSims[0]

##################################################################################


##################################################################################
# takses wiki text and a list of language codes, and returns the interlanguage links
# language code list:
# ar arabic
# en english
# fr french
# es Español
# de Deutsch
# it Italiano
# pt portuguese
# fa farsi
# ur urdo
# he hebrew
# ps peshto (Afghānī)
# sd Sindhi (sindi)
# ug Uyghur أويغورية
# pnb punjabi (Pakistan - India)
# ckb kurdi
# arz egyptian

lang_list = ['ar', 'en', 'fr', 'es', 'it', 'de', 'fa', 'he', 'ur', 'ps', 'sd', 'ug', 'pnb', 'ckb', 'arz', 'simple']

def get_interlanguage_links_from_wikitext(wiki_text, language_code_list=lang_list):
	interlinks = []
	for code in language_code_list:
		link = find_between(wiki_text, '[[' + code + ':', ']]')
		if link: interlinks.append('[[' + code + ':' + link + ']]')
	return interlinks	
##################################################################################
def get_interlanguage_links_sql(doc_id, db_cursor, lang_code):
	interlinks = []
	sql = '''
	SELECT ll_lang, ll_title
	FROM %s_langlinks
	where
    	ll_from = '%d' ''' % (lang_code, doc_id)
	db_cursor.execute(sql)
	results = db_cursor.fetchall()
	for row in results: 
		lang = row[0] ; title = row[1]
		interlinks.append('[[' + lang + ':' + title + ']]')
	return interlinks
##################################################################################

def load_interlanguage_links(wiki_doc):
	links = find_between(wiki_doc , '<interlanguage_links>', '</interlanguage_links>' )
	return links

##################################################################################

def get_title_from_interlanguage_links(links, language_code):
	title = find_between(links, '[[' + language_code + ':' , ']]')
	return title

##################################################################################

def aligning_documents_by_interlanguage_links(source_corpus_file, target_corpus_file, source_language, target_language, output_path):
		
	if not output_path.endswith('/'): output_path = output_path + '/'
	check_dir(output_path) # if directory does not exist, then create
		
	logging.info( 'aliging %s and %s wikipeida documents using interlanguage links',  source_language, target_language)
	source_docs = split_wikipedia_docs_into_array(source_corpus_file)
	logging.info( 'source corpus is loaded')
	target_docs = split_wikipedia_docs_into_array(target_corpus_file)
	logging.info( 'target corpus is loaded')
	
	target_titles = [get_title_from_interlanguage_links(d, source_language) for d in target_docs]
	
	logging.info( 'start aligning...')
	source_out = open(output_path +  source_language + '-wiki.txt', 'w') 
	target_out = open(output_path +  target_language + '-wiki.txt', 'w') 
	count = 1
	
	my_prperc = pyprind.ProgPercent(len(source_docs)) 
	
	for i in range(len(source_docs)):
		my_prperc.update() # print progress 
		source_title = get_title_from_interlanguage_links(source_docs[i], source_language)
		try: 
			index = target_titles.index(source_title)
			text_out = source_docs[i] 
			print>>source_out, text_out.encode('utf-8')
			text_out = target_docs[index]
			print>>target_out, text_out.encode('utf-8')
			count += 1
		except: continue
				
				
	logging.info( 'aliging by document interlanguage links is done! ... \n %d documents are aligned', count)
##################################################################################

##################################################################################

def aligning_doc_by_interlanguage_links(source_doc, target_corpus, source_language, target_language, output_path):
	
	source = None
	target = None
	
	source_title = get_title_from_interlanguage_links(source_doc, source_language)			
		
	for d in target_corpus:
		target_title = get_title_from_interlanguage_links(d, target_language)
		if source_title == target_title:
			source = source_doc
			target = d
			
	return source, target		
				
##################################################################################

# takes a wikipedia corpus (extracted by WikiExtractor.py) and splits the corpus into documents and clean them 
def split_wikipedia_docs(corpus_file, output_path, doc_len=30):
	corpus = open(corpus_file).read().split('</doc>')
	logging.info( 'processing %d wikipedia documents...', len(corpus))
	count = 1
	for d in corpus:
		doc = strip_html_tags(d)
		if len(doc.split()) > doc_len: # if the number of words in the document is greater than doc_len, then the document will be extracted
			out = open(output_path + os.path.basename(corpus_file) + str('-%07d' % count) + '.txt', 'w')
			print>>out, doc.encode('utf-8')
			out.close(); count+=1
	logging.info('%d documents are extracted', count)
##################################################################################


# takes a wikipedia corpus (extracted by WikiExtractor.py) and splits the corpus into documents and clean them and returns an array
def split_wikipedia_docs_into_array(corpus_file, doc_len=30):
	documents = []
	corpus = open(corpus_file).read().decode('utf-8').split('</doc>')
	for d in corpus:
		#d = strip_html_tags(d)
		# if the number of words in the document is greater than doc_len, then the document will be extracted
		if len(d.split()) > doc_len: 
			documents.append(d + '\n</doc>')
			
	return 	documents
##################################################################################

def check_dir(path):
	if not path.endswith('/'): path = path + '/'
	if not os.path.exists(path): # if directory does not exist, then create
		print path, 'does not exist... creating ....'
		os.makedirs(path)

##################################################################################
# TODO: group words according to their synset IDs
def omw_syn(word, language):
	syn = omw.synsets(word, language)[0]
	return syn.lemma_names(lang=language)
##################################################################################
def split_list(L, n_parts):
	chunk_size = len(L) / n_parts
	chunks=[L[x:x+chunk_size] for x in xrange(0, len(L), chunk_size)]
	return chunks
##################################################################################

##################################################################################


##################################################################################