amrdata_it.py

#!/usr/bin/env python
#coding=utf-8

'''
AMRDataset reads the file generated by preprocessing.sh and it generates a AMRSentence instance for each sentence,
containing all information necessary to the parser.
@author: Marco Damonte (m.damonte@sms.ed.ac.uk)
@since: 3-10-16
'''

import re
from alignments import Alignments as Alignments
import sys
import amrevaluation.smatch.amr_edited as amrannot
sys.path.append("..")
reload(sys)
sys.setdefaultencoding('utf8')

class AMRSentence:
	def __init__(self, tokens, pos, lemmas, nes, dependencies, variables = None, relations = None, graph = None, alignments = None):
		self.tokens = tokens
		self.pos = pos
		self.lemmas = lemmas
		self.nes = nes
		self.dependencies = dependencies
		if variables is not None:
			self.variables = [(str(k),str(variables[k])) for k in variables] 
		if relations is not None:
			self.relations = [r for r in relations if r[0] != r[2]]
		self.graph = graph
		self.alignments = alignments

class AMRDataset:
	def _var2concept(self, amr):
		v2c = {}
		for n, v in zip(amr.nodes, amr.node_values):
			v2c[n] = v
		return v2c

	def __init__(self, prefix, amrs, demo = False, normalize = True):
		self.normalize = normalize
		self.sentences = []

		if demo:
			blocks = prefix.split("\n\n")
		else:
			blocks = open(prefix + ".out", 'r').read().split("\n\n")

		alltokens, allpos, alllemmas, allnes, alldependencies = self._loadFromCoreNLP(blocks)

		if amrs:
			allgraphs = open(prefix + ".graphs").read().split("\n\n")
			a = Alignments(prefix + ".alignments", allgraphs)
			allalignments = a.alignments
			for graph, alignments, dependencies, tokens, pos, lemmas, nes in zip(allgraphs, allalignments, alldependencies, alltokens, allpos, alllemmas, allnes):
				graph = graph.strip()
				amr = amrannot.AMR.parse_AMR_line(graph.replace("\n",""), False)
				variables = {}
				for n, v in zip(amr.nodes, amr.node_values):
					variables[n] = v
		                role_triples = amr.get_triples3()
				relations = []
				for (var1,label,var2) in role_triples:
					if label == "TOP":
						relations.append(("TOP",":top",var1))
					else:
						relations.append((str(var1),":" + str(label),str(var2)))
				self.sentences.append(AMRSentence(tokens, pos, lemmas, nes, dependencies, variables, relations, graph, alignments))
		else:
			for dependencies, tokens, pos, lemmas, nes in zip(alldependencies, alltokens, allpos, alllemmas, allnes):
				self.sentences.append(AMRSentence(tokens, pos, lemmas, nes, dependencies))


	def getSent(self, index):
		return self.sentences[index]

	def getAllSents(self):
		return self.sentences

	def _loadFromCoreNLP(self, blocks):
		alltokens = []
		allpos = []
		alllemmas = []
		allnes = []
		alldependencies = []
		while True:
			if len(blocks) == 1:
				break
			block = blocks.pop(0).strip()

			tokens = []
			lemmas = []
			nes = []
			pos = []
			dependencies = []
			for line in block.split("\n"):
				if line.startswith("#") == False:
					fields = line.split("\t")
					assert(len(fields) == 8)
					tokens.append(fields[2])
					lemmas.append(fields[3])
					pos.append(fields[4])
					nes.append(fields[5])
					if fields[7] == "root":
						dependencies.append((int(fields[1]) - 1, "ROOT", int(fields[1]) - 1))
					else:
						dependencies.append((int(fields[6]) - 1, fields[7], int(fields[1]) - 1))

			tokens2 = []
			lemmas2 = []
			nes2 = []
			for token, lemma, ne in zip(tokens, lemmas, nes):
				nesplit = ne.split()
				if len(nesplit) > 1:
					mne = re.match("^([a-zA-Z\%\>\<\$\~\=]*)([0-9\.]*.*)", nesplit[1][25:].encode('ascii', 'ignore'))
				else:
					mne = None
	
				if self.normalize and len(nesplit) == 2 and re.match("^[0-9].*", nesplit[1][25:]) is not None: #numbers
					[name, norm] = nesplit
					norm = norm[25:]
					m = re.match("([0-9\.][0-9\.]*)E([0-9][0-9]*)$",norm)
					if m is not None:
						n = m.groups()[0]
						z = "".join(["0"]*int(m.groups()[1]))
						norm = str(float(n)*int("1"+z))
					if token.endswith(".0") == False:
						norm = re.sub("\.0$","",norm)
					if token.replace(",","").replace(".","").isdigit() == False and lastnorm is not None:
						norm = "," 
						token = ","
						name = "O"
					lastnorm = norm
					tokens2.append(norm)
					lemmas2.append(token)
					nes2.append(name)
				else:
					lastnorm = None
					tokens2.append(token)
					lemmas2.append(lemma)
					nes2.append(nesplit[0])
			assert(len(tokens2) == len(pos) == len(lemmas2))
			alltokens.append(tokens2)
			alllemmas.append(lemmas2)
			allnes.append(nes2)
			allpos.append(pos)
			alldependencies.append(dependencies)
		return (alltokens, allpos, alllemmas, allnes, alldependencies)