-
Notifications
You must be signed in to change notification settings - Fork 4
/
metrics.py
115 lines (90 loc) · 4.17 KB
/
metrics.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
import re
import string
import itertools
import collections
import numpy as np
def normalize_answer(s):
"""Lower text and remove punctuation, articles and extra whitespace."""
def remove_articles(text):
regex = re.compile(r"\b(a|an|the)\b", re.UNICODE)
return re.sub(regex, " ", text)
def white_space_fix(text):
return " ".join(text.split())
def remove_punc(text):
exclude = set(string.punctuation)
return "".join(ch for ch in text if ch not in exclude)
def lower(text):
return text.lower()
return white_space_fix(remove_articles(remove_punc(lower(s))))
def get_tokens(s):
if not s:
return []
return normalize_answer(s).split()
def compute_exact(a_gold, a_pred):
return int(normalize_answer(a_gold) == normalize_answer(a_pred))
def compute_f1(a_gold, a_pred):
gold_toks = get_tokens(a_gold)
pred_toks = get_tokens(a_pred)
common = collections.Counter(gold_toks) & collections.Counter(pred_toks)
num_same = sum(common.values())
if len(gold_toks) == 0 or len(pred_toks) == 0:
# If either is no-answer, then F1 is 1 if they agree, 0 otherwise
return int(gold_toks == pred_toks)
if num_same == 0:
return 0
precision = 1.0 * num_same / len(pred_toks)
recall = 1.0 * num_same / len(gold_toks)
f1 = (2 * precision * recall) / (precision + recall)
return f1
def em(samples, pred_answers, kg, aliases=True):
if not isinstance(pred_answers, list):
samples = [samples]
pred_answers = [pred_answers]
assert len(samples) == len(pred_answers)
num_all_answers = 0
num_correct_answers = 0
for sample, pred_answer in zip(samples, pred_answers):
gold_entities = set(list(itertools.chain(*
[[entity['mention'] for entity in sample['answer_entities'] if entity['mention'] not in [None, '']]] + \
[kg.get_aliases(entity['name']) for entity in sample['answer_entities'] if aliases == True and kg is not None and entity['name'] is not None]
)))
if len(gold_entities) == 0: continue
num_all_answers += 1
num_correct_answers += 1 \
if np.count_nonzero([compute_exact(gold_entity, pred_answer) for gold_entity in gold_entities]) != 0 \
else 0
return num_correct_answers / (num_all_answers + 1e-16)
def f1(samples, pred_answers, kg, aliases=True):
if not isinstance(pred_answers, list):
samples = [samples]
pred_answers = [pred_answers]
assert len(samples) == len(pred_answers)
num_all_answers = 0
num_correct_answers = 0
for sample, pred_answer in zip(samples, pred_answers):
gold_entities = set(list(itertools.chain(*
[[entity['mention'] for entity in sample['answer_entities'] if entity['mention'] not in [None, '']]] + \
[kg.get_aliases(entity['name']) for entity in sample['answer_entities'] if aliases == True and kg is not None and entity['name'] is not None]
)))
if len(gold_entities) == 0: continue
num_all_answers += 1
num_correct_answers += max([compute_f1(gold_entity, pred_answer) for gold_entity in gold_entities])
return num_correct_answers / (num_all_answers + 1e-16)
def accuracy(samples, pred_answers, kg, aliases=True):
if not isinstance(pred_answers, list):
samples = [samples]
pred_answers = [pred_answers]
assert len(samples) == len(pred_answers)
num_all_answers = 0
num_correct_answers = 0
for sample, pred_answer in zip(samples, pred_answers):
gold_entities = set(list(itertools.chain(*
[[entity['mention'] for entity in sample['answer_entities'] if entity['mention'] not in [None, '']]] + \
[kg.get_aliases(entity['name']) for entity in sample['answer_entities'] if aliases == True and kg is not None and entity['name'] is not None]
)))
if len(gold_entities) == 0: continue
num_all_answers += 1
num_correct_answers += 1 \
if np.count_nonzero([normalize_answer(gold_entity) in normalize_answer(pred_answer) for gold_entity in gold_entities]) != 0 \
else 0
return num_correct_answers / (num_all_answers + 1e-16)