-
Notifications
You must be signed in to change notification settings - Fork 0
/
NERDemo.java
171 lines (142 loc) · 6.63 KB
/
NERDemo.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
import edu.stanford.nlp.ie.AbstractSequenceClassifier;
import edu.stanford.nlp.ie.crf.*;
import edu.stanford.nlp.io.IOUtils;
import edu.stanford.nlp.ling.CoreLabel;
import edu.stanford.nlp.ling.CoreAnnotations;
import edu.stanford.nlp.sequences.DocumentReaderAndWriter;
import edu.stanford.nlp.util.Triple;
import java.util.List;
/** This is a demo of calling CRFClassifier programmatically.
* <p>
* Usage: {@code java -mx400m -cp "*" NERDemo [serializedClassifier [fileName]] }
* <p>
* If arguments aren't specified, they default to
* classifiers/english.all.3class.distsim.crf.ser.gz and some hardcoded sample text.
* If run with arguments, it shows some of the ways to get k-best labelings and
* probabilities out with CRFClassifier. If run without arguments, it shows some of
* the alternative output formats that you can get.
* <p>
* To use CRFClassifier from the command line:
* </p><blockquote>
* {@code java -mx400m edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier [classifier] -textFile [file] }
* </blockquote><p>
* Or if the file is already tokenized and one word per line, perhaps in
* a tab-separated value format with extra columns for part-of-speech tag,
* etc., use the version below (note the 's' instead of the 'x'):
* </p><blockquote>
* {@code java -mx400m edu.stanford.nlp.ie.crf.CRFClassifier -loadClassifier [classifier] -testFile [file] }
* </blockquote>
*
* @author Jenny Finkel
* @author Christopher Manning
*/
public class NERDemo {
public static void main(String[] args) throws Exception {
String serializedClassifier = "classifiers/english.all.3class.distsim.crf.ser.gz";
if (args.length > 0) {
serializedClassifier = args[0];
}
AbstractSequenceClassifier<CoreLabel> classifier = CRFClassifier.getClassifier(serializedClassifier);
/* For either a file to annotate or for the hardcoded text example, this
demo file shows several ways to process the input, for teaching purposes.
*/
if (args.length > 1) {
/* For the file, it shows (1) how to run NER on a String, (2) how
to get the entities in the String with character offsets, and
(3) how to run NER on a whole file (without loading it into a String).
*/
String fileContents = IOUtils.slurpFile(args[1]);
List<List<CoreLabel>> out = classifier.classify(fileContents);
for (List<CoreLabel> sentence : out) {
for (CoreLabel word : sentence) {
System.out.print(word.word() + '/' + word.get(CoreAnnotations.AnswerAnnotation.class) + ' ');
}
System.out.println();
}
System.out.println("---");
out = classifier.classifyFile(args[1]);
for (List<CoreLabel> sentence : out) {
for (CoreLabel word : sentence) {
System.out.print(word.word() + '/' + word.get(CoreAnnotations.AnswerAnnotation.class) + ' ');
}
System.out.println();
}
System.out.println("---");
List<Triple<String, Integer, Integer>> list = classifier.classifyToCharacterOffsets(fileContents);
for (Triple<String, Integer, Integer> item : list) {
System.out.println(item.first() + ": " + fileContents.substring(item.second(), item.third()));
}
System.out.println("---");
System.out.println("Ten best entity labelings");
DocumentReaderAndWriter<CoreLabel> readerAndWriter = classifier.makePlainTextReaderAndWriter();
classifier.classifyAndWriteAnswersKBest(args[1], 10, readerAndWriter);
System.out.println("---");
System.out.println("Per-token marginalized probabilities");
classifier.printProbs(args[1], readerAndWriter);
// -- This code prints out the first order (token pair) clique probabilities.
// -- But that output is a bit overwhelming, so we leave it commented out by default.
// System.out.println("---");
// System.out.println("First Order Clique Probabilities");
// ((CRFClassifier) classifier).printFirstOrderProbs(args[1], readerAndWriter);
} else {
/* For the hard-coded String, it shows how to run it on a single
sentence, and how to do this and produce several formats, including
slash tags and an inline XML output format. It also shows the full
contents of the {@code CoreLabel}s that are constructed by the
classifier. And it shows getting out the probabilities of different
assignments and an n-best list of classifications with probabilities.
*/
String[] example = {"Good afternoon Rajat Raina, how are you today?",
"I go to school at Stanford University, which is located in California." };
for (String str : example) {
System.out.println(classifier.classifyToString(str));
}
System.out.println("---");
for (String str : example) {
// This one puts in spaces and newlines between tokens, so just print not println.
System.out.print(classifier.classifyToString(str, "slashTags", false));
}
System.out.println("---");
for (String str : example) {
// This one is best for dealing with the output as a TSV (tab-separated column) file.
// The first column gives entities, the second their classes, and the third the remaining text in a document
System.out.print(classifier.classifyToString(str, "tabbedEntities", false));
}
System.out.println("---");
for (String str : example) {
System.out.println(classifier.classifyWithInlineXML(str));
}
System.out.println("---");
for (String str : example) {
System.out.println(classifier.classifyToString(str, "xml", true));
}
System.out.println("---");
for (String str : example) {
System.out.print(classifier.classifyToString(str, "tsv", false));
}
System.out.println("---");
// This gets out entities with character offsets
int j = 0;
for (String str : example) {
j++;
List<Triple<String,Integer,Integer>> triples = classifier.classifyToCharacterOffsets(str);
for (Triple<String,Integer,Integer> trip : triples) {
System.out.printf("%s over character offsets [%d, %d) in sentence %d.%n",
trip.first(), trip.second(), trip.third, j);
}
}
System.out.println("---");
// This prints out all the details of what is stored for each token
int i=0;
for (String str : example) {
for (List<CoreLabel> lcl : classifier.classify(str)) {
for (CoreLabel cl : lcl) {
System.out.print(i++ + ": ");
System.out.println(cl.toShorterString());
}
}
}
System.out.println("---");
}
}
}