Merge branch 'master' into gh-pages-new

lintool · Jan 22, 2013 · c9c76e5 · c9c76e5
2 parents bf487de + fedf0b0
commit c9c76e5
Show file tree

Hide file tree

Showing 1,129 changed files with 4,773 additions and 4,248 deletions.
diff --git a/.classpath b/.classpath
@@ -42,10 +42,8 @@
 	<classpathentry kind="lib" path="lib/hadoop-annotations-2.0.0-cdh4.1.2.jar"/>
 	<classpathentry kind="lib" path="lib/hadoop-auth-2.0.0-cdh4.1.2.jar"/>
 	<classpathentry kind="lib" path="lib/hadoop-common-2.0.0-cdh4.1.2.jar"/>
-	<classpathentry kind="lib" path="lib/hadoop-core-2.0.0-mr1-cdh4.1.2.jar"/>
 	<classpathentry kind="lib" path="lib/hadoop-hdfs-2.0.0-cdh4.1.2.jar"/>
 	<classpathentry kind="lib" path="lib/hadoop-mapreduce-client-core-2.0.0-cdh4.1.2.jar"/>
-	<classpathentry kind="lib" path="lib/hadoop-streaming-2.0.0-mr1-cdh4.1.2.jar"/>
 	<classpathentry kind="lib" path="lib/hadoop-yarn-api-2.0.0-cdh4.1.2.jar"/>
 	<classpathentry kind="lib" path="lib/hadoop-yarn-common-2.0.0-cdh4.1.2.jar"/>
 	<classpathentry kind="lib" path="lib/hadoop-yarn-server-common-2.0.0-cdh4.1.2.jar"/>
@@ -114,5 +112,9 @@
 	<classpathentry kind="lib" path="lib/ant-1.6.5.jar"/>
 	<classpathentry kind="lib" path="lib/ant-1.8.1.jar"/>
 	<classpathentry kind="lib" path="lib/ant-launcher-1.8.1.jar"/>
+	<classpathentry kind="lib" path="lib/hadoop-mapreduce-client-common-2.0.0-cdh4.1.2.jar"/>
+	<classpathentry kind="lib" path="lib/hadoop-mapreduce-client-jobclient-2.0.0-cdh4.1.2.jar"/>
+	<classpathentry kind="lib" path="lib/hadoop-mapreduce-client-shuffle-2.0.0-cdh4.1.2.jar"/>
+	<classpathentry kind="lib" path="lib/hadoop-streaming-2.0.0-cdh4.1.2.jar"/>
 	<classpathentry kind="output" path="build"/>
 </classpath>
diff --git a/.gitignore b/.gitignore
@@ -4,6 +4,7 @@
 ivy/ivy.jar
 etc/junit.*
 etc/run.*
+etc/run-integration.*
 data/bible+shakes.nopunc
 data/bible+shakes.nopunc.json.packed
 data/bible+shakes.nopunc.tuple1.packed

diff --git a/build.xml b/build.xml
@@ -11,15 +11,16 @@
   <property name="test.junit.dir" location="${test.dir}/junit" />
   <property name="integration.dir" location="${test.dir}/integration" />
 
-  <property name="version" value="1.4.2"/>
+  <property name="version" value="1.4.4"/>
 
   <!-- paths used for compilation and run  -->
   <path id="lib.path.id">
     <fileset dir="${lib.dir}" />
   </path>
   <path id="run.path.id">
     <path refid="lib.path.id" />
-    <path location="${build.dir}" />
+    <fileset dir="${dist.dir}" />
+    <!-- path location="${build.dir}" /-->
   </path>
 
   <property name="maven.ant.task.version" value="2.1.2"/>
@@ -242,7 +243,7 @@
 
 		<property name="java_command" value="${java} -Xmx2048m -classpath &quot;${run.path.id}&quot;" />
 
-		<condition property="script_prefix" value="#!/bin/sh${line.separator}if test -s ~/.bashrc${line.separator}then${line.separator}source ~/.bashrc${line.separator}fi${line.separator}">
+		<condition property="script_prefix" value="#!/bin/sh${line.separator}export HADOOP_CLASSPATH=&quot;${run.path.id}:$HADOOP_CLASSPATH&quot;${line.separator}if test -s ~/.bashrc${line.separator}then${line.separator}source ~/.bashrc${line.separator}fi${line.separator}">
 			<equals arg1="${platform}" arg2="unix" />
 		</condition>
 		<condition property="script_prefix" value="">
@@ -257,6 +258,9 @@
 		<echo file="./etc/run.${suffix}" message="${java_command} " append="true" />
 		<echo file="./etc/run.${suffix}" message="${param_prefix}1 ${param_prefix}2 ${param_prefix}3 ${param_prefix}4 ${param_prefix}5 ${param_prefix}6 ${param_prefix}7 ${param_prefix}8 ${param_prefix}9" append="true" />
 
+		<echo file="./etc/run-integration.${suffix}" message="${script_prefix}" />
+		<echo file="./etc/run-integration.${suffix}" message="${line.separator}ant clean${line.separator}ant${line.separator}ant -lib etc/mail.jar integration${line.separator}" append="true" />
+
 	</target>
 
   <target name="gen-pom" depends="install-ivy">

diff --git a/docs/api/allclasses-frame.html b/docs/api/allclasses-frame.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:31 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:50 EST 2013 -->
 <TITLE>
 All Classes
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/allclasses-noframe.html b/docs/api/allclasses-noframe.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:31 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:50 EST 2013 -->
 <TITLE>
 All Classes
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/constant-values.html b/docs/api/constant-values.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:30 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:49 EST 2013 -->
 <TITLE>
 Constant Field Values
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
 
@@ -253,6 +253,12 @@ <H1>
 <TH ALIGN="left" COLSPAN="3">edu.umd.cloud9.collection.clue.<A HREF="edu/umd/cloud9/collection/clue/CountClueWarcRecords.html" title="class in edu.umd.cloud9.collection.clue">CountClueWarcRecords</A></TH>
 </TR>
 <TR BGCOLOR="white" CLASS="TableRowColor">
+<A NAME="edu.umd.cloud9.collection.clue.CountClueWarcRecords.COUNT_OPTION"><!-- --></A><TD ALIGN="right"><FONT SIZE="-1">
+<CODE>public&nbsp;static&nbsp;final&nbsp;<A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</A></CODE></FONT></TD>
+<TD ALIGN="left"><CODE><A HREF="edu/umd/cloud9/collection/clue/CountClueWarcRecords.html#COUNT_OPTION">COUNT_OPTION</A></CODE></TD>
+<TD ALIGN="right"><CODE>"countOutput"</CODE></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
 <A NAME="edu.umd.cloud9.collection.clue.CountClueWarcRecords.MAPPING_OPTION"><!-- --></A><TD ALIGN="right"><FONT SIZE="-1">
 <CODE>public&nbsp;static&nbsp;final&nbsp;<A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</A></CODE></FONT></TD>
 <TD ALIGN="left"><CODE><A HREF="edu/umd/cloud9/collection/clue/CountClueWarcRecords.html#MAPPING_OPTION">MAPPING_OPTION</A></CODE></TD>
@@ -355,6 +361,12 @@ <H1>
 <TD ALIGN="right"><CODE>"collection"</CODE></TD>
 </TR>
 <TR BGCOLOR="white" CLASS="TableRowColor">
+<A NAME="edu.umd.cloud9.collection.trec.CountTrecDocuments.COUNT_OPTION"><!-- --></A><TD ALIGN="right"><FONT SIZE="-1">
+<CODE>public&nbsp;static&nbsp;final&nbsp;<A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</A></CODE></FONT></TD>
+<TD ALIGN="left"><CODE><A HREF="edu/umd/cloud9/collection/trec/CountTrecDocuments.html#COUNT_OPTION">COUNT_OPTION</A></CODE></TD>
+<TD ALIGN="right"><CODE>"countOutput"</CODE></TD>
+</TR>
+<TR BGCOLOR="white" CLASS="TableRowColor">
 <A NAME="edu.umd.cloud9.collection.trec.CountTrecDocuments.MAPPING_OPTION"><!-- --></A><TD ALIGN="right"><FONT SIZE="-1">
 <CODE>public&nbsp;static&nbsp;final&nbsp;<A HREF="http://download.oracle.com/javase/6/docs/api/java/lang/String.html?is-external=true" title="class or interface in java.lang">String</A></CODE></FONT></TD>
 <TD ALIGN="left"><CODE><A HREF="edu/umd/cloud9/collection/trec/CountTrecDocuments.html#MAPPING_OPTION">MAPPING_OPTION</A></CODE></TD>

diff --git a/docs/api/deprecated-list.html b/docs/api/deprecated-list.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:31 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:50 EST 2013 -->
 <TITLE>
 Deprecated List
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/edu/umd/cloud9/collection/DocnoMapping.Builder.html b/docs/api/edu/umd/cloud9/collection/DocnoMapping.Builder.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:27 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:46 EST 2013 -->
 <TITLE>
 DocnoMapping.Builder
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/edu/umd/cloud9/collection/DocnoMapping.BuilderUtils.html b/docs/api/edu/umd/cloud9/collection/DocnoMapping.BuilderUtils.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:27 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:46 EST 2013 -->
 <TITLE>
 DocnoMapping.BuilderUtils
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/edu/umd/cloud9/collection/DocnoMapping.DefaultBuilderOptions.html b/docs/api/edu/umd/cloud9/collection/DocnoMapping.DefaultBuilderOptions.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:27 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:46 EST 2013 -->
 <TITLE>
 DocnoMapping.DefaultBuilderOptions
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/edu/umd/cloud9/collection/DocnoMapping.html b/docs/api/edu/umd/cloud9/collection/DocnoMapping.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:27 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:46 EST 2013 -->
 <TITLE>
 DocnoMapping
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/edu/umd/cloud9/collection/DocumentForwardIndex.html b/docs/api/edu/umd/cloud9/collection/DocumentForwardIndex.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:27 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:46 EST 2013 -->
 <TITLE>
 DocumentForwardIndex
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/edu/umd/cloud9/collection/DocumentForwardIndexHttpServer.FetchDocidServlet.html b/docs/api/edu/umd/cloud9/collection/DocumentForwardIndexHttpServer.FetchDocidServlet.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:27 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:46 EST 2013 -->
 <TITLE>
 DocumentForwardIndexHttpServer.FetchDocidServlet
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/edu/umd/cloud9/collection/DocumentForwardIndexHttpServer.FetchDocnoServlet.html b/docs/api/edu/umd/cloud9/collection/DocumentForwardIndexHttpServer.FetchDocnoServlet.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:27 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:46 EST 2013 -->
 <TITLE>
 DocumentForwardIndexHttpServer.FetchDocnoServlet
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/edu/umd/cloud9/collection/DocumentForwardIndexHttpServer.HomeServlet.html b/docs/api/edu/umd/cloud9/collection/DocumentForwardIndexHttpServer.HomeServlet.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:27 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:46 EST 2013 -->
 <TITLE>
 DocumentForwardIndexHttpServer.HomeServlet
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/edu/umd/cloud9/collection/DocumentForwardIndexHttpServer.html b/docs/api/edu/umd/cloud9/collection/DocumentForwardIndexHttpServer.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:27 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:46 EST 2013 -->
 <TITLE>
 DocumentForwardIndexHttpServer
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/edu/umd/cloud9/collection/ExtractHTMLFieldCollection.MyMapper.HeadingTagFilter.html b/docs/api/edu/umd/cloud9/collection/ExtractHTMLFieldCollection.MyMapper.HeadingTagFilter.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:27 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:46 EST 2013 -->
 <TITLE>
 ExtractHTMLFieldCollection.MyMapper.HeadingTagFilter
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/edu/umd/cloud9/collection/ExtractHTMLFieldCollection.MyMapper.html b/docs/api/edu/umd/cloud9/collection/ExtractHTMLFieldCollection.MyMapper.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:27 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:46 EST 2013 -->
 <TITLE>
 ExtractHTMLFieldCollection.MyMapper
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/edu/umd/cloud9/collection/ExtractHTMLFieldCollection.html b/docs/api/edu/umd/cloud9/collection/ExtractHTMLFieldCollection.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:27 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:46 EST 2013 -->
 <TITLE>
 ExtractHTMLFieldCollection
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">
 

diff --git a/docs/api/edu/umd/cloud9/collection/Indexable.html b/docs/api/edu/umd/cloud9/collection/Indexable.html
@@ -2,12 +2,12 @@
 <!--NewPage-->
 <HTML>
 <HEAD>
-<!-- Generated by javadoc (build 1.6.0_37) on Tue Dec 25 22:07:27 EST 2012 -->
+<!-- Generated by javadoc (build 1.6.0_37) on Mon Jan 14 16:09:46 EST 2013 -->
 <TITLE>
 Indexable
 </TITLE>
 
-<META NAME="date" CONTENT="2012-12-25">
+<META NAME="date" CONTENT="2013-01-14">
 
 <LINK REL ="stylesheet" TYPE="text/css" HREF="../../../../stylesheet.css" TITLE="Style">