up add viterbi

NLPchina · Jun 27, 2017 · 24676fe · 24676fe
1 parent 0438b0e
commit 24676fe
Show file tree

Hide file tree

Showing 8 changed files with 424 additions and 71 deletions.
diff --git a/src/main/java/org/nlpcn/commons/lang/tire/ObjTree.java b/src/main/java/org/nlpcn/commons/lang/tire/ObjTree.java
@@ -0,0 +1,134 @@
+package org.nlpcn.commons.lang.tire;
+
+import java.io.Serializable;
+import java.util.*;
+
+/**
+ * Created by Ansj on 25/05/2017.数组对象的tire树
+ */
+public class ObjTree<T> implements Serializable {
+
+	private Map<Object, ObjTree<T>> branches;
+
+	private T obj;
+
+	private int status;
+
+	/**
+	 * 增加对象数组到树种
+	 *
+	 * @param objs
+	 */
+	public void add(T t, Object... objs) {
+		ObjTree<T> objTree = this;
+		int len = objs.length;
+		for (int i = 0; i < len; i++) {
+			objTree = objTree.getOrCreateObjTree(objs[i]);
+			if (objTree.status == 3) {
+				objTree.status = 2;
+			} else if (objTree.status == 0) {
+				objTree.status = 1;
+			}
+		}
+		objTree.setObj(t);
+
+		if (objTree.status == 1) {
+			objTree.status = 2;
+		} else if (objTree.status == 0) {
+			objTree.status = 3;
+		}
+	}
+
+
+	/**
+	 * 取得当前分支下所有的截止值
+	 *
+	 * @param objs
+	 */
+	public T get(Object... objs) {
+		ObjTree<T> objTree = getObjTree(objs);
+		if (objTree == null) return null;
+		return objTree.getObj();
+	}
+
+	/**
+	 * 取得当然分支的节点
+	 *
+	 * @param objs
+	 * @return
+	 */
+	public ObjTree<T> getObjTree(Object... objs) {
+		ObjTree objTree = this;
+		for (int i = 0; i < objs.length; i++) {
+			objTree = objTree.getObjTree(objs[i]);
+			if (objTree == null) {
+				return null;
+			}
+		}
+		return objTree;
+	}
+
+	/**
+	 * 判断一个路径是否存在.且是截止
+	 *
+	 * @param objs
+	 * @return
+	 */
+	public boolean contains(Object... objs) {
+		ObjTree objTree = this;
+		for (int i = 0; i < objs.length; i++) {
+			objTree = objTree.getObjTree(objs[i]);
+			if (objTree == null) {
+				return false;
+			}
+		}
+
+		if (objTree.status > 1) {
+			return true;
+		} else {
+			return false;
+		}
+	}
+
+	private ObjTree getObjTree(Object obj) {
+		if (branches == null) {
+			return null;
+		}
+		return branches.get(obj);
+	}
+
+
+	/**
+	 * 获得对象树,如果不存在则创建一个
+	 *
+	 * @param obj
+	 * @return
+	 */
+	private ObjTree<T> getOrCreateObjTree(Object obj) {
+		if (this.branches == null) {
+			this.branches = new HashMap<Object, ObjTree<T>>();
+		}
+		ObjTree<T> objTree = this.branches.get(obj);
+		if (objTree == null) {
+			objTree = new ObjTree();
+			this.branches.put(obj, objTree);
+		}
+		return objTree;
+	}
+
+	public Map<Object, ObjTree<T>> getBranches() {
+		return branches;
+	}
+
+	public int getStatus() {
+		return status;
+	}
+
+	public T getObj() {
+		return obj;
+	}
+
+	public void setObj(T obj) {
+		this.obj = obj;
+	}
+}
diff --git a/src/main/java/org/nlpcn/commons/lang/util/IOUtil.java b/src/main/java/org/nlpcn/commons/lang/util/IOUtil.java
@@ -1,5 +1,8 @@
 package org.nlpcn.commons.lang.util;
 
+import org.nlpcn.commons.lang.util.logging.Log;
+import org.nlpcn.commons.lang.util.logging.LogFactory;
+
 import java.io.*;
 import java.util.*;
 import java.util.Map.Entry;
@@ -10,6 +13,8 @@
  * @author ansj
  */
 public class IOUtil {
+	private static final Log LOG =  LogFactory.getLog() ;
+
 	public static final String UTF8 = "utf-8";
 	public static final String GBK = "gbk";
 	public static final String TABLE = "\t";
@@ -264,7 +269,43 @@ public static <K, V> HashMap<K, V> loadMap(String path, String charEncoding, Cla
 				String readLine = iteartor.next();
 				split = readLine.split("\t");
 				if (split.length < 2) {
-					System.err.println(path + " line:" + index + " has err :" + readLine + " err to load !");
+					LOG.error(path + " line:" + index + " has err :" + readLine + " err to load !");
+					continue;
+				}
+				hm.put((K) ObjConver.conversion(split[0], key), (V) ObjConver.conversion(split[1], value));
+			}
+		} finally {
+			iteartor.close();
+		}
+		return hm;
+	}
+
+	/**
+	 * 加载一个文件到hashMap
+	 *
+	 * @param is
+	 * @param charEncoding
+	 * @param key
+	 * @param value
+	 * @return
+	 * @throws java.io.UnsupportedEncodingException
+	 */
+	@SuppressWarnings({ "unchecked" })
+	public static <K, V> HashMap<K, V> loadMap(InputStream is, String charEncoding, Class<K> key, Class<V> value) throws UnsupportedEncodingException {
+
+		FileIterator iteartor = null;
+		HashMap<K, V> hm = null;
+		try {
+			iteartor = instanceFileIterator(is, charEncoding);
+			hm = new HashMap<K, V>();
+			String[] split = null;
+			int index = 0;
+			while (iteartor.hasNext()) {
+				index++;
+				String readLine = iteartor.next();
+				split = readLine.split("\t");
+				if (split.length < 2) {
+					LOG.error(" line:" + index + " has err :" + readLine + " err to load !");
 					continue;
 				}
 				hm.put((K) ObjConver.conversion(split[0], key), (V) ObjConver.conversion(split[1], value));

diff --git a/src/main/java/org/nlpcn/commons/lang/util/StringUtil.java b/src/main/java/org/nlpcn/commons/lang/util/StringUtil.java
@@ -1,10 +1,6 @@
 package org.nlpcn.commons.lang.util;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -60,40 +56,40 @@ public static String rmHtmlTag(String input) {
 			}
 
 			switch (filter[c]) {
-			case -1:
-				break;
-			case 0:
-				sb.append(c);
-				break;
-			case 1:
-				if (sb.length() > 0 && sb.charAt(sb.length() - 1) != c)
+				case -1:
+					break;
+				case 0:
 					sb.append(c);
-				do {
-					i++;
-				} while (i < length && input.charAt(i) == c);
-
-				if (i < length || input.charAt(length - 1) != c)
-					i--;
-				break;
-			default:
-				tl = filter[c] + i;
-				int tempOff = i;
-				boolean flag = false;
-				char end = (char) filterEnd[c];
-				for (i++; i < length && i < tl; i++) {
-					c = input.charAt(i);
-					if (c > 127)
-						continue;
-					if (c == end) {
-						flag = true;
-						break;
+					break;
+				case 1:
+					if (sb.length() > 0 && sb.charAt(sb.length() - 1) != c)
+						sb.append(c);
+					do {
+						i++;
+					} while (i < length && input.charAt(i) == c);
+
+					if (i < length || input.charAt(length - 1) != c)
+						i--;
+					break;
+				default:
+					tl = filter[c] + i;
+					int tempOff = i;
+					boolean flag = false;
+					char end = (char) filterEnd[c];
+					for (i++; i < length && i < tl; i++) {
+						c = input.charAt(i);
+						if (c > 127)
+							continue;
+						if (c == end) {
+							flag = true;
+							break;
+						}
 					}
-				}
-				if (!flag) {
-					i = tempOff;
-					sb.append(input.charAt(i));
-				}
-				break;
+					if (!flag) {
+						i = tempOff;
+						sb.append(input.charAt(i));
+					}
+					break;
 			}
 		}
 		return sb.toString();
@@ -191,6 +187,22 @@ public static String joiner(double[] doubles, String split) {
 		return sb.toString();
 	}
 
+	public static String joiner(String[] strs, String split) {
+
+		if (strs.length == 0) {
+			return EMPTY;
+		}
+
+		StringBuilder sb = new StringBuilder(String.valueOf(strs[0]));
+
+		for (int i = 1; i < strs.length; i++) {
+			sb.append(split);
+			sb.append(strs[i]);
+		}
+
+		return sb.toString();
+	}
+
 	public static String joiner(float[] floats, String split) {
 
 		if (floats.length == 0) {
@@ -265,7 +277,7 @@ public static boolean isBlank(char[] chars) {
 
 	/**
 	 * 正则匹配第一个
-	 * 
+	 *
 	 * @param regex
 	 * @param input
 	 * @return
@@ -281,7 +293,7 @@ public static String matcherFirst(String regex, String input) {
 
 	/**
 	 * trim 一个字符串.扩展了string类原生的trim.对BOM和中文空格进行trim
-	 * 
+	 *
 	 * @return
 	 */
 	public static String trim(String value) {
@@ -305,7 +317,7 @@ public static String trim(String value) {
 
 	/**
 	 * 正则匹配全部
-	 * 
+	 *
 	 * @param regex
 	 * @param input
 	 * @return
@@ -321,7 +333,7 @@ public static List<String> matcherAll(String regex, String input) {
 
 	/**
 	 * 正则匹配全部
-	 * 
+	 *
 	 * @param regex
 	 * @param input
 	 * @return
@@ -334,4 +346,5 @@ public static String matcherLast(String regex, String input) {
 			return result.get(result.size() - 1);
 		}
 	}
+
 }