Merge pull request huaban#11 from douban/optimize

performance: use tree data structure and StringBuilder to accelerate long text analysing speed

src/main/java/com/huaban/analysis/jieba/JiebaSegmenter.java

@@ -173,45 +173,45 @@ public List<String> sentenceProcess(String sentence) {
 
         int x = 0;
         int y = 0;
-        String buf = "";
+        String buf;
+        StringBuilder sb = new StringBuilder();
         while (x < N) {
             y = route.get(x).key + 1;
             String lWord = sentence.substring(x, y);
             if (y - x == 1)
-                buf += lWord;
+                sb.append(lWord);
             else {
-                if (buf.length() > 0) {
+                if (sb.length() > 0) {
+                    buf = sb.toString();
+                    sb = new StringBuilder();
                     if (buf.length() == 1) {
-                        tokens.add(buf.toString());
-                        buf = "";
+                        tokens.add(buf);
                     }
                     else {
                         if (wordDict.containsWord(buf)) {
-                            tokens.add(buf.toString());
+                            tokens.add(buf);
                         }
                         else {
                             finalSeg.cut(buf, tokens);
                         }
-                        buf = "";
                     }
                 }
                 tokens.add(lWord);
             }
             x = y;
         }
+        buf = sb.toString();
         if (buf.length() > 0) {
             if (buf.length() == 1) {
-                tokens.add(buf.toString());
-                buf = "";
+                tokens.add(buf);
             }
             else {
                 if (wordDict.containsWord(buf)) {
-                    tokens.add(buf.toString());
+                    tokens.add(buf);
                 }
                 else {
                     finalSeg.cut(buf, tokens);
                 }
-                buf = "";
             }
 
         }

src/main/java/com/huaban/analysis/jieba/Node.java

@@ -0,0 +1,11 @@
+package com.huaban.analysis.jieba;
+
+public class Node {
+    public Character value;
+    public Node parent;
+
+    public Node(Character value, Node parent) {
+        this.value = value;
+        this.parent = parent;
+    }
+}

src/main/java/com/huaban/analysis/jieba/viterbi/FinalSeg.java

@@ -10,9 +10,11 @@
 import java.util.Map;
 import java.util.Vector;
 import java.util.regex.Matcher;
+import java.util.Collections;
 
 import com.huaban.analysis.jieba.CharacterUtil;
 import com.huaban.analysis.jieba.Pair;
+import com.huaban.analysis.jieba.Node;
 
 
 public class FinalSeg {
@@ -136,22 +138,21 @@ public void cut(String sentence, List<String> tokens) {
 
     public void viterbi(String sentence, List<String> tokens) {
         Vector<Map<Character, Double>> v = new Vector<Map<Character, Double>>();
-        Map<Character, Vector<Character>> path = new HashMap<Character, Vector<Character>>();
+        Map<Character, Node> path = new HashMap<Character, Node>();
 
         v.add(new HashMap<Character, Double>());
         for (char state : states) {
             Double emP = emit.get(state).get(sentence.charAt(0));
             if (null == emP)
                 emP = MIN_FLOAT;
             v.get(0).put(state, start.get(state) + emP);
-            path.put(state, new Vector<Character>());
-            path.get(state).add(state);
+            path.put(state, new Node(state, null));
         }
 
         for (int i = 1; i < sentence.length(); ++i) {
             Map<Character, Double> vv = new HashMap<Character, Double>();
             v.add(vv);
-            Map<Character, Vector<Character>> newPath = new HashMap<Character, Vector<Character>>();
+            Map<Character, Node> newPath = new HashMap<Character, Node>();
             for (char y : states) {
                 Double emp = emit.get(y).get(sentence.charAt(i));
                 if (emp == null)
@@ -170,20 +171,24 @@ else if (candidate.freq <= tranp) {
                     }
                 }
                 vv.put(y, candidate.freq);
-                Vector<Character> newPathValue = new Vector<Character>();
-                newPathValue.addAll(path.get(candidate.key));
-                newPathValue.add(y);
-                newPath.put(y, newPathValue);
+                newPath.put(y, new Node(y, path.get(candidate.key)));
             }
             path = newPath;
         }
         double probE = v.get(sentence.length() - 1).get('E');
         double probS = v.get(sentence.length() - 1).get('S');
-        Vector<Character> posList;
+        Vector<Character> posList = new Vector<Character>(sentence.length());
+        Node win;
         if (probE < probS)
-            posList = path.get('S');
+            win = path.get('S');
         else
-            posList = path.get('E');
+            win = path.get('E');
+
+        while (win != null) {
+            posList.add(win.value);
+            win = win.parent;
+        }
+        Collections.reverse(posList);
 
         int begin = 0, next = 0;
         for (int i = 0; i < sentence.length(); ++i) {