Merge pull request #144 from go-ego/range-pr

add more stop world function and test code

gse_test.go

@@ -266,10 +266,16 @@ func TestStop(t *testing.T) {
 	s = FilterHtml(t2)
 	tt.Equal(t, "test:   bot 机器人 <<银河系漫游指南>> ", s)
 
+	prodSeg.AddStop(`"`)
+	prodSeg.AddStopArr("class", "div", "=")
+	tt.True(t, prodSeg.IsStop("="))
+	s1 := prodSeg.CutStop(t2, false)
+	tt.Equal(t, "[p test : p bot bot 机器人 银河系 漫游 指南]", s1)
+
 	s = prodSeg.CutTrimHtmls(t2, true)
 	tt.Equal(t, "test bot 机器人 银河系 漫游 指南", s)
 
-	s1 := Range("hibot, 机器人")
+	s1 = Range("hibot, 机器人")
 	tt.Equal(t, "[h i b o t ,   机 器 人]", s1)
 	s = RangeText("hibot, 机器人")
 	tt.Equal(t, "h i b o t ,   机 器 人 ", s)

stop.go

@@ -84,6 +84,11 @@ func (seg *Segmenter) AddStop(text string) {
 	seg.StopWordMap[text] = true
 }
 
+// AddStopArr add array stop token to stop dictionaries
+func (seg *Segmenter) AddStopArr(text ...string) {
+	seg.LoadStopArr(text)
+}
+
 // RemoveStop remove a token from the StopWord dictionary.
 func (seg *Segmenter) RemoveStop(text string) {
 	delete(seg.StopWordMap, text)

trim.go

@@ -84,6 +84,16 @@ func (seg *Segmenter) TrimWithPos(se []SegPos, pos ...string) (re []SegPos) {
 	return
 }
 
+// Stop trim []string stop word
+func (seg *Segmenter) Stop(s []string) (r []string) {
+	for _, v := range s {
+		if !seg.IsStop(v) && v != "" {
+			r = append(r, v)
+		}
+	}
+	return
+}
+
 // Trim trim []string exclude symbol, space and punct
 func (seg *Segmenter) Trim(s []string) (r []string) {
 	for i := 0; i < len(s); i++ {
@@ -128,6 +138,11 @@ func (seg *Segmenter) TrimPos(s []SegPos) (r []SegPos) {
 	return
 }
 
+// CutStop cut string and tirm stop
+func (seg *Segmenter) CutStop(str string, hmm ...bool) []string {
+	return seg.Stop(seg.Cut(str, hmm...))
+}
+
 // CutTrim cut string and tirm
 func (seg *Segmenter) CutTrim(str string, hmm ...bool) []string {
 	s := seg.Cut(str, hmm...)