Fix narrow width: Added StrictEmojiNarrow

Author: mattn

benchmark_test.go

@@ -7,6 +7,78 @@ import (
 
 var benchSink int
 
+//
+// RuneWidth
+//
+
+func benchRuneWidth(b *testing.B, eastAsianWidth bool, start, stop rune, want int) int {
+	n := 0
+	got := -1
+	c := NewCondition()
+	c.EastAsianWidth = eastAsianWidth
+	for i := 0; i < b.N; i++ {
+		got = n
+		for r := start; r < stop; r++ {
+			n += c.RuneWidth(r)
+		}
+		got = n - got
+	}
+	if want != 0 && got != want { // some extra checks
+		b.Errorf("got %d, want %d\n", got, want)
+	}
+	return n
+}
+func BenchmarkRuneWidthAll(b *testing.B) {
+	benchSink = benchRuneWidth(b, false, 0, utf8.MaxRune+1, 1293932)
+}
+func BenchmarkRuneWidth768(b *testing.B) {
+	benchSink = benchRuneWidth(b, false, 0, 0x300, 702)
+}
+func BenchmarkRuneWidthAllEastAsian(b *testing.B) {
+	benchSink = benchRuneWidth(b, true, 0, utf8.MaxRune+1, 1432558)
+}
+func BenchmarkRuneWidth768EastAsian(b *testing.B) {
+	benchSink = benchRuneWidth(b, true, 0, 0x300, 794)
+}
+
+//
+// String1Width - strings which consist of a single rune
+//
+
+func benchString1Width(b *testing.B, eastAsianWidth bool, start, stop rune, want int) int {
+	n := 0
+	got := -1
+	c := NewCondition()
+	c.EastAsianWidth = eastAsianWidth
+	for i := 0; i < b.N; i++ {
+		got = n
+		for r := start; r < stop; r++ {
+			s := string(r)
+			n += c.StringWidth(s)
+		}
+		got = n - got
+	}
+	if want != 0 && got != want { // some extra checks
+		b.Errorf("got %d, want %d\n", got, want)
+	}
+	return n
+}
+func BenchmarkString1WidthAll(b *testing.B) {
+	benchSink = benchString1Width(b, false, 0, utf8.MaxRune+1, 1295980)
+}
+func BenchmarkString1Width768(b *testing.B) {
+	benchSink = benchString1Width(b, false, 0, 0x300, 702)
+}
+func BenchmarkString1WidthAllEastAsian(b *testing.B) {
+	benchSink = benchString1Width(b, true, 0, utf8.MaxRune+1, 1436654)
+}
+func BenchmarkString1Width768EastAsian(b *testing.B) {
+	benchSink = benchString1Width(b, true, 0, 0x300, 794)
+}
+
+//
+// tables
+//
 func benchTable(b *testing.B, tbl table) int {
 	n := 0
 	for i := 0; i < b.N; i++ {

runewidth.go

@@ -12,8 +12,14 @@ var (
 	// EastAsianWidth will be set true if the current locale is CJK
 	EastAsianWidth bool
 
+	// StrictEmojiNeutral should be set false if handle broken fonts
+	StrictEmojiNeutral bool = true
+
 	// DefaultCondition is a condition in current locale
-	DefaultCondition = &Condition{}
+	DefaultCondition = &Condition{
+		EastAsianWidth:     false,
+		StrictEmojiNeutral: true,
+	}
 )
 
 func init() {
@@ -83,28 +89,52 @@ var nonprint = table{
 
 // Condition have flag EastAsianWidth whether the current locale is CJK or not.
 type Condition struct {
-	EastAsianWidth bool
+	EastAsianWidth     bool
+	StrictEmojiNeutral bool
 }
 
 // NewCondition return new instance of Condition which is current locale.
 func NewCondition() *Condition {
 	return &Condition{
-		EastAsianWidth: EastAsianWidth,
+		EastAsianWidth:     EastAsianWidth,
+		StrictEmojiNeutral: StrictEmojiNeutral,
 	}
 }
 
 // RuneWidth returns the number of cells in r.
 // See http://www.unicode.org/reports/tr11/
 func (c *Condition) RuneWidth(r rune) int {
-	switch {
-	case r < 0 || r > 0x10FFFF || inTables(r, nonprint, combining):
-		return 0
-	case inTables(r, narrow):
-		return 1
-	case (c.EastAsianWidth && IsAmbiguousWidth(r)) || inTables(r, doublewidth, neutral):
-		return 2
-	default:
-		return 1
+	// optimized version, verified by TestRuneWidthChecksums()
+	if !c.EastAsianWidth {
+		switch {
+		case r < 0x20 || r > 0x10FFFF:
+			return 0
+		case (r >= 0x7F && r <= 0x9F) || r == 0xAD: // nonprint
+			return 0
+		case r < 0x300:
+			return 1
+		case inTable(r, narrow):
+			return 1
+		case inTables(r, nonprint, combining):
+			return 0
+		case inTable(r, doublewidth):
+			return 2
+		default:
+			return 1
+		}
+	} else {
+		switch {
+		case r < 0 || r > 0x10FFFF || inTables(r, nonprint, combining):
+			return 0
+		case inTable(r, narrow):
+			return 1
+		case inTables(r, ambiguous, doublewidth):
+			return 2
+		case !c.StrictEmojiNeutral && inTables(r, ambiguous, emoji, narrow):
+			return 2
+		default:
+			return 1
+		}
 	}
 }
 

runewidth_test.go

@@ -65,6 +65,31 @@ func TestTableChecksums(t *testing.T) {
 	}
 }
 
+func TestRuneWidthChecksums(t *testing.T) {
+	var testcases = []struct {
+		name           string
+		eastAsianWidth bool
+		wantSHA        string
+	}{
+		{"ea-no", false, "4eb632b105d3b2c800dda9141381d0b8a95250a3a5c7f1a5ca2c4d4daaa85234"},
+		{"ea-yes", true, "c2ddc3bdf42d81d4c23050e21eda46eb639b38b15322d35e8eb6c26f3b83ce92"},
+	}
+
+	for _, testcase := range testcases {
+		c := NewCondition()
+		c.EastAsianWidth = testcase.eastAsianWidth
+		buf := make([]byte, utf8.MaxRune+1)
+		for r := rune(0); r <= utf8.MaxRune; r++ {
+			buf[r] = byte(c.RuneWidth(r))
+		}
+		gotSHA := fmt.Sprintf("%x", sha256.Sum256(buf))
+		if gotSHA != testcase.wantSHA {
+			t.Errorf("TestRuneWidthChecksums = %s,\n\tsha256 = %s want %s",
+				testcase.name, gotSHA, testcase.wantSHA)
+		}
+	}
+}
+
 func checkInterval(first, last rune) bool {
 	return first >= 0 && first <= utf8.MaxRune &&
 		last >= 0 && last <= utf8.MaxRune &&
@@ -87,99 +112,71 @@ func isCompact(t *testing.T, ti *tableInfo) bool {
 	return true
 }
 
-// This is a utility function in case that a table has changed.
-func printCompactTable(tbl table) {
-	counter := 0
-	printEntry := func(first, last rune) {
-		if counter%3 == 0 {
-			fmt.Printf("\t")
-		}
-		fmt.Printf("{0x%04X, 0x%04X},", first, last)
-		if (counter+1)%3 == 0 {
-			fmt.Printf("\n")
-		} else {
-			fmt.Printf(" ")
-		}
-		counter++
-	}
-
-	sort.Sort(&tbl) // just in case
-	first := rune(-1)
-	for i := range tbl {
-		e := tbl[i]
-		if !checkInterval(e.first, e.last) { // sanity check
-			panic("invalid table")
-		}
-		if first < 0 {
-			first = e.first
-		}
-		if i+1 < len(tbl) && e.last+1 >= tbl[i+1].first { // can be combined into one entry
-			continue
-		}
-		printEntry(first, e.last)
-		first = -1
-	}
-	fmt.Printf("\n\n")
-}
-
 func TestSorted(t *testing.T) {
 	for _, ti := range tables {
 		if !sort.IsSorted(&ti.tbl) {
 			t.Errorf("table not sorted: %s", ti.name)
 		}
 		if !isCompact(t, &ti) {
 			t.Errorf("table not compact: %s", ti.name)
-			//printCompactTable(ti.tbl)
 		}
 	}
 }
 
 var runewidthtests = []struct {
-	in    rune
-	out   int
-	eaout int
+	in     rune
+	out    int
+	eaout  int
+	nseout int
 }{
-	{'世', 2, 2},
-	{'界', 2, 2},
-	{'ｾ', 1, 1},
-	{'ｶ', 1, 1},
-	{'ｲ', 1, 1},
-	{'☆', 1, 2}, // double width in ambiguous
-	{'☺', 1, 1},
-	{'☻', 1, 1},
-	{'♥', 1, 2},
-	{'♦', 1, 1},
-	{'♣', 1, 2},
-	{'♠', 1, 2},
-	{'♂', 1, 2},
-	{'♀', 1, 2},
-	{'♪', 1, 2},
-	{'♫', 1, 1},
-	{'☼', 1, 1},
-	{'↕', 1, 2},
-	{'‼', 1, 1},
-	{'↔', 1, 2},
-	{'\x00', 0, 0},
-	{'\x01', 0, 0},
-	{'\u0300', 0, 0},
-	{'\u2028', 0, 0},
-	{'\u2029', 0, 0},
-	{'a', 1, 1}, // ASCII classified as "na" (narrow)
-	{'⟦', 1, 1}, // non-ASCII classified as "na" (narrow)
+	{'世', 2, 2, 2},
+	{'界', 2, 2, 2},
+	{'ｾ', 1, 1, 1},
+	{'ｶ', 1, 1, 1},
+	{'ｲ', 1, 1, 1},
+	{'☆', 1, 2, 2}, // double width in ambiguous
+	{'☺', 1, 1, 2},
+	{'☻', 1, 1, 2},
+	{'♥', 1, 2, 2},
+	{'♦', 1, 1, 2},
+	{'♣', 1, 2, 2},
+	{'♠', 1, 2, 2},
+	{'♂', 1, 2, 2},
+	{'♀', 1, 2, 2},
+	{'♪', 1, 2, 2},
+	{'♫', 1, 1, 2},
+	{'☼', 1, 1, 2},
+	{'↕', 1, 2, 2},
+	{'‼', 1, 1, 2},
+	{'↔', 1, 2, 2},
+	{'\x00', 0, 0, 0},
+	{'\x01', 0, 0, 0},
+	{'\u0300', 0, 0, 0},
+	{'\u2028', 0, 0, 0},
+	{'\u2029', 0, 0, 0},
+	{'a', 1, 1, 1}, // ASCII classified as "na" (narrow)
+	{'⟦', 1, 1, 1}, // non-ASCII classified as "na" (narrow)
+	{'👁', 1, 1, 2},
 }
 
 func TestRuneWidth(t *testing.T) {
 	c := NewCondition()
 	c.EastAsianWidth = false
 	for _, tt := range runewidthtests {
 		if out := c.RuneWidth(tt.in); out != tt.out {
-			t.Errorf("RuneWidth(%q) = %d, want %d", tt.in, out, tt.out)
+			t.Errorf("RuneWidth(%q) = %d, want %d (EastAsianWidth=false)", tt.in, out, tt.out)
 		}
 	}
 	c.EastAsianWidth = true
 	for _, tt := range runewidthtests {
 		if out := c.RuneWidth(tt.in); out != tt.eaout {
-			t.Errorf("RuneWidth(%q) = %d, want %d", tt.in, out, tt.eaout)
+			t.Errorf("RuneWidth(%q) = %d, want %d (EastAsianWidth=true)", tt.in, out, tt.eaout)
+		}
+	}
+	c.StrictEmojiNeutral = false
+	for _, tt := range runewidthtests {
+		if out := c.RuneWidth(tt.in); out != tt.nseout {
+			t.Errorf("RuneWidth(%q) = %d, want %d (StrictEmojiNeutral=false)", tt.in, out, tt.eaout)
 		}
 	}
 }