bpo-36390: simplify classifyws() and add unit tests

Lib/idlelib/editor.py

@@ -1281,7 +1281,7 @@ def smart_indent_event(self, event):
                 text.delete(first, last)
                 text.mark_set("insert", first)
             prefix = text.get("insert linestart", "insert")
-            raw, effective = classifyws(prefix, self.tabwidth)
+            raw, effective = get_line_indent(prefix, self.tabwidth)
             if raw == len(prefix):
                 # only whitespace to the left
                 self.reindent_to(effective + self.indentwidth)
@@ -1415,7 +1415,7 @@ def indent_region_event(self, event):
         for pos in range(len(lines)):
             line = lines[pos]
             if line:
-                raw, effective = classifyws(line, self.tabwidth)
+                raw, effective = get_line_indent(line, self.tabwidth)
                 effective = effective + self.indentwidth
                 lines[pos] = self._make_blanks(effective) + line[raw:]
         self.set_region(head, tail, chars, lines)
@@ -1426,7 +1426,7 @@ def dedent_region_event(self, event):
         for pos in range(len(lines)):
             line = lines[pos]
             if line:
-                raw, effective = classifyws(line, self.tabwidth)
+                raw, effective = get_line_indent(line, self.tabwidth)
                 effective = max(effective - self.indentwidth, 0)
                 lines[pos] = self._make_blanks(effective) + line[raw:]
         self.set_region(head, tail, chars, lines)
@@ -1461,7 +1461,7 @@ def tabify_region_event(self, event):
         for pos in range(len(lines)):
             line = lines[pos]
             if line:
-                raw, effective = classifyws(line, tabwidth)
+                raw, effective = get_line_indent(line, tabwidth)
                 ntabs, nspaces = divmod(effective, tabwidth)
                 lines[pos] = '\t' * ntabs + ' ' * nspaces + line[raw:]
         self.set_region(head, tail, chars, lines)
@@ -1575,8 +1575,8 @@ def _asktabwidth(self):
     def guess_indent(self):
         opener, indented = IndentSearcher(self.text, self.tabwidth).run()
         if opener and indented:
-            raw, indentsmall = classifyws(opener, self.tabwidth)
-            raw, indentlarge = classifyws(indented, self.tabwidth)
+            raw, indentsmall = get_line_indent(opener, self.tabwidth)
+            raw, indentlarge = get_line_indent(indented, self.tabwidth)
         else:
             indentsmall = indentlarge = 0
         return indentlarge - indentsmall
@@ -1585,23 +1585,16 @@ def guess_indent(self):
 def index2line(index):
     return int(float(index))
 
-# Look at the leading whitespace in s.
-# Return pair (# of leading ws characters,
-#              effective # of leading blanks after expanding
-#              tabs to width tabwidth)
-
-def classifyws(s, tabwidth):
-    raw = effective = 0
-    for ch in s:
-        if ch == ' ':
-            raw = raw + 1
-            effective = effective + 1
-        elif ch == '\t':
-            raw = raw + 1
-            effective = (effective // tabwidth + 1) * tabwidth
-        else:
-            break
-    return raw, effective
+
+_line_indent_re = re.compile(r'[ \t]*')
+def get_line_indent(line, tabwidth):
+    """Return a line's indentation as (# chars, effective # of spaces).
+
+    The effective # of spaces is the length after properly "expanding"
+    the tabs into spaces, as done by str.expandtabs(tabwidth).
+    """
+    m = _line_indent_re.match(line)
+    return m.end(), len(m.group().expandtabs(tabwidth))
 
 
 class IndentSearcher(object):

Lib/idlelib/idle_test/test_editor.py

@@ -42,5 +42,66 @@ class dummy():
             self.assertEqual(func(dummy, inp), out)
 
 
+class TestGetLineIndent(unittest.TestCase):
+    def test_empty_lines(self):
+        for tabwidth in [1, 2, 4, 6, 8]:
+            for line in ['', '\n']:
+                with self.subTest(line=line, tabwidth=tabwidth):
+                    self.assertEqual(
+                        editor.get_line_indent(line, tabwidth=tabwidth),
+                        (0, 0),
+                    )
+
+    def test_tabwidth_4(self):
+        #        (line, (raw, effective))
+        tests = (('no spaces', (0, 0)),
+                 # Internal space isn't counted.
+                 ('    space test', (4, 4)),
+                 ('\ttab test', (1, 4)),
+                 ('\t\tdouble tabs test', (2, 8)),
+                 # Different results when mixing tabs and spaces.
+                 ('    \tmixed test', (5, 8)),
+                 ('  \t  mixed test', (5, 6)),
+                 ('\t    mixed test', (5, 8)),
+                 # Spaces not divisible by tabwidth.
+                 ('  \tmixed test', (3, 4)),
+                 (' \t mixed test', (3, 5)),
+                 ('\t  mixed test', (3, 6)),
+                 # Only checks spaces and tabs.
+                 ('\nnewline test', (0, 0)))
+
+        for line, expected in tests:
+            with self.subTest(line=line):
+                self.assertEqual(
+                    editor.get_line_indent(line, tabwidth=4),
+                    expected,
+                )
+
+    def test_tabwidth_8(self):
+        #        (line, (raw, effective))
+        tests = (('no spaces', (0, 0)),
+                 # Internal space isn't counted.
+                 ('        space test', (8, 8)),
+                 ('\ttab test', (1, 8)),
+                 ('\t\tdouble tabs test', (2, 16)),
+                 # Different results when mixing tabs and spaces.
+                 ('        \tmixed test', (9, 16)),
+                 ('      \t  mixed test', (9, 10)),
+                 ('\t        mixed test', (9, 16)),
+                 # Spaces not divisible by tabwidth.
+                 ('  \tmixed test', (3, 8)),
+                 (' \t mixed test', (3, 9)),
+                 ('\t  mixed test', (3, 10)),
+                 # Only checks spaces and tabs.
+                 ('\nnewline test', (0, 0)))
+
+        for line, expected in tests:
+            with self.subTest(line=line):
+                self.assertEqual(
+                    editor.get_line_indent(line, tabwidth=8),
+                    expected,
+                )
+
+
 if __name__ == '__main__':
     unittest.main(verbosity=2)