Make "int" token values be long type when [Ll] specified.

Author: Dylan Trotter

pythonparser/lexer.py

@@ -11,6 +11,7 @@
 if sys.version_info[0] == 3:
     unichr = chr
     byte = lambda x: bytes([x])
+    long = int
 else:
     byte = chr
 
@@ -335,33 +336,24 @@ def _refill(self, eof_token):
             self.queue.append(Token(tok_range, "complex", int(match.group(7)) * 1j))
 
         elif match.group(8) is not None: # integer literal, dec
-            literal = match.group(8)
-            self._check_long_literal(tok_range, match.group(1))
-            self.queue.append(Token(tok_range, "int", int(literal)))
+            self.queue.append(self._make_int_token(tok_range, match.group(1), 10))
 
         elif match.group(9) is not None: # integer literal, oct
-            literal = match.group(9)
-            self._check_long_literal(tok_range, match.group(1))
-            self.queue.append(Token(tok_range, "int", int(literal, 8)))
+            self.queue.append(self._make_int_token(tok_range, match.group(1), 8))
 
         elif match.group(10) is not None: # integer literal, hex
-            literal = match.group(10)
-            self._check_long_literal(tok_range, match.group(1))
-            self.queue.append(Token(tok_range, "int", int(literal, 16)))
+            self.queue.append(self._make_int_token(tok_range, match.group(1), 16))
 
         elif match.group(11) is not None: # integer literal, bin
-            literal = match.group(11)
-            self._check_long_literal(tok_range, match.group(1))
-            self.queue.append(Token(tok_range, "int", int(literal, 2)))
+            self.queue.append(self._make_int_token(tok_range, match.group(1), 2))
 
         elif match.group(12) is not None: # integer literal, bare oct
-            literal = match.group(12)
-            if len(literal) > 1 and self.version >= (3, 0):
+            if len(match.group(12)) > 1 and self.version >= (3, 0):
                 error = diagnostic.Diagnostic(
                     "error", "in Python 3, decimal literals must not start with a zero", {},
                     source.Range(self.source_buffer, tok_range.begin_pos, tok_range.begin_pos + 1))
                 self.diagnostic_engine.process(error)
-            self.queue.append(Token(tok_range, "int", int(literal, 8)))
+            self.queue.append(self._make_int_token(tok_range, match.group(1), 8))
 
         elif match.group(14) is not None: # long string literal
             self._string_literal(
@@ -546,12 +538,15 @@ def _replace_escape_bytes(self, value):
 
         return b"".join(chunks)
 
-    def _check_long_literal(self, range, literal):
-        if literal[-1] in "lL" and self.version >= (3, 0):
+    def _make_int_token(self, tok_range, literal, base):
+        if literal[-1] not in "lL":
+            return Token(tok_range, "int", int(literal, base))
+        if self.version >= (3, 0):
             error = diagnostic.Diagnostic(
                 "error", "in Python 3, long integer literals were removed", {},
-                source.Range(self.source_buffer, range.end_pos - 1, range.end_pos))
+                source.Range(self.source_buffer, tok_range.end_pos - 1, tok_range.end_pos))
             self.diagnostic_engine.process(error)
+        return Token(tok_range, "long", long(literal[:-1], base))
 
     def _match_pair_delim(self, range, kwop):
         if kwop == "(":

pythonparser/parser.py

@@ -1516,6 +1516,10 @@ def atom_2(self, tok):
     def atom_3(self, tok):
         return ast.Num(n=tok.value, loc=tok.loc)
 
+    @action(Tok("long"))
+    def atom_3_long(self, tok):
+        return ast.Num(n=tok.value, loc=tok.loc)
+
     @action(Seq(Tok("strbegin"), Tok("strdata"), Tok("strend")))
     def atom_4(self, begin_tok, data_tok, end_tok):
         return ast.Str(s=data_tok.value,
@@ -1542,7 +1546,7 @@ def atom_5(self, strings):
                             empty=lambda self: ast.Dict(keys=[], values=[], colon_locs=[],
                                                         loc=None)),
                    BeginEnd("`", atom_1, "`"),
-                   atom_2, atom_3, atom_5)
+                   atom_2, atom_3, atom_3_long, atom_5)
     """
     (2.6)
     atom: ('(' [yield_expr|testlist_gexp] ')' |

pythonparser/test/test_lexer.py

@@ -142,10 +142,10 @@ def test_integer_py3(self):
                          "int", 83)
         self.assertLexesVersions(
                          "123L", [(2,6)],
-                         "int", 123)
+                         "long", 123)
         self.assertLexesVersions(
                          "123l", [(2,6)],
-                         "int", 123)
+                         "long", 123)
 
         self.assertDiagnosesVersions(
                          "0123", [(3,0)],

pythonparser/test/test_parser.py

@@ -242,6 +242,13 @@ def test_complex(self):
             "1j",
             "~~ loc")
 
+    def test_long(self):
+        self.assertParsesExpr(
+            {"ty": "Num", "n": test_utils.LongOnly(1)},
+            "1L",
+            "~~ loc",
+            only_if=lambda ver: ver < (3,))
+
     def test_string(self):
         self.assertParsesExpr(
             {"ty": "Str", "s": "foo"},

pythonparser/test/test_utils.py

@@ -1,6 +1,7 @@
 # coding:utf-8
 
 from __future__ import absolute_import, division, print_function, unicode_literals
+import sys
 
 unicode = type("")
 
@@ -22,3 +23,13 @@ def __eq__(self, o):
 
     def __ne__(self, o):
         return not self == o
+
+if sys.version_info >= (3,):
+    LongOnly = int
+else:
+    class LongOnly(long):
+        def __eq__(self, o):
+            return isinstance(o, long) and long.__cmp__(self, o) == 0
+
+    def __ne__(self, o):
+        return not self == o