Merge pull request #261 from thatch/syntax-3.0

Support Python 3 syntax back to 3.0
Instagram · Mar 20, 2020 · d2b86be · d2b86be
2 parents a65c67d + 62fd4e5
commit d2b86be
Show file tree

Hide file tree

Showing 23 changed files with 481 additions and 55 deletions.
diff --git a/README.rst b/README.rst
@@ -33,8 +33,8 @@ A Concrete Syntax Tree (CST) parser and serializer library for Python
 
 .. intro-start
 
-LibCST parses Python 3.5, 3.6, 3.7 or 3.8 source code as a CST tree that keeps all
-formatting details (comments, whitespaces, parentheses, etc). It's useful for
+LibCST parses Python 3.0, 3.1, 3.3, 3.5, 3.6, 3.7 or 3.8 source code as a CST tree that keeps
+all formatting details (comments, whitespaces, parentheses, etc). It's useful for
 building automated refactoring (codemod) applications and linters.
 
 .. intro-end

diff --git a/libcst/__init__.py b/libcst/__init__.py
@@ -185,7 +185,10 @@
     TrailingWhitespace,
 )
 from libcst._parser.entrypoints import parse_expression, parse_module, parse_statement
-from libcst._parser.types.config import PartialParserConfig
+from libcst._parser.types.config import (
+    KNOWN_PYTHON_VERSION_STRINGS,
+    PartialParserConfig,
+)
 from libcst._removal_sentinel import RemovalSentinel, RemoveFromParent
 from libcst._version import LIBCST_VERSION
 from libcst._visitors import CSTNodeT, CSTTransformer, CSTVisitor, CSTVisitorT
@@ -201,6 +204,7 @@
 
 
 __all__ = [
+    "KNOWN_PYTHON_VERSION_STRINGS",
     "LIBCST_VERSION",
     "BatchableCSTVisitor",
     "CSTNodeT",

diff --git a/libcst/_nodes/tests/base.py b/libcst/_nodes/tests/base.py
@@ -41,6 +41,20 @@ def _cst_node_equality_func(
         raise AssertionError(f"\n{a!r}\nis not deeply equal to \n{b!r}{suffix}")
 
 
+def parse_expression_as(**config: Any) -> Callable[[str], cst.BaseExpression]:
+    def inner(code: str) -> cst.BaseExpression:
+        return cst.parse_expression(code, config=cst.PartialParserConfig(**config))
+
+    return inner
+
+
+def parse_statement_as(**config: Any) -> Callable[[str], cst.BaseStatement]:
+    def inner(code: str) -> cst.BaseStatement:
+        return cst.parse_statement(code, config=cst.PartialParserConfig(**config))
+
+    return inner
+
+
 # We can't use an ABCMeta here, because of metaclass conflicts
 class CSTNodeTest(UnitTest):
     def setUp(self) -> None:
@@ -215,6 +229,18 @@ def __assert_visit_returns_identity(self, node: cst.CSTNode) -> None:
         # later version and tighten this check.
         self.assertEqual(node, node.visit(_NOOPVisitor()))
 
+    def assert_parses(
+        self,
+        code: str,
+        parser: Callable[[str], cst.BaseExpression],
+        expect_success: bool,
+    ) -> None:
+        if not expect_success:
+            with self.assertRaises(cst.ParserSyntaxError):
+                parser(code)
+        else:
+            parser(code)
+
 
 @dataclass(frozen=True)
 class DummyIndentedBlock(cst.CSTNode):

diff --git a/libcst/_nodes/tests/test_atom.py b/libcst/_nodes/tests/test_atom.py
@@ -9,7 +9,7 @@
 
 import libcst as cst
 from libcst import parse_expression
-from libcst._nodes.tests.base import CSTNodeTest
+from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -1023,6 +1023,23 @@ def test_valid_no_parse(self, **kwargs: Any) -> None:
     def test_invalid(self, **kwargs: Any) -> None:
         self.assert_invalid(**kwargs)
 
+    @data_provider(
+        (
+            {
+                "code": "u'x'",
+                "parser": parse_expression_as(python_version="3.3"),
+                "expect_success": True,
+            },
+            {
+                "code": "u'x'",
+                "parser": parse_expression_as(python_version="3.1"),
+                "expect_success": False,
+            },
+        )
+    )
+    def test_versions(self, **kwargs: Any) -> None:
+        self.assert_parses(**kwargs)
+
 
 class StringHelperTest(CSTNodeTest):
     def test_string_prefix_and_quotes(self) -> None:

diff --git a/libcst/_nodes/tests/test_dict.py b/libcst/_nodes/tests/test_dict.py
@@ -8,7 +8,7 @@
 
 import libcst as cst
 from libcst import parse_expression
-from libcst._nodes.tests.base import CSTNodeTest
+from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -172,3 +172,20 @@ def test_valid(self, **kwargs: Any) -> None:
     )
     def test_invalid(self, **kwargs: Any) -> None:
         self.assert_invalid(**kwargs)
+
+    @data_provider(
+        (
+            {
+                "code": "{**{}}",
+                "parser": parse_expression_as(python_version="3.5"),
+                "expect_success": True,
+            },
+            {
+                "code": "{**{}}",
+                "parser": parse_expression_as(python_version="3.3"),
+                "expect_success": False,
+            },
+        )
+    )
+    def test_versions(self, **kwargs: Any) -> None:
+        self.assert_parses(**kwargs)
diff --git a/libcst/_nodes/tests/test_funcdef.py b/libcst/_nodes/tests/test_funcdef.py
@@ -8,7 +8,7 @@
 
 import libcst as cst
 from libcst import parse_statement
-from libcst._nodes.tests.base import CSTNodeTest, DummyIndentedBlock
+from libcst._nodes.tests.base import CSTNodeTest, DummyIndentedBlock, parse_statement_as
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -1979,3 +1979,30 @@ def test_valid(self, node: cst.CSTNode, code: str) -> None:
     )
     def test_valid_38(self, node: cst.CSTNode, code: str) -> None:
         self.validate_node(node, code, _parse_statement_force_38)
+
+    @data_provider(
+        (
+            {
+                "code": "async def foo(): pass",
+                "parser": parse_statement_as(python_version="3.7"),
+                "expect_success": True,
+            },
+            {
+                "code": "async def foo(): pass",
+                "parser": parse_statement_as(python_version="3.6"),
+                "expect_success": True,
+            },
+            {
+                "code": "async def foo(): pass",
+                "parser": parse_statement_as(python_version="3.5"),
+                "expect_success": True,
+            },
+            {
+                "code": "async def foo(): pass",
+                "parser": parse_statement_as(python_version="3.3"),
+                "expect_success": False,
+            },
+        )
+    )
+    def test_versions(self, **kwargs: Any) -> None:
+        self.assert_parses(**kwargs)
diff --git a/libcst/_nodes/tests/test_list.py b/libcst/_nodes/tests/test_list.py
@@ -8,7 +8,7 @@
 
 import libcst as cst
 from libcst import parse_expression, parse_statement
-from libcst._nodes.tests.base import CSTNodeTest
+from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -111,3 +111,20 @@ def test_invalid(
         self, get_node: Callable[[], cst.CSTNode], expected_re: str
     ) -> None:
         self.assert_invalid(get_node, expected_re)
+
+    @data_provider(
+        (
+            {
+                "code": "[a, *b]",
+                "parser": parse_expression_as(python_version="3.5"),
+                "expect_success": True,
+            },
+            {
+                "code": "[a, *b]",
+                "parser": parse_expression_as(python_version="3.3"),
+                "expect_success": False,
+            },
+        )
+    )
+    def test_versions(self, **kwargs: Any) -> None:
+        self.assert_parses(**kwargs)
diff --git a/libcst/_nodes/tests/test_matrix_multiply.py b/libcst/_nodes/tests/test_matrix_multiply.py
@@ -0,0 +1,73 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# pyre-strict
+from typing import Any
+
+import libcst as cst
+from libcst._nodes.tests.base import (
+    CSTNodeTest,
+    parse_expression_as,
+    parse_statement_as,
+)
+from libcst.testing.utils import data_provider
+
+
+class NamedExprTest(CSTNodeTest):
+    @data_provider(
+        (
+            {
+                "node": cst.BinaryOperation(
+                    left=cst.Name("a"),
+                    operator=cst.MatrixMultiply(),
+                    right=cst.Name("b"),
+                ),
+                "code": "a @ b",
+                "parser": parse_expression_as(python_version="3.8"),
+            },
+            {
+                "node": cst.SimpleStatementLine(
+                    body=(
+                        cst.AugAssign(
+                            target=cst.Name("a"),
+                            operator=cst.MatrixMultiplyAssign(),
+                            value=cst.Name("b"),
+                        ),
+                    ),
+                ),
+                "code": "a @= b\n",
+                "parser": parse_statement_as(python_version="3.8"),
+            },
+        )
+    )
+    def test_valid(self, **kwargs: Any) -> None:
+        self.validate_node(**kwargs)
+
+    @data_provider(
+        (
+            {
+                "code": "a @ b",
+                "parser": parse_expression_as(python_version="3.6"),
+                "expect_success": True,
+            },
+            {
+                "code": "a @ b",
+                "parser": parse_expression_as(python_version="3.3"),
+                "expect_success": False,
+            },
+            {
+                "code": "a @= b",
+                "parser": parse_statement_as(python_version="3.6"),
+                "expect_success": True,
+            },
+            {
+                "code": "a @= b",
+                "parser": parse_statement_as(python_version="3.3"),
+                "expect_success": False,
+            },
+        )
+    )
+    def test_versions(self, **kwargs: Any) -> None:
+        self.assert_parses(**kwargs)
diff --git a/libcst/_nodes/tests/test_set.py b/libcst/_nodes/tests/test_set.py
@@ -8,7 +8,7 @@
 
 import libcst as cst
 from libcst import parse_expression
-from libcst._nodes.tests.base import CSTNodeTest
+from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
 from libcst.testing.utils import data_provider
 
 
@@ -118,3 +118,20 @@ def test_invalid(
         self, get_node: Callable[[], cst.CSTNode], expected_re: str
     ) -> None:
         self.assert_invalid(get_node, expected_re)
+
+    @data_provider(
+        (
+            {
+                "code": "{*x, 2}",
+                "parser": parse_expression_as(python_version="3.5"),
+                "expect_success": True,
+            },
+            {
+                "code": "{*x, 2}",
+                "parser": parse_expression_as(python_version="3.3"),
+                "expect_success": False,
+            },
+        )
+    )
+    def test_versions(self, **kwargs: Any) -> None:
+        self.assert_parses(**kwargs)
diff --git a/libcst/_nodes/tests/test_tuple.py b/libcst/_nodes/tests/test_tuple.py
@@ -8,7 +8,7 @@
 
 import libcst as cst
 from libcst import parse_expression, parse_statement
-from libcst._nodes.tests.base import CSTNodeTest
+from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -264,3 +264,20 @@ def test_invalid(
         self, get_node: Callable[[], cst.CSTNode], expected_re: str
     ) -> None:
         self.assert_invalid(get_node, expected_re)
+
+    @data_provider(
+        (
+            {
+                "code": "(a, *b)",
+                "parser": parse_expression_as(python_version="3.5"),
+                "expect_success": True,
+            },
+            {
+                "code": "(a, *b)",
+                "parser": parse_expression_as(python_version="3.3"),
+                "expect_success": False,
+            },
+        )
+    )
+    def test_versions(self, **kwargs: Any) -> None:
+        self.assert_parses(**kwargs)
diff --git a/libcst/_nodes/tests/test_with.py b/libcst/_nodes/tests/test_with.py
@@ -8,7 +8,7 @@
 
 import libcst as cst
 from libcst import PartialParserConfig, parse_statement
-from libcst._nodes.tests.base import CSTNodeTest, DummyIndentedBlock
+from libcst._nodes.tests.base import CSTNodeTest, DummyIndentedBlock, parse_statement_as
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -215,3 +215,20 @@ def test_valid(self, **kwargs: Any) -> None:
     )
     def test_invalid(self, **kwargs: Any) -> None:
         self.assert_invalid(**kwargs)
+
+    @data_provider(
+        (
+            {
+                "code": "with a, b: pass",
+                "parser": parse_statement_as(python_version="3.1"),
+                "expect_success": True,
+            },
+            {
+                "code": "with a, b: pass",
+                "parser": parse_statement_as(python_version="3.0"),
+                "expect_success": False,
+            },
+        )
+    )
+    def test_versions(self, **kwargs: Any) -> None:
+        self.assert_parses(**kwargs)