Changed the way how parameters are being built

ChangeLog

@@ -2,6 +2,25 @@ Change log for the astroid package (used to be astng)
 =====================================================
 
 --
+    * Changed the way how parameters are being built
+
+      The old way consisted in having the parameter names, their
+      defaults and their annotations separated in different components
+      of the Arguments node. We introduced a new Param node, which holds
+      the name of a parameter, its default value and its annotation.
+      If any of the last two values are missing, then that slot will be
+      filled with a new node kind, Empty, which is used for specifying the
+      lack of something (None could have been used instead, but that means having
+      non-AST nodes in the Arguments node).
+      We're also having support for positional only arguments, for the moment
+      only in raw_building.
+
+    * We don't support nested arguments in functions in Python 2
+      anymore.
+
+      This was dropped in order to simplify the implementation.
+      When they are encountered, we'll unflatten them into a list
+      of parameters, as if they were not nested from the beginning.      
 
     * NodeNG.nearest was removed. It's not an API that we were using
       and it was buggy.

astroid/inference.py

@@ -698,6 +698,7 @@ def infer_arguments(self, context=None, nodes=None):
 
 @infer.register(treeabc.AssignName)
 @infer.register(treeabc.AssignAttr)
+@infer.register(treeabc.Parameter)
 @decorators.path_wrapper
 def infer_assign(self, context=None):
     """infer a AssignName/AssignAttr: need to inspect the RHS part of the

astroid/interpreter/lookup.py

@@ -238,6 +238,7 @@ def locals_new_scope(node, locals_):
 @_get_locals.register(treeabc.DelName)
 @_get_locals.register(treeabc.FunctionDef)
 @_get_locals.register(treeabc.ClassDef)
+@_get_locals.register(treeabc.Parameter)
 def locals_name(node, locals_):
     '''These nodes add a name to the local variables.  AssignName and
     DelName have no children while FunctionDef and ClassDef start a

astroid/interpreter/scope.py

@@ -41,20 +41,23 @@ def _scope_by_parent(parent, node):
     # in order to decouple the implementation for the normal cases.
 
 
+def _node_arguments(node):
+    for arg in itertools.chain(node.positional_and_keyword, node.keyword_only,
+                               (node.vararg, ), (node.kwarg, )):
+        if arg and arg.annotation:
+            yield arg
+
+
 @_scope_by_parent.register(treeabc.Arguments)
 def _scope_by_argument_parent(parent, node):
     args = parent
-    if node in itertools.chain(args.defaults, args.kw_defaults):
-        return args.parent.parent.scope()
-    if six.PY3:
-        look_for = itertools.chain(
-            (args.kwargannotation, ),
-            (args.varargannotation, ),
-            args.kwonly_annotations,
-            args.annotations)
-        if node in look_for:
+    for param in itertools.chain(args.positional_and_keyword, args.keyword_only):
+        if param.default == node:
             return args.parent.parent.scope()
 
+    if six.PY3 and node in _node_arguments(args):
+        return args.parent.parent.scope()
+
 
 @_scope_by_parent.register(treeabc.FunctionDef)
 def _scope_by_function_parent(parent, node):

astroid/nodes.py

@@ -40,9 +40,9 @@
     Arguments, AssignAttr, Assert, Assign,
     AssignName, AugAssign, Repr, BinOp, BoolOp, Break, Call, Compare,
     Comprehension, Const, Continue, Decorators, DelAttr, DelName, Delete,
-    Dict, Expr, Ellipsis, ExceptHandler, Exec, ExtSlice, For,
+    Dict, Empty, Expr, Ellipsis, ExceptHandler, Exec, ExtSlice, For,
     ImportFrom, Attribute, Global, If, IfExp, Import, Index, Keyword,
-    List, Name, NameConstant, Nonlocal, Pass, Print, Raise, Return, Set, Slice,
+    List, Name, NameConstant, Nonlocal, Pass, Parameter, Print, Raise, Return, Set, Slice,
     Starred, Subscript, TryExcept, TryFinally, Tuple, UnaryOp, While, With,
     WithItem, Yield, YieldFrom, AsyncFor, Await, AsyncWith,
     # Node not present in the builtin ast module.
@@ -74,7 +74,7 @@
     Lambda, List, ListComp,
     Name, NameConstant, Nonlocal,
     Module,
-    Pass, Print,
+    Parameter, Pass, Print,
     Raise, ReservedName, Return,
     Set, SetComp, Slice, Starred, Subscript,
     TryExcept, TryFinally, Tuple,

astroid/protocols.py

@@ -304,14 +304,15 @@ def mulass_assigned_stmts(self, nodes, node=None, context=None, assign_path=None
 
 @assigned_stmts.register(treeabc.AssignName)
 @assigned_stmts.register(treeabc.AssignAttr)
+@assigned_stmts.register(treeabc.Parameter)
 def assend_assigned_stmts(self, nodes, node=None, context=None, assign_path=None):
     return self.parent.assigned_stmts(self, context=context)
 
 
 def _arguments_infer_argname(self, name, context, nodes):
     # arguments information may be missing, in which case we can't do anything
     # more
-    if not (self.args or self.vararg or self.kwarg):
+    if not self.args and (not self.vararg and not self.kwarg):
         yield util.Uninferable
         return
     # first argument of instance/class method
@@ -336,10 +337,10 @@ def _arguments_infer_argname(self, name, context, nodes):
         return
 
     # TODO: just provide the type here, no need to have an empty Dict.
-    if name == self.vararg:
+    if self.vararg and name == self.vararg.name:
         yield nodes.Tuple(parent=self)
         return
-    if name == self.kwarg:
+    if self.kwarg and name == self.kwarg.name:
         yield nodes.Dict(parent=self)
         return
     # if there is a default value, yield it. And then yield Uninferable to reflect

astroid/raw_building.py

@@ -319,65 +319,71 @@ def ast_from_function(func, built_objects, module, name=None, parent=None):
                   itertools.groupby(signature.parameters.values(),
                                     operator.attrgetter('kind'))}
 
-    def extract_args(parameters, parent):
-        '''Takes an iterator over Parameter objects and returns three
-        sequences, arg names, default values, and annotations.
-
-        '''
-        names = []
-        defaults = []
-        annotations = []
+    def _extract_args(parameters, parent):
+        """Generate an iterator of Parameter nodes from a list of inspect.Parameter objects."""
         for parameter in parameters:
-            names.append(parameter.name)
+            param = node_classes.Parameter(name=parameter.name,
+                                           col_offset=parent.col_offset,
+                                           lineno=parent.lineno,
+                                           parent=parent)
+            default = node_classes.Empty
+            annotation = node_classes.Empty
             if parameter.default is not _Parameter.empty:
-                defaults.extend(_ast_from_object(parameter.default, built_objects, module, parent=parent))
+                default = _ast_from_object(parameter.default, built_objects,
+                                           module, parent=parent)
+
             if parameter.annotation is not _Parameter.empty:
-                annotations.extend(_ast_from_object(parameter.annotation, built_objects, module, parent=parent))
-            else:
-                annotations.append(None)
-        return names, defaults, annotations
+                annotation = _ast_from_object(parameter.annotation, built_objects,
+                                              module, parent=parent)
 
-    def extract_vararg(parameter):
-        '''Takes a single-element iterator possibly containing a Parameter and
-        returns a name and an annotation.
+            param.postinit(default=default, annotation=annotation)
+            yield param
 
-        '''
+    def _extract_vararg(parameter, parent):
+        """Build a variadic Parameter node from an inspect.Parameter object."""
         try:
-            return parameter[0].name
+            parameter = parameter[0]
         except IndexError:
-            return None
-
-    vararg = parameters.get(_Parameter.VAR_POSITIONAL, ())
-    kwarg = parameters.get(_Parameter.VAR_KEYWORD, ())
-    vararg_name = extract_vararg(vararg)
-    kwarg_name = extract_vararg(kwarg)
-    args_node = node_classes.Arguments(vararg=vararg_name, kwarg=kwarg_name, parent=func_node)
-
-    # This ignores POSITIONAL_ONLY args, because they only appear in
-    # functions implemented in C and can't be mimicked by any Python
-    # function.
-    names, defaults, annotations = extract_args(parameters.get(_Parameter.POSITIONAL_OR_KEYWORD, ()), args_node)
-    kwonlynames, kw_defaults, kwonly_annotations = extract_args(parameters.get(_Parameter.KEYWORD_ONLY, ()), args_node)
-    args = [node_classes.AssignName(name=n, parent=args_node) for n in names]
-    kwonlyargs = [node_classes.AssignName(name=n, parent=args_node) for n in kwonlynames]
-    if vararg_name and vararg[0].annotation is not _Parameter.empty:
-        varargannotation = vararg.annotation
-    else:
-        varargannotation = None
-    if kwarg_name and kwarg[0].annotation is not _Parameter.empty:
-        kwargannotation = kwarg.annotation
-    else:
-        kwargannotation = None
+            return node_classes.Empty
+
+        if parameter.annotation is not _Parameter.empty:
+            annotation = _ast_from_object(parameter.annotation,
+                                          built_objects, module, parent=parent)[0]
+        else:
+            annotation = node_classes.Empty
+
+        param = node_classes.Parameter(name=parameter.name,
+                                       lineno=parent.lineno,
+                                       col_offset=parent.col_offset,
+                                       parent=parent)
+        param.postinit(annotation=annotation, default=node_classes.Empty)
+        return param
+
+    args_node = node_classes.Arguments(parent=func_node)
+    args = _extract_args(parameters.get(_Parameter.POSITIONAL_OR_KEYWORD, ()),
+                         args_node)
+    keyword_only = _extract_args(parameters.get(_Parameter.KEYWORD_ONLY, ()),
+                                 args_node)
+    positional_only = _extract_args(parameters.get(_Parameter.POSITIONAL_ONLY, ()),
+                                    args_node)
+    python_vararg = parameters.get(_Parameter.VAR_POSITIONAL, ())
+    python_kwarg = parameters.get(_Parameter.VAR_KEYWORD, ())
+    vararg = _extract_vararg(python_vararg, args_node)
+    kwarg = _extract_vararg(python_kwarg, args_node)
+
     returns = None
     if signature.return_annotation is not _Parameter.empty:
         returns = _ast_from_object(signature.return_annotation,
                                    built_objects,
                                    module,
                                    parent=func_node)[0]
-    args_node.postinit(args, defaults, kwonlyargs, kw_defaults,
-                       annotations, kwonly_annotations,
-                       varargannotation, kwargannotation)
+    args_node.postinit(args=list(args),
+                       vararg=vararg,
+                       kwarg=kwarg,
+                       keyword_only=list(keyword_only),
+                       positional_only=list(positional_only))
     func_node.postinit(args=args_node, body=[], returns=returns)
+
     for name in set(dir(func)) - set(dir(type(func))):
         # This checks against method special attributes because
         # methods are also dispatched through this function.

astroid/test_utils.py

@@ -19,6 +19,7 @@
 # when calling extract_node.
 _STATEMENT_SELECTOR = '#@'
 
+
 def _extract_expressions(node):
     """Find expressions in a call to _TRANSIENT_FUNCTION and extract them.
 
@@ -46,8 +47,18 @@ def _extract_expressions(node):
             child = getattr(node.parent, name)
             if isinstance(child, (list, tuple)):
                 for idx, compound_child in enumerate(child):
-                    if compound_child is node:
+
+                    # Can't find a cleaner way to do this.
+                    if isinstance(compound_child, nodes.Parameter):
+                        if compound_child.default is node:
+                            child[idx].default = real_expr
+                        elif compound_child.annotation is node:
+                            child[idx].annotation = real_expr
+                        else:
+                            child[idx] = real_expr
+                    elif compound_child is node:
                         child[idx] = real_expr
+
             elif child is node:
                 setattr(node.parent, name, real_expr)
         yield real_expr

astroid/tests/unittest_brain.py

@@ -84,7 +84,9 @@ def test_hashlib(self):
             self.assertIn('block_size', class_obj)
             self.assertIn('digest_size', class_obj)
             self.assertEqual(len(class_obj['__init__'].args.args), 2)
-            self.assertEqual(len(class_obj['__init__'].args.defaults), 1)
+            default = class_obj['__init__'].args.args[1].default
+            self.assertIsInstance(default, nodes.Const)
+            self.assertEqual(default.value, '')
             self.assertEqual(len(class_obj['update'].args.args), 2)
             self.assertEqual(len(class_obj['digest'].args.args), 1)
             self.assertEqual(len(class_obj['hexdigest'].args.args), 1)

astroid/tests/unittest_nodes.py

@@ -491,6 +491,9 @@ def hello(False):
 
 
 class ArgumentsNodeTC(unittest.TestCase):
+
+    @unittest.skipIf(sys.version_info[:2] == (3, 3),
+                     "Line numbering is broken on Python 3.3.")
     def test_linenumbering(self):
         ast = builder.parse('''
             def func(a,

astroid/tests/unittest_python3.py

@@ -20,6 +20,7 @@
 
 from astroid import nodes
 from astroid.tree.node_classes import Assign, Expr, YieldFrom, Name, Const
+from astroid import raw_building
 from astroid.builder import AstroidBuilder
 from astroid.tree.scoped_nodes import ClassDef, FunctionDef
 from astroid.test_utils import require_version, extract_node
@@ -187,31 +188,31 @@ def test(a: int, b: str, c: None, d, e,
             pass
         """))
         func = astroid['test']
-        self.assertIsInstance(func.args.varargannotation, Name)
-        self.assertEqual(func.args.varargannotation.name, 'float')
-        self.assertIsInstance(func.args.kwargannotation, Name)
-        self.assertEqual(func.args.kwargannotation.name, 'int')
+        self.assertIsInstance(func.args.vararg.annotation, Name)
+        self.assertEqual(func.args.vararg.annotation.name, 'float')
+        self.assertIsInstance(func.args.kwarg.annotation, Name)
+        self.assertEqual(func.args.kwarg.annotation.name, 'int')
         self.assertIsInstance(func.returns, Name)
         self.assertEqual(func.returns.name, 'int')
         arguments = func.args
-        self.assertIsInstance(arguments.annotations[0], Name)
-        self.assertEqual(arguments.annotations[0].name, 'int')
-        self.assertIsInstance(arguments.annotations[1], Name)
-        self.assertEqual(arguments.annotations[1].name, 'str')
-        self.assertIsInstance(arguments.annotations[2], Const)
-        self.assertIsNone(arguments.annotations[2].value)
-        self.assertIsNone(arguments.annotations[3])
-        self.assertIsNone(arguments.annotations[4])
+        self.assertIsInstance(arguments.args[0].annotation, Name)
+        self.assertEqual(arguments.args[0].annotation.name, 'int')
+        self.assertIsInstance(arguments.args[1].annotation, Name)
+        self.assertEqual(arguments.args[1].annotation.name, 'str')
+        self.assertIsInstance(arguments.args[2].annotation, Const)
+        self.assertIsNone(arguments.args[2].annotation.value)
+        self.assertIs(arguments.args[3].annotation, nodes.Empty)
+        self.assertIs(arguments.args[4].annotation, nodes.Empty)
 
         astroid = self.builder.string_build(dedent("""
         def test(a: int=1, b: str=2):
             pass
         """))
         func = astroid['test']
-        self.assertIsInstance(func.args.annotations[0], Name)
-        self.assertEqual(func.args.annotations[0].name, 'int')
-        self.assertIsInstance(func.args.annotations[1], Name)
-        self.assertEqual(func.args.annotations[1].name, 'str')
+        self.assertIsInstance(func.args.args[0].annotation, Name)
+        self.assertEqual(func.args.args[0].annotation.name, 'int')
+        self.assertIsInstance(func.args.args[1].annotation, Name)
+        self.assertEqual(func.args.args[1].annotation.name, 'str')
         self.assertIsNone(func.returns)
 
     @require_version('3.0')
@@ -249,6 +250,13 @@ def test_unpacking_in_dict_getitem(self):
             self.assertIsInstance(value, nodes.Const)
             self.assertEqual(value.value, expected)
 
+    @require_version('3.4')
+    def test_positional_only_parameters(self):
+        ast = raw_building.ast_from_object(issubclass)
+        self.assertEqual(len(ast.args.positional_only), 2)
+        for name, arg in zip(('cls', 'class_or_tuple'), ast.args.positional_only):
+            self.assertEqual(arg.name, name)
+
 
 if __name__ == '__main__':
     unittest.main()

astroid/tests/unittest_scoped_nodes.py

@@ -284,7 +284,7 @@ def nested_args(a, (b, c, d)):
         tree = builder.parse(code)
         func = tree['nested_args']
         self.assertEqual(sorted(func.locals), ['a', 'b', 'c', 'd'])
-        self.assertEqual(func.args.format_args(), 'a, (b, c, d)')
+        self.assertEqual(func.args.format_args(), 'a, b, c, d')
 
     def test_four_args(self):
         func = self.module['four_args']

astroid/tree/base.py

@@ -633,7 +633,7 @@ def _filter_stmts(self, stmts, frame, offset):
                 if not (optional_assign or interpreterutil.are_exclusive(_stmts[pindex], node)):
                     del _stmt_parents[pindex]
                     del _stmts[pindex]
-            if isinstance(node, treeabc.AssignName):
+            if isinstance(node, (treeabc.Parameter, treeabc.AssignName)):
                 if not optional_assign and stmt.parent is mystmt.parent:
                     _stmts = []
                     _stmt_parents = []