bpo-29463: Add docstring field to some AST nodes. (#46)

* bpo-29463: Add docstring field to some AST nodes.

ClassDef, ModuleDef, FunctionDef, and AsyncFunctionDef has docstring
field for now.  It was first statement of there body.

* fix document.  thanks travis!

* doc fixes

Doc/library/ast.rst

@@ -133,9 +133,17 @@ and classes for traversing abstract syntax trees:
 .. function:: get_docstring(node, clean=True)
 
    Return the docstring of the given *node* (which must be a
-   :class:`FunctionDef`, :class:`ClassDef` or :class:`Module` node), or ``None``
-   if it has no docstring.  If *clean* is true, clean up the docstring's
-   indentation with :func:`inspect.cleandoc`.
+   :class:`FunctionDef`, :class:`AsyncFunctionDef`, :class:`ClassDef`,
+   or :class:`Module` node), or ``None`` if it has no docstring.
+   If *clean* is true, clean up the docstring's indentation with
+   :func:`inspect.cleandoc`.
+
+   .. versionchanged:: 3.5
+      :class:`AsyncFunctionDef` is now supported.
+
+   .. versionchanged:: 3.7
+      The docstring is now exported from the node docstring field, instead of
+      the first body statement.
 
 
 .. function:: fix_missing_locations(node)

Doc/whatsnew/3.7.rst

@@ -196,6 +196,12 @@ Changes in the Python API
   Use the :meth:`~http.cookies.Morsel.set` method for setting them.
   (Contributed by Serhiy Storchaka in :issue:`29192`.)
 
+* ``Module``, ``FunctionDef``, ``AsyncFunctionDef``, and
+  ``ClassDef`` AST nodes now have a new ``docstring`` field.
+  The first statement in their body is not considered as a docstring
+  anymore.  ``co_firstlineno`` and ``co_lnotab`` of code object for class
+  and module are affected by this change.
+  (Contributed by INADA Naoki and Eugene Toder in :issue:`29463`.)
 
 CPython bytecode changes
 ------------------------

Include/Python-ast.h

@@ -46,6 +46,7 @@ struct _mod {
     union {
         struct {
             asdl_seq *body;
+            string docstring;
         } Module;
 
         struct {
@@ -80,6 +81,7 @@ struct _stmt {
             asdl_seq *body;
             asdl_seq *decorator_list;
             expr_ty returns;
+            string docstring;
         } FunctionDef;
 
         struct {
@@ -88,6 +90,7 @@ struct _stmt {
             asdl_seq *body;
             asdl_seq *decorator_list;
             expr_ty returns;
+            string docstring;
         } AsyncFunctionDef;
 
         struct {
@@ -96,6 +99,7 @@ struct _stmt {
             asdl_seq *keywords;
             asdl_seq *body;
             asdl_seq *decorator_list;
+            string docstring;
         } ClassDef;
 
         struct {
@@ -439,26 +443,27 @@ struct _withitem {
 };
 
 
-#define Module(a0, a1) _Py_Module(a0, a1)
-mod_ty _Py_Module(asdl_seq * body, PyArena *arena);
+#define Module(a0, a1, a2) _Py_Module(a0, a1, a2)
+mod_ty _Py_Module(asdl_seq * body, string docstring, PyArena *arena);
 #define Interactive(a0, a1) _Py_Interactive(a0, a1)
 mod_ty _Py_Interactive(asdl_seq * body, PyArena *arena);
 #define Expression(a0, a1) _Py_Expression(a0, a1)
 mod_ty _Py_Expression(expr_ty body, PyArena *arena);
 #define Suite(a0, a1) _Py_Suite(a0, a1)
 mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
-#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
+#define FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_FunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
-                        asdl_seq * decorator_list, expr_ty returns, int lineno,
-                        int col_offset, PyArena *arena);
-#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7)
+                        asdl_seq * decorator_list, expr_ty returns, string
+                        docstring, int lineno, int col_offset, PyArena *arena);
+#define AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_AsyncFunctionDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_AsyncFunctionDef(identifier name, arguments_ty args, asdl_seq *
                              body, asdl_seq * decorator_list, expr_ty returns,
-                             int lineno, int col_offset, PyArena *arena);
-#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
+                             string docstring, int lineno, int col_offset,
+                             PyArena *arena);
+#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8)
 stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
-                     asdl_seq * body, asdl_seq * decorator_list, int lineno,
-                     int col_offset, PyArena *arena);
+                     asdl_seq * body, asdl_seq * decorator_list, string
+                     docstring, int lineno, int col_offset, PyArena *arena);
 #define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3)
 stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3)

Lib/ast.py

@@ -197,15 +197,7 @@ def get_docstring(node, clean=True):
     """
     if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)):
         raise TypeError("%r can't have docstrings" % node.__class__.__name__)
-    if not(node.body and isinstance(node.body[0], Expr)):
-        return
-    node = node.body[0].value
-    if isinstance(node, Str):
-        text = node.s
-    elif isinstance(node, Constant) and isinstance(node.value, str):
-        text = node.value
-    else:
-        return
+    text = node.docstring
     if clean:
         import inspect
         text = inspect.cleandoc(text)