Add initial version of stubtest script developed by Jukka

(With some minor fixes.)

Author: ddfisher

scripts/dumpmodule.py

@@ -0,0 +1,151 @@
+"""Dump the runtime structure of a module as JSON.
+
+This is used for testing stubs.
+
+This needs to run in Python 2.7 and 3.x.
+"""
+
+from __future__ import print_function
+
+import importlib
+import json
+import sys
+import types
+from typing import Text
+
+
+if sys.version_info >= (3, 0):
+    import inspect
+    long = int
+else:
+    import inspect2 as inspect
+
+
+
+def dump_module(id):
+    m = importlib.import_module(id)
+    data = module_to_json(m)
+    print(json.dumps(data, ensure_ascii=True, indent=4, sort_keys=True))
+
+
+def module_to_json(m):
+    result = {}
+    for name, value in m.__dict__.items():
+        # Filter out some useless attributes.
+        if name in ('__file__',
+                    '__doc__',
+                    '__name__',
+                    '__builtins__',
+                    '__package__'):
+            continue
+        if name == '__all__':
+            result[name] = sorted(value)
+        else:
+            result[name] = dump_value(value)
+    return result
+
+
+def dump_value(value, depth=0):
+    if depth > 10:
+        return 'max_recursion_depth_exceeded'
+    if isinstance(value, type):
+        return dump_class(value, depth + 1)
+    if inspect.isfunction(value):
+        return dump_function(value)
+    if callable(value):
+        return 'callable'  # TODO more information
+    if isinstance(value, types.ModuleType):
+        return 'module'  # TODO module name
+    if inspect.isdatadescriptor(value):
+        return 'datadescriptor'
+
+    if inspect.ismemberdescriptor(value):
+        return 'memberdescriptor'
+    return dump_simple(value)
+
+
+def dump_simple(value):
+    if type(value) in (int, bool, float, str, bytes, Text, long, list, set, dict, tuple):
+        return type(value).__name__
+    if value is None:
+        return 'None'
+    if value is inspect.Parameter.empty:
+        return None
+    return 'unknown'
+
+
+def dump_class(value, depth):
+    return {
+        'type': 'class',
+        'attributes': dump_attrs(value, depth),
+    }
+
+
+special_methods = [
+    '__init__',
+    '__str__',
+    '__int__',
+    '__float__',
+    '__bool__',
+    '__contains__',
+    '__iter__',
+]
+
+
+def dump_attrs(d, depth):
+    result = []
+    seen = set()
+    try:
+        mro = d.mro()
+    except TypeError:
+        mro = [d]
+    for base in mro:
+        v = vars(base)
+        for name, value in v.items():
+            if name not in seen:
+                result.append([name, dump_value(value, depth + 1)])
+                seen.add(name)
+    for m in special_methods:
+        if hasattr(d, m) and m not in seen:
+            result.append([m, dump_value(getattr(d, m), depth + 1)])
+    return result
+
+
+kind_map = {
+    inspect.Parameter.POSITIONAL_ONLY: 'POS_ONLY',
+    inspect.Parameter.POSITIONAL_OR_KEYWORD: 'POS_OR_KW',
+    inspect.Parameter.VAR_POSITIONAL: 'VAR_POS',
+    inspect.Parameter.KEYWORD_ONLY: 'KW_ONLY',
+    inspect.Parameter.VAR_KEYWORD: 'VAR_KW',
+}
+
+
+def param_kind(p):
+    s = kind_map[p.kind]
+    if p.default != inspect.Parameter.empty:
+        assert s in ('POS_ONLY', 'POS_OR_KW', 'KW_ONLY')
+        s += '_OPT'
+    return s
+
+
+def dump_function(value):
+    try:
+        sig = inspect.signature(value)
+    except ValueError:
+        # The signature call sometimes fails for some reason.
+        return 'invalid_signature'
+    params = list(sig.parameters.items())
+    return {
+        'type': 'function',
+        'args': [(name,
+             param_kind(p),
+             dump_simple(p.default))
+            for name, p in params],
+    }
+
+
+if __name__ == '__main__':
+    import sys
+    if len(sys.argv) != 2:
+        sys.exit('usage: dumpmodule.py module-name')
+    dump_module(sys.argv[1])

scripts/stubtest.py

@@ -0,0 +1,122 @@
+"""Tests for stubs.
+
+Verify that various things in stubs are consistent with how things behave
+at runtime.
+"""
+
+import importlib
+import json
+import subprocess
+import sys
+from typing import Dict, Any
+
+from mypy import build
+from mypy.build import default_data_dir, default_lib_path, find_modules_recursive
+from mypy.errors import CompileError
+from mypy.nodes import MypyFile, TypeInfo, FuncItem
+from mypy.options import Options
+
+
+skipped = {
+    '_importlib_modulespec',
+    '_subprocess',
+    'distutils.command.bdist_msi',
+    'distutils.command.bdist_packager',
+    'msvcrt',
+    'wsgiref.types',
+}
+
+
+class Errors:
+    def __init__(self, id):
+        self.id = id
+        self.num_errors = 0
+
+    def fail(self, msg):
+        print('{}: {}'.format(self.id, msg))
+        self.num_errors += 1
+
+
+def test_stub(id: str) -> None:
+    result = build_stubs(id)
+    verify_stubs(result.files, prefix=id)
+
+
+def verify_stubs(files: Dict[str, MypyFile], prefix: str) -> None:
+    for id, node in files.items():
+        if not (id == prefix or id.startswith(prefix + '.')):
+            # Not one of the target modules
+            continue
+        if id in skipped:
+            # There's some issue with processing this module; skip for now
+            continue
+        dumped = dump_module(id)
+        verify_stub(id, node.names, dumped)
+
+
+def verify_stub(id, symbols, dumped):
+    errors = Errors(id)
+    for name, node in symbols.items():
+        if name.startswith('_'):
+            # Private attribute
+            continue
+        if not node.module_public:
+            # Again, a private attribute
+            continue
+        if name not in dumped:
+            errors.fail('"{}" defined in stub but not at runtime'.format(name))
+        else:
+            verify_node(name, node, dumped[name], errors)
+
+
+def verify_node(name, node, dump, errors):
+    if isinstance(node.node, TypeInfo):
+        if not isinstance(dump, dict) or dump['type'] != 'class':
+            errors.fail('"{}" is a class in stub but not at runtime'.format(name))
+            return
+        all_attrs = {x[0] for x in dump['attributes']}
+        for attr, attr_node in node.node.names.items():
+            if isinstance(attr_node.node, FuncItem) and attr not in all_attrs:
+                errors.fail(
+                    ('"{}.{}" defined as a method in stub but not defined '
+                     'at runtime in class object').format(
+                        name, attr))
+    # TODO other kinds of nodes
+
+
+def dump_module(id: str) -> Dict[str, Any]:
+    try:
+        o = subprocess.check_output(
+            ['python', 'scripts/dumpmodule.py', id])
+    except subprocess.CalledProcessError:
+        print('Failure to dump module contents of "{}"'.format(id))
+        sys.exit(1)
+    return json.loads(o.decode('ascii'))
+
+
+def build_stubs(id):
+    data_dir = default_data_dir(None)
+    options = Options()
+    options.python_version = (3, 6)
+    lib_path = default_lib_path(data_dir,
+                                options.python_version,
+                                custom_typeshed_dir=None)
+    sources = find_modules_recursive(id, lib_path)
+    if not sources:
+        sys.exit('Error: Cannot find module {}'.format(repr(id)))
+    msg = []
+    try:
+        res = build.build(sources=sources,
+                          options=options)
+        msg = res.errors
+    except CompileError as e:
+        msg = e.messages
+    if msg:
+        for m in msg:
+            print(m)
+        sys.exit(1)
+    return res
+
+
+if __name__ == '__main__':
+    test_stub(sys.argv[1])