Support windows (mypyc/mypyc#502)

This adds support for compiling on and for windows using MSVC.

There were two interesting things that needed to be done, and
a handful of uninteresting ones:
 1. As far as I could tell, Windows does not have the equivalents
    to rpath/$ORIGIN/@loader_path that we would need to have the
    dynamic loader automatically load our shared library.
    So instead we do it manually, computing a library
    location and using `LoadLibrary`/`GetProcAddress` to call our init
    functions.
 2. The Windows dynamic loader is less powerful than the Linux/OS X
    loaders, and does not allow you to initialize globals with values
    taken from other libraries. This means that we need to manually
    initialize such globals instead.

I didn't want to fight with running gtest even a little bit, so I
skipped the runtime tests on windows.

This passes the full test suite. I haven't tried mypy yet.

I haven't set up CI for windows yet, but it is important to do soon,
since it will be pretty easy to forget about the dynamic loader
limitations.

README.md

@@ -27,7 +27,9 @@ Linux Requirements
 Windows Requirements
 --------------------
 
-Windows is currently unsupported.
+* Windows has been tested with Windows 10 and MSVC 2017.
+
+* Python 3.5+ (64-bit)
 
 Quick Start for Contributors
 ----------------------------

lib-rt/CPy.h

@@ -64,6 +64,12 @@ static inline PyObject *CPyType_FromTemplate(PyTypeObject *template_,
     PyObject *bases = NULL;
     PyObject *slots;
 
+    // If the type of the class (the metaclass) is NULL, we default it
+    // to being type.  (This allows us to avoid needing to initialize
+    // it explicitly on windows.)
+    if (!Py_TYPE(template_)) {
+        Py_TYPE(template_) = &PyType_Type;
+    }
     PyTypeObject *metaclass = Py_TYPE(template_);
 
     if (orig_bases) {
@@ -897,7 +903,7 @@ static void CPy_AddTraceback(const char *filename, const char *funcname, int lin
 // exception APIs that might want to return NULL pointers instead
 // return properly refcounted pointers to this dummy object.
 struct ExcDummyStruct { PyObject_HEAD };
-static struct ExcDummyStruct _CPy_ExcDummyStruct = { PyObject_HEAD_INIT(&PyBaseObject_Type) };
+static struct ExcDummyStruct _CPy_ExcDummyStruct = { PyObject_HEAD_INIT(NULL) };
 static PyObject *_CPy_ExcDummy = (PyObject *)&_CPy_ExcDummyStruct;
 
 static inline void _CPy_ToDummy(PyObject **p) {
@@ -1014,6 +1020,14 @@ static void CPy_GetExcInfo(PyObject **p_type, PyObject **p_value, PyObject **p_t
     _CPy_ToNone(p_traceback);
 }
 
+// Because its dynamic linker is more restricted than linux/OS X,
+// Windows doesn't allow initializing globals with values from
+// other dynamic libraries. This means we need to initialize
+// things at load time.
+static void CPy_Init(void) {
+    _CPy_ExcDummyStruct.ob_base.ob_type = &PyBaseObject_Type;
+}
+
 #ifdef __cplusplus
 }
 #endif

lib-rt/mypyc_util.h

@@ -5,9 +5,21 @@
 #include <frameobject.h>
 #include <assert.h>
 
+#if defined(__clang__) || defined(__GNUC__)
 #define likely(x)       __builtin_expect((x),1)
 #define unlikely(x)     __builtin_expect((x),0)
 #define CPy_Unreachable() __builtin_unreachable()
+#else
+#define likely(x)       (x)
+#define unlikely(x)     (x)
+#define CPy_Unreachable() abort()
+#endif
+
+#if defined(_MSC_VER)
+#define CPy_dllexport __declspec(dllexport)
+#else
+#define CPy_dllexport
+#endif
 
 #define CPY_TAGGED_MAX ((1LL << 62) - 1)
 #define CPY_TAGGED_MIN (-(1LL << 62))

mypyc/build.py

@@ -57,7 +57,7 @@
     from distutils.core import setup, Extension
     from distutils.command.build_ext import build_ext  # type: ignore
 
-from distutils import sysconfig
+from distutils import sysconfig, ccompiler
 
 
 def setup_mypycify_vars() -> None:
@@ -126,7 +126,7 @@ def get_mypy_config(paths: List[str],
     return sources, options
 
 
-shim_template = """\
+shim_template_unix = """\
 #include <Python.h>
 
 PyObject *CPyInit_{full_modname}(void);
@@ -138,13 +138,76 @@ def get_mypy_config(paths: List[str],
 }}
 """
 
+# As far as I could tell, Windows lacks the rpath style features we
+# would need in automatically load the shared library (located
+# relative to the module library) when a module library is loaded,
+# which means that instead we get to do it dynamically.
+#
+# We do this by, at module initialization time, finding the location
+# of the module dll and using it to compute the location of the shared
+# library. We then load the shared library with LoadLibrary, find the
+# appropriate CPyInit_ routine using GetProcAddress, and call it.
+#
+# The relative path of the shared library (from the shim library) is provided
+# as the preprocessor define MYPYC_LIBRARY.
+shim_template_windows = r"""\
+#include <Python.h>
+#include <windows.h>
+#include <stdlib.h>
+#include <stdio.h>
+
+EXTERN_C IMAGE_DOS_HEADER __ImageBase;
+
+typedef PyObject *(__cdecl *INITPROC)();
+
+PyMODINIT_FUNC
+PyInit_{modname}(void)
+{{
+    char path[MAX_PATH];
+    char drive[MAX_PATH];
+    char directory[MAX_PATH];
+    HINSTANCE hinstLib;
+    INITPROC proc;
+
+    // get the file name of this dll
+    DWORD res = GetModuleFileName((HINSTANCE)&__ImageBase, path, sizeof(path));
+    if (res == 0 || res == sizeof(path)) {{
+        PyErr_SetString(PyExc_RuntimeError, "GetModuleFileName failed");
+        return NULL;
+    }}
+
+    // find the directory this dll is in
+    _splitpath(path, drive, directory, NULL, NULL);
+    // and use it to construct a path to the shared library
+    snprintf(path, sizeof(path), "%s%s%s", drive, directory, MYPYC_LIBRARY);
+
+    hinstLib = LoadLibrary(path);
+    if (!hinstLib) {{
+        PyErr_SetString(PyExc_RuntimeError, "LoadLibrary failed");
+        return NULL;
+    }}
+    proc = (INITPROC)GetProcAddress(hinstLib, "CPyInit_{full_modname}");
+    if (!proc) {{
+        PyErr_SetString(PyExc_RuntimeError, "GetProcAddress failed");
+        return NULL;
+    }}
+
+    return proc();
+}}
+
+// distutils sometimes spuriously tells cl to export CPyInit___init__,
+// so provide that so it chills out
+PyMODINIT_FUNC PyInit___init__(void) {{ return PyInit_{modname}(); }}
+"""
+
 
 def generate_c_extension_shim(full_module_name: str, module_name: str, dirname: str) -> str:
     """Create a C extension shim with a passthrough PyInit function."""
     cname = '%s.c' % full_module_name.replace('.', '___')  # XXX
     cpath = os.path.join(dirname, cname)
 
     with open(cpath, 'w') as f:
+        shim_template = shim_template_windows if sys.platform == 'win32' else shim_template_unix
         f.write(shim_template.format(modname=module_name,
                                      full_modname=exported_name(full_module_name)))
 
@@ -164,7 +227,7 @@ def include_dir() -> str:
 
 
 def generate_c(sources: List[BuildSource], options: Options,
-               use_shared_lib: bool) -> Tuple[str, str]:
+               shared_lib_name: Optional[str]) -> Tuple[str, str]:
     """Drive the actual core compilation step.
 
     Returns the C source code and (for debugging) the pretty printed IR.
@@ -184,7 +247,7 @@ def generate_c(sources: List[BuildSource], options: Options,
     print("Parsed and typechecked in {:.3f}s".format(t1 - t0))
 
     ops = []  # type: List[str]
-    ctext = emitmodule.compile_modules_to_c(result, module_names, use_shared_lib, ops=ops)
+    ctext = emitmodule.compile_modules_to_c(result, module_names, shared_lib_name, ops=ops)
 
     t2 = time.time()
     print("Compiled to C in {:.3f}s".format(t2 - t1))
@@ -272,6 +335,12 @@ def mypycify(paths: List[str],
 
     setup_mypycify_vars()
 
+    # Create a compiler object so we can make decisions based on what
+    # compiler is being used. typeshed is missing some attribues on the
+    # compiler object so we give it type Any
+    compiler = ccompiler.new_compiler()  # type: Any
+    sysconfig.customize_compiler(compiler)
+
     expanded_paths = []
     for path in paths:
         expanded_paths.extend(glob.glob(path))
@@ -289,29 +358,38 @@ def mypycify(paths: List[str],
     use_shared_lib = len(sources) > 1 or any('.' in x.module for x in sources)
     cfile = os.path.join(build_dir, '__native.c')
 
+    lib_name = shared_lib_name([source.module for source in sources]) if use_shared_lib else None
+
     # We let the test harness make us skip doing the full compilation
     # so that it can do a corner-cutting version without full stubs.
     # TODO: Be able to do this based on file mtimes?
     if not skip_cgen:
-        ctext, ops_text = generate_c(sources, options, use_shared_lib)
+        ctext, ops_text = generate_c(sources, options, lib_name)
         # TODO: unique names?
         with open(os.path.join(build_dir, 'ops.txt'), 'w') as f:
             f.write(ops_text)
-        with open(cfile, 'w') as f:
+        with open(cfile, 'w', encoding='utf-8') as f:
             f.write(ctext)
 
-    cflags = [
-        '-O{}'.format(opt_level),
-        '-Werror', '-Wno-unused-function', '-Wno-unused-label',
-        '-Wno-unreachable-code', '-Wno-unused-variable', '-Wno-trigraphs',
-        '-Wno-unused-command-line-argument'
-    ]
-    if sys.platform == 'linux' and 'clang' not in os.getenv('CC', ''):
-        # This flag is needed for gcc but does not exist on clang.
-        cflags += ['-Wno-unused-but-set-variable']
+    cflags = []  # type: List[str]
+    if compiler.compiler_type == 'unix':
+        cflags += [
+            '-O{}'.format(opt_level), '-Werror', '-Wno-unused-function', '-Wno-unused-label',
+            '-Wno-unreachable-code', '-Wno-unused-variable', '-Wno-trigraphs',
+            '-Wno-unused-command-line-argument'
+        ]
+        if 'gcc' in compiler.compiler[0]:
+            # This flag is needed for gcc but does not exist on clang.
+            cflags += ['-Wno-unused-but-set-variable']
+    elif compiler.compiler_type == 'msvc':
+        if opt_level == '3':
+            opt_level = '2'
+        cflags += [
+            '/O{}'.format(opt_level)
+        ]
 
     if use_shared_lib:
-        lib_name = shared_lib_name([source.module for source in sources])
+        assert lib_name
         extensions = build_using_shared_lib(sources, lib_name, cfile, build_dir, cflags)
     else:
         extensions = build_single_module(sources, cfile, cflags)
@@ -358,8 +436,18 @@ def build_extension(self, ext: MypycifyExtension) -> None:
             shared_dir, shared_file = os.path.split(
                 self.get_ext_fullpath(ext.mypyc_shared_target.name))
             shared_name = os.path.splitext(shared_file)[0][3:]
-            ext.libraries.append(shared_name)
-            ext.library_dirs.append(shared_dir)
+            if sys.platform == 'win32':
+                # On windows, instead of linking against the shared library,
+                # we dynamically load it at runtime. We generate our C shims
+                # before we have found out what the library filename is, so
+                # pass it in as a preprocessor define.
+                path = os.path.join(relative_lib_path, shared_file)
+                ext.extra_compile_args.append(
+                    '/DMYPYC_LIBRARY=\\"{}\\"'.format(path.replace('\\', '\\\\')))
+            else:
+                # On other platforms we link against the library normally
+                ext.libraries.append(shared_name)
+                ext.library_dirs.append(shared_dir)
             if sys.platform == 'linux':
                 ext.runtime_library_dirs.append('$ORIGIN/{}'.format(
                     relative_lib_path))

mypyc/emitclass.py

@@ -186,7 +186,7 @@ def emit_line() -> None:
     fields['tp_flags'] = ' | '.join(flags)
 
     emitter.emit_line("static PyTypeObject {}_template_ = {{".format(emitter.type_struct_name(cl)))
-    emitter.emit_line("PyVarObject_HEAD_INIT(&PyType_Type, 0)")
+    emitter.emit_line("PyVarObject_HEAD_INIT(NULL, 0)")
     for field, value in fields.items():
         emitter.emit_line(".{} = {},".format(field, value))
     emitter.emit_line("};")
@@ -305,6 +305,9 @@ def generate_vtable(entries: VTableEntries,
             cl, attr, is_setter = entry
             namer = native_setter_name if is_setter else native_getter_name
             emitter.emit_line('(CPyVTableItem){},'.format(namer(cl, attr, emitter.names)))
+    # msvc doesn't allow empty arrays; maybe allowing them at all is an extension?
+    if not entries:
+        emitter.emit_line('NULL')
     emitter.emit_line('};')