Rough-draft of Python 3 compatibility

setup.py

@@ -21,7 +21,7 @@
 
 
 if not sys.version_info[:2] in [(2, 6), (2, 7), (3, 4)]:
-    raise RuntimeError("Python version 2.6 or 2.7 required.")
+    raise RuntimeError("Python version 2.6, 2.7, or 3.4 required.")
 
 
 CLASSIFIERS = """\

weave/__init__.py

@@ -22,7 +22,7 @@
 
 
 if not sys.version_info[:2] in [(2, 6), (2, 7), (3, 4)]:
-    raise RuntimeError("Weave only supports Python 2.6 and 2.7")
+    raise RuntimeError("Weave only supports Python 2.6, 2.7, and 3.4")
 
 
 from weave.version import version as __version__

weave/_dumbdbm_patched.py

@@ -54,15 +54,14 @@ def __init__(self, file):
         self._update()
 
     def _update(self):
-        import string
         self._index = {}
         try:
             f = _open(self._dirfile)
         except IOError:
             pass
         else:
             while 1:
-                line = string.rstrip(f.readline())
+                line = f.readline().rstrip()
                 if not line:
                     break
                 key, (pos, siz) = eval(line)
@@ -102,7 +101,7 @@ def _addval(self, val):
 ##      pos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE
 ##      f.seek(pos)
         npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE
-        f.write('\0'*(npos-pos))
+        f.write(b'\0'*(npos-pos))
         pos = npos
 
         f.write(val)

weave/build_tools.py

@@ -261,9 +261,12 @@ def build_extension(module_path,compiler_name='',build_dir=None,
         ext = create_extension(module_path,**kw)
         # the switcheroo on SystemExit here is meant to keep command line
         # sessions from exiting when compiles fail.
-        builtin = sys.modules['__builtin__']
-        old_SysExit = builtin.__dict__['SystemExit']
-        builtin.__dict__['SystemExit'] = CompileError
+        if sys.version_info.major == 3:
+            builtin_dict = __builtins__
+        else:
+            builtin_dict = sys.modules['__builtin__'].__dict__
+        old_SysExit = builtin_dict['SystemExit']
+        builtin_dict['SystemExit'] = CompileError
 
         # change current working directory to 'build_dir' so compiler won't
         # pick up anything by mistake
@@ -280,7 +283,7 @@ def build_extension(module_path,compiler_name='',build_dir=None,
             # restore state
             os.environ = environ
             # restore SystemExit
-            builtin.__dict__['SystemExit'] = old_SysExit
+            builtin_dict['SystemExit'] = old_SysExit
             # restore working directory to one before setup
             os.chdir(oldcwd)
         t2 = time.time()
@@ -398,16 +401,6 @@ def msvc_exists():
             pass
     return result
 
-if os.name == 'nt':
-    def run_command(command):
-        """ not sure how to get exit status on nt. """
-        p = subprocess.Popen(['cl'], shell=True, stdout=subprocess.PIPE,
-                stderr=subprocess.STDOUT)
-        text = p.stdout.read()
-        return 0, text
-else:
-    run_command = commands.getstatusoutput
-
 
 def configure_temp_dir(temp_dir=None):
     if temp_dir is None:

weave/c_spec.py

@@ -217,76 +217,19 @@ class string_converter(common_base_converter):
     def init_info(self):
         common_base_converter.init_info(self)
         self.type_name = 'string'
-        self.check_func = 'PyString_Check'
+        self.check_func = 'PyUnicode_Check'
         self.c_type = 'std::string'
         self.return_type = 'std::string'
-        self.to_c_return = "std::string(PyString_AsString(py_obj))"
-        self.matching_types = [bytes]
+        self.to_c_return = "std::string(PyUnicode_AsUTF8(py_obj))"
+        self.matching_types = [str]
         self.headers.append('<string>')
 
     def c_to_py_code(self):
         # !! Need to dedent returned code.
         code = """
                PyObject* string_to_py(std::string s)
                {
-                   return PyString_FromString(s.c_str());
-               }
-               """
-        return code
-
-#----------------------------------------------------------------------------
-# Unicode Converter
-#----------------------------------------------------------------------------
-
-
-class unicode_converter(common_base_converter):
-    def init_info(self):
-        common_base_converter.init_info(self)
-        self.type_name = 'unicode'
-        self.check_func = 'PyUnicode_Check'
-        # This isn't supported by gcc 2.95.3 -- MSVC works fine with it.
-        # self.c_type = 'std::wstring'
-        # self.to_c_return = "std::wstring(PyUnicode_AS_UNICODE(py_obj))"
-        self.c_type = 'Py_UNICODE*'
-        self.return_type = self.c_type
-        self.to_c_return = "PyUnicode_AS_UNICODE(py_obj)"
-        self.matching_types = [str]
-        # self.headers.append('<string>')
-
-    def declaration_code(self,templatize=0,inline=0):
-        # since wstring doesn't seem to work everywhere, we need to provide
-        # the length variable Nxxx for the unicode string xxx.
-        code = '%(py_var)s = %(var_lookup)s;\n'                     \
-               '%(c_type)s %(name)s = %(var_convert)s;\n'           \
-               'int N%(name)s = PyUnicode_GET_SIZE(%(py_var)s);\n'  \
-               % self.template_vars(inline=inline)
-
-        return code
-#----------------------------------------------------------------------------
-# File Converter
-#----------------------------------------------------------------------------
-
-
-class file_converter(common_base_converter):
-    def init_info(self):
-        common_base_converter.init_info(self)
-        self.type_name = 'file'
-        self.check_func = 'PyFile_Check'
-        self.c_type = 'FILE*'
-        self.return_type = self.c_type
-        self.to_c_return = "PyFile_AsFile(py_obj)"
-        self.headers = ['<stdio.h>']
-        self.matching_types = [io.IOBase]
-
-    def c_to_py_code(self):
-        # !! Need to dedent returned code.
-        code = """
-               PyObject* file_to_py(FILE* file, const char* name,
-                                    const char* mode)
-               {
-                   return (PyObject*) PyFile_FromFile(file,
-                     const_cast<char*>(name),
-                     const_cast<char*>(mode), fclose);
+                   return PyUnicode_FromString(s.c_str());
                }
                """
         return code
@@ -308,8 +251,6 @@ def c_to_py_code(self):
 num_to_c_types[type(1)] = 'long'
 num_to_c_types[type(1.)] = 'double'
 num_to_c_types[type(1.+1.j)] = 'std::complex<double> '
-# !! hmmm. The following is likely unsafe...
-num_to_c_types[long] = 'npy_longlong'
 
 #----------------------------------------------------------------------------
 # Numeric array Python numeric --> C type maps
@@ -364,9 +305,9 @@ def init_info(self):
         # !! long to int conversion isn't safe!
         self.type_name = 'long'
         self.check_func = 'PyLong_Check'
-        self.c_type = 'longlong'
-        self.return_type = 'longlong'
-        self.to_c_return = "(longlong) PyLong_AsLongLong(py_obj)"
+        self.c_type = 'long'
+        self.return_type = 'long'
+        self.to_c_return = "(long) PyLong_AsLong(py_obj)"
         self.matching_types = [int]
 
 
@@ -451,23 +392,6 @@ def init_info(self):
         # ref counting handled by py::dict
         self.use_ref_count = 0
 
-#----------------------------------------------------------------------------
-# Instance Converter
-#----------------------------------------------------------------------------
-
-
-class instance_converter(scxx_converter):
-    def init_info(self):
-        scxx_converter.init_info(self)
-        self.type_name = 'instance'
-        self.check_func = 'PyInstance_Check'
-        self.c_type = 'py::object'
-        self.return_type = 'py::object'
-        self.to_c_return = 'py::object(py_obj)'
-        self.matching_types = [object]
-        # ref counting handled by py::object
-        self.use_ref_count = 0
-
 #----------------------------------------------------------------------------
 # Catchall Converter
 #

weave/catalog.py

@@ -92,7 +92,7 @@ def expr_to_filename(expr):
     base = 'sc_'
     # 32 chars is enough for unique filenames; too long names don't work for
     # MSVC (see gh-3216).  Don't use md5, gives a FIPS warning.
-    return base + sha256(expr).hexdigest()[:32]
+    return base + sha256(bytes(expr, 'utf-8')).hexdigest()[:32]
 
 
 def unique_file(d,expr):
@@ -107,7 +107,7 @@ def unique_file(d,expr):
     """
     files = os.listdir(d)
     base = expr_to_filename(expr)
-    for i in xrange(1000000):
+    for i in range(1000000):
         fname = base + repr(i)
         if not (fname+'.cpp' in files or
                 fname+'.o' in files or
@@ -634,7 +634,7 @@ def file_test(x):
                     access(os.path.dirname(x),W_OK))
         writable = filter(file_test,files)
         if writable:
-            file = writable[0]
+            file = next(writable)
         else:
             file = None
         return file

weave/common_info.py

@@ -20,18 +20,15 @@
 {
     if(py_obj == NULL) return "C NULL value";
     if(PyCallable_Check(py_obj)) return "callable";
-    if(PyString_Check(py_obj)) return "string";
-    if(PyInt_Check(py_obj)) return "int";
+    if(PyUnicode_Check(py_obj)) return "string";
+    if(PyLong_Check(py_obj)) return "long";
     if(PyFloat_Check(py_obj)) return "float";
     if(PyDict_Check(py_obj)) return "dict";
     if(PyList_Check(py_obj)) return "list";
     if(PyTuple_Check(py_obj)) return "tuple";
-    if(PyFile_Check(py_obj)) return "file";
     if(PyModule_Check(py_obj)) return "module";
 
-    //should probably do more intergation (and thinking) on these.
-    if(PyCallable_Check(py_obj) && PyInstance_Check(py_obj)) return "callable";
-    if(PyInstance_Check(py_obj)) return "instance";
+    //should probably do more integration (and thinking) on these.
     if(PyCallable_Check(py_obj)) return "callable";
     return "unknown type";
 }

weave/converters.py

@@ -7,16 +7,13 @@
 # The "standard" conversion classes
 #----------------------------------------------------------------------------
 
-default = [c_spec.int_converter(),
+default = [c_spec.long_converter(),
            c_spec.float_converter(),
            c_spec.complex_converter(),
-           c_spec.unicode_converter(),
            c_spec.string_converter(),
            c_spec.list_converter(),
            c_spec.dict_converter(),
-           c_spec.tuple_converter(),
-           c_spec.file_converter(),
-           c_spec.instance_converter(),]
+           c_spec.tuple_converter(),]
 
 #----------------------------------------------------------------------------
 # add numpy array converters to the default

weave/ext_tools.py

@@ -200,22 +200,12 @@ def add_function(self,func):
 
     def module_code(self):
         code = '\n'.join([
-            """\
-#ifdef __CPLUSPLUS__
-extern "C" {
-#endif
-""",
             self.warning_code(),
             self.header_code(),
             self.support_code(),
             self.function_code(),
             self.python_function_definition_code(),
             self.module_init_code(),
-            """\
-#ifdef __CPLUSCPLUS__
-}
-#endif
-"""
             ])
         return code
 
@@ -234,6 +224,7 @@ def build_information(self):
         for i in info:
             i.set_compiler(self.compiler)
         return info
+        return base_info.info_list(info)
 
     def get_headers(self):
         all_headers = self.build_information().headers()
@@ -292,11 +283,40 @@ def python_function_definition_code(self):
     def module_init_code(self):
         init_code_list = self.build_information().module_init_code()
         init_code = indent(''.join(init_code_list),4)
-        code = 'PyMODINIT_FUNC init%s(void)\n' \
-               '{\n' \
-               '%s' \
-               '    (void) Py_InitModule("%s", compiled_methods);\n' \
-               '}\n' % (self.name,init_code,self.name)
+        if sys.version_info.major < 3:
+            code = 'PyMODINIT_FUNC init%s(void)\n' \
+                   '{\n' \
+                   '%s' \
+                   '    (void) Py_InitModule("%s", compiled_methods);\n' \
+                   '}\n' % (self.name,init_code,self.name)
+        else:
+            code = 'static int traverse(PyObject *m, visitproc visit, void *arg) {\n' \
+                   '    return 0;\n' \
+                   '}\n' \
+                   '\n' \
+                   'static int clear(PyObject *m) {\n' \
+                   '    return 0;\n' \
+                   '}\n' \
+                   '\n' \
+                   'static struct PyModuleDef moduledef = {\n' \
+                   '        PyModuleDef_HEAD_INIT,\n' \
+                   '        "%s", NULL,\n' \
+                   '        0,\n' \
+                   '        compiled_methods,\n' \
+                   '        NULL,\n' \
+                   '        traverse,\n' \
+                   '        clear,\n' \
+                   '        NULL\n' \
+                   '};\n' \
+                   '\n' \
+                   'extern "C"\n' \
+                   'PyObject *PyInit_%s(void)\n' \
+                   '{\n' \
+                   '    PyObject *module = PyModule_Create(&moduledef);\n' \
+                   '%s' \
+                   '    return module;\n' \
+                   '}\n' % (self.name,self.name,init_code)
+
         return code
 
     def generate_file(self,file_name="",location='.'):
@@ -338,7 +358,7 @@ def build_kw_and_file(self,location,kw):
                                    info.extra_compile_args()
         kw['extra_link_args'] = kw.get('extra_link_args',[]) + \
                                    info.extra_link_args()
-        kw['sources'] = kw.get('sources',[]) + source_files
+        kw['sources'] = kw.get('sources',[]) + list(source_files)
         file = self.generate_file(location=location)
         return kw,file