From f8ef226dbd1c2b9079cec786d41c8c7fa22ac345 Mon Sep 17 00:00:00 2001 From: Peter Iannucci Date: Fri, 31 Jul 2015 16:52:58 -0400 Subject: [PATCH] Rough-draft of Python 3 compatibility --- setup.py | 2 +- weave/__init__.py | 2 +- weave/_dumbdbm_patched.py | 5 +-- weave/build_tools.py | 21 +++------ weave/c_spec.py | 90 +++------------------------------------ weave/catalog.py | 6 +-- weave/common_info.py | 9 ++-- weave/converters.py | 7 +-- weave/ext_tools.py | 52 +++++++++++++++------- weave/inline_tools.py | 8 ++-- weave/platform_info.py | 2 +- weave/scxx/number.h | 2 +- weave/scxx/object.h | 45 +++++++------------- weave/scxx/str.h | 8 ++-- 14 files changed, 88 insertions(+), 171 deletions(-) mode change 100644 => 100755 weave/_dumbdbm_patched.py diff --git a/setup.py b/setup.py index e3f1efa..a02f981 100755 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ if not sys.version_info[:2] in [(2, 6), (2, 7), (3, 4)]: - raise RuntimeError("Python version 2.6 or 2.7 required.") + raise RuntimeError("Python version 2.6, 2.7, or 3.4 required.") CLASSIFIERS = """\ diff --git a/weave/__init__.py b/weave/__init__.py index fb95b26..d32a934 100644 --- a/weave/__init__.py +++ b/weave/__init__.py @@ -22,7 +22,7 @@ if not sys.version_info[:2] in [(2, 6), (2, 7), (3, 4)]: - raise RuntimeError("Weave only supports Python 2.6 and 2.7") + raise RuntimeError("Weave only supports Python 2.6, 2.7, and 3.4") from weave.version import version as __version__ diff --git a/weave/_dumbdbm_patched.py b/weave/_dumbdbm_patched.py old mode 100644 new mode 100755 index 0462ec6..cb62025 --- a/weave/_dumbdbm_patched.py +++ b/weave/_dumbdbm_patched.py @@ -54,7 +54,6 @@ def __init__(self, file): self._update() def _update(self): - import string self._index = {} try: f = _open(self._dirfile) @@ -62,7 +61,7 @@ def _update(self): pass else: while 1: - line = string.rstrip(f.readline()) + line = f.readline().rstrip() if not line: break key, (pos, siz) = eval(line) @@ -102,7 +101,7 @@ def _addval(self, val): ## pos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE ## f.seek(pos) npos = ((pos + _BLOCKSIZE - 1) // _BLOCKSIZE) * _BLOCKSIZE - f.write('\0'*(npos-pos)) + f.write(b'\0'*(npos-pos)) pos = npos f.write(val) diff --git a/weave/build_tools.py b/weave/build_tools.py index a4de6cd..a07d3cd 100644 --- a/weave/build_tools.py +++ b/weave/build_tools.py @@ -261,9 +261,12 @@ def build_extension(module_path,compiler_name='',build_dir=None, ext = create_extension(module_path,**kw) # the switcheroo on SystemExit here is meant to keep command line # sessions from exiting when compiles fail. - builtin = sys.modules['__builtin__'] - old_SysExit = builtin.__dict__['SystemExit'] - builtin.__dict__['SystemExit'] = CompileError + if sys.version_info.major == 3: + builtin_dict = __builtins__ + else: + builtin_dict = sys.modules['__builtin__'].__dict__ + old_SysExit = builtin_dict['SystemExit'] + builtin_dict['SystemExit'] = CompileError # change current working directory to 'build_dir' so compiler won't # pick up anything by mistake @@ -280,7 +283,7 @@ def build_extension(module_path,compiler_name='',build_dir=None, # restore state os.environ = environ # restore SystemExit - builtin.__dict__['SystemExit'] = old_SysExit + builtin_dict['SystemExit'] = old_SysExit # restore working directory to one before setup os.chdir(oldcwd) t2 = time.time() @@ -398,16 +401,6 @@ def msvc_exists(): pass return result -if os.name == 'nt': - def run_command(command): - """ not sure how to get exit status on nt. """ - p = subprocess.Popen(['cl'], shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - text = p.stdout.read() - return 0, text -else: - run_command = commands.getstatusoutput - def configure_temp_dir(temp_dir=None): if temp_dir is None: diff --git a/weave/c_spec.py b/weave/c_spec.py index 0234a51..961d666 100644 --- a/weave/c_spec.py +++ b/weave/c_spec.py @@ -217,11 +217,11 @@ class string_converter(common_base_converter): def init_info(self): common_base_converter.init_info(self) self.type_name = 'string' - self.check_func = 'PyString_Check' + self.check_func = 'PyUnicode_Check' self.c_type = 'std::string' self.return_type = 'std::string' - self.to_c_return = "std::string(PyString_AsString(py_obj))" - self.matching_types = [bytes] + self.to_c_return = "std::string(PyUnicode_AsUTF8(py_obj))" + self.matching_types = [str] self.headers.append('') def c_to_py_code(self): @@ -229,64 +229,7 @@ def c_to_py_code(self): code = """ PyObject* string_to_py(std::string s) { - return PyString_FromString(s.c_str()); - } - """ - return code - -#---------------------------------------------------------------------------- -# Unicode Converter -#---------------------------------------------------------------------------- - - -class unicode_converter(common_base_converter): - def init_info(self): - common_base_converter.init_info(self) - self.type_name = 'unicode' - self.check_func = 'PyUnicode_Check' - # This isn't supported by gcc 2.95.3 -- MSVC works fine with it. - # self.c_type = 'std::wstring' - # self.to_c_return = "std::wstring(PyUnicode_AS_UNICODE(py_obj))" - self.c_type = 'Py_UNICODE*' - self.return_type = self.c_type - self.to_c_return = "PyUnicode_AS_UNICODE(py_obj)" - self.matching_types = [str] - # self.headers.append('') - - def declaration_code(self,templatize=0,inline=0): - # since wstring doesn't seem to work everywhere, we need to provide - # the length variable Nxxx for the unicode string xxx. - code = '%(py_var)s = %(var_lookup)s;\n' \ - '%(c_type)s %(name)s = %(var_convert)s;\n' \ - 'int N%(name)s = PyUnicode_GET_SIZE(%(py_var)s);\n' \ - % self.template_vars(inline=inline) - - return code -#---------------------------------------------------------------------------- -# File Converter -#---------------------------------------------------------------------------- - - -class file_converter(common_base_converter): - def init_info(self): - common_base_converter.init_info(self) - self.type_name = 'file' - self.check_func = 'PyFile_Check' - self.c_type = 'FILE*' - self.return_type = self.c_type - self.to_c_return = "PyFile_AsFile(py_obj)" - self.headers = [''] - self.matching_types = [io.IOBase] - - def c_to_py_code(self): - # !! Need to dedent returned code. - code = """ - PyObject* file_to_py(FILE* file, const char* name, - const char* mode) - { - return (PyObject*) PyFile_FromFile(file, - const_cast(name), - const_cast(mode), fclose); + return PyUnicode_FromString(s.c_str()); } """ return code @@ -308,8 +251,6 @@ def c_to_py_code(self): num_to_c_types[type(1)] = 'long' num_to_c_types[type(1.)] = 'double' num_to_c_types[type(1.+1.j)] = 'std::complex ' -# !! hmmm. The following is likely unsafe... -num_to_c_types[long] = 'npy_longlong' #---------------------------------------------------------------------------- # Numeric array Python numeric --> C type maps @@ -364,9 +305,9 @@ def init_info(self): # !! long to int conversion isn't safe! self.type_name = 'long' self.check_func = 'PyLong_Check' - self.c_type = 'longlong' - self.return_type = 'longlong' - self.to_c_return = "(longlong) PyLong_AsLongLong(py_obj)" + self.c_type = 'long' + self.return_type = 'long' + self.to_c_return = "(long) PyLong_AsLong(py_obj)" self.matching_types = [int] @@ -451,23 +392,6 @@ def init_info(self): # ref counting handled by py::dict self.use_ref_count = 0 -#---------------------------------------------------------------------------- -# Instance Converter -#---------------------------------------------------------------------------- - - -class instance_converter(scxx_converter): - def init_info(self): - scxx_converter.init_info(self) - self.type_name = 'instance' - self.check_func = 'PyInstance_Check' - self.c_type = 'py::object' - self.return_type = 'py::object' - self.to_c_return = 'py::object(py_obj)' - self.matching_types = [object] - # ref counting handled by py::object - self.use_ref_count = 0 - #---------------------------------------------------------------------------- # Catchall Converter # diff --git a/weave/catalog.py b/weave/catalog.py index cd63aae..61fa9fb 100644 --- a/weave/catalog.py +++ b/weave/catalog.py @@ -92,7 +92,7 @@ def expr_to_filename(expr): base = 'sc_' # 32 chars is enough for unique filenames; too long names don't work for # MSVC (see gh-3216). Don't use md5, gives a FIPS warning. - return base + sha256(expr).hexdigest()[:32] + return base + sha256(bytes(expr, 'utf-8')).hexdigest()[:32] def unique_file(d,expr): @@ -107,7 +107,7 @@ def unique_file(d,expr): """ files = os.listdir(d) base = expr_to_filename(expr) - for i in xrange(1000000): + for i in range(1000000): fname = base + repr(i) if not (fname+'.cpp' in files or fname+'.o' in files or @@ -634,7 +634,7 @@ def file_test(x): access(os.path.dirname(x),W_OK)) writable = filter(file_test,files) if writable: - file = writable[0] + file = next(writable) else: file = None return file diff --git a/weave/common_info.py b/weave/common_info.py index c09aed7..cf849cb 100644 --- a/weave/common_info.py +++ b/weave/common_info.py @@ -20,18 +20,15 @@ { if(py_obj == NULL) return "C NULL value"; if(PyCallable_Check(py_obj)) return "callable"; - if(PyString_Check(py_obj)) return "string"; - if(PyInt_Check(py_obj)) return "int"; + if(PyUnicode_Check(py_obj)) return "string"; + if(PyLong_Check(py_obj)) return "long"; if(PyFloat_Check(py_obj)) return "float"; if(PyDict_Check(py_obj)) return "dict"; if(PyList_Check(py_obj)) return "list"; if(PyTuple_Check(py_obj)) return "tuple"; - if(PyFile_Check(py_obj)) return "file"; if(PyModule_Check(py_obj)) return "module"; - //should probably do more intergation (and thinking) on these. - if(PyCallable_Check(py_obj) && PyInstance_Check(py_obj)) return "callable"; - if(PyInstance_Check(py_obj)) return "instance"; + //should probably do more integration (and thinking) on these. if(PyCallable_Check(py_obj)) return "callable"; return "unknown type"; } diff --git a/weave/converters.py b/weave/converters.py index 79b3267..5c93a48 100644 --- a/weave/converters.py +++ b/weave/converters.py @@ -7,16 +7,13 @@ # The "standard" conversion classes #---------------------------------------------------------------------------- -default = [c_spec.int_converter(), +default = [c_spec.long_converter(), c_spec.float_converter(), c_spec.complex_converter(), - c_spec.unicode_converter(), c_spec.string_converter(), c_spec.list_converter(), c_spec.dict_converter(), - c_spec.tuple_converter(), - c_spec.file_converter(), - c_spec.instance_converter(),] + c_spec.tuple_converter(),] #---------------------------------------------------------------------------- # add numpy array converters to the default diff --git a/weave/ext_tools.py b/weave/ext_tools.py index 03a7fe0..81cc847 100644 --- a/weave/ext_tools.py +++ b/weave/ext_tools.py @@ -200,22 +200,12 @@ def add_function(self,func): def module_code(self): code = '\n'.join([ - """\ -#ifdef __CPLUSPLUS__ -extern "C" { -#endif -""", self.warning_code(), self.header_code(), self.support_code(), self.function_code(), self.python_function_definition_code(), self.module_init_code(), - """\ -#ifdef __CPLUSCPLUS__ -} -#endif -""" ]) return code @@ -234,6 +224,7 @@ def build_information(self): for i in info: i.set_compiler(self.compiler) return info + return base_info.info_list(info) def get_headers(self): all_headers = self.build_information().headers() @@ -292,11 +283,40 @@ def python_function_definition_code(self): def module_init_code(self): init_code_list = self.build_information().module_init_code() init_code = indent(''.join(init_code_list),4) - code = 'PyMODINIT_FUNC init%s(void)\n' \ - '{\n' \ - '%s' \ - ' (void) Py_InitModule("%s", compiled_methods);\n' \ - '}\n' % (self.name,init_code,self.name) + if sys.version_info.major < 3: + code = 'PyMODINIT_FUNC init%s(void)\n' \ + '{\n' \ + '%s' \ + ' (void) Py_InitModule("%s", compiled_methods);\n' \ + '}\n' % (self.name,init_code,self.name) + else: + code = 'static int traverse(PyObject *m, visitproc visit, void *arg) {\n' \ + ' return 0;\n' \ + '}\n' \ + '\n' \ + 'static int clear(PyObject *m) {\n' \ + ' return 0;\n' \ + '}\n' \ + '\n' \ + 'static struct PyModuleDef moduledef = {\n' \ + ' PyModuleDef_HEAD_INIT,\n' \ + ' "%s", NULL,\n' \ + ' 0,\n' \ + ' compiled_methods,\n' \ + ' NULL,\n' \ + ' traverse,\n' \ + ' clear,\n' \ + ' NULL\n' \ + '};\n' \ + '\n' \ + 'extern "C"\n' \ + 'PyObject *PyInit_%s(void)\n' \ + '{\n' \ + ' PyObject *module = PyModule_Create(&moduledef);\n' \ + '%s' \ + ' return module;\n' \ + '}\n' % (self.name,self.name,init_code) + return code def generate_file(self,file_name="",location='.'): @@ -338,7 +358,7 @@ def build_kw_and_file(self,location,kw): info.extra_compile_args() kw['extra_link_args'] = kw.get('extra_link_args',[]) + \ info.extra_link_args() - kw['sources'] = kw.get('sources',[]) + source_files + kw['sources'] = kw.get('sources',[]) + list(source_files) file = self.generate_file(location=location) return kw,file diff --git a/weave/inline_tools.py b/weave/inline_tools.py index 569467c..5ea18f9 100644 --- a/weave/inline_tools.py +++ b/weave/inline_tools.py @@ -331,7 +331,7 @@ def inline(code,arg_names=[],local_dict=None, global_dict=None, else: # 1. try local cache try: - results = apply(function_cache[code],(local_dict,global_dict)) + results = function_cache[code](local_dict,global_dict) return results except TypeError as msg: msg = str(msg).strip() @@ -377,7 +377,7 @@ def attempt_function_call(code,local_dict,global_dict): global function_catalog # 1. try local cache try: - results = apply(function_cache[code],(local_dict,global_dict)) + results = function_cache[code](local_dict,global_dict) return results except TypeError as msg: msg = str(msg).strip() @@ -397,7 +397,7 @@ def attempt_function_call(code,local_dict,global_dict): function_list = function_catalog.get_functions_fast(code) for func in function_list: try: - results = apply(func,(local_dict,global_dict)) + results = func(local_dict,global_dict) function_catalog.fast_cache(code,func) function_cache[code] = func return results @@ -421,7 +421,7 @@ def attempt_function_call(code,local_dict,global_dict): function_list = function_catalog.get_functions(code,module_dir) for func in function_list: try: - results = apply(func,(local_dict,global_dict)) + results = func(local_dict,global_dict) function_catalog.fast_cache(code,func) function_cache[code] = func return results diff --git a/weave/platform_info.py b/weave/platform_info.py index 0c95697..bad4ddf 100644 --- a/weave/platform_info.py +++ b/weave/platform_info.py @@ -98,7 +98,7 @@ def compiler_exe_path(exe_name): def check_sum(file): from hashlib import sha256 try: - f = open(file,'r') + f = open(file,'rb') bytes = f.read(-1) except IOError: bytes = '' diff --git a/weave/scxx/number.h b/weave/scxx/number.h index acbad4d..86f9363 100644 --- a/weave/scxx/number.h +++ b/weave/scxx/number.h @@ -99,7 +99,7 @@ class number : public object PyObject* Int = PyNumber_Int(_obj); if (Int==0) fail(PyExc_TypeError, "Cannot convert to long"); - long r = PyInt_AS_LONG(Int); + long r = PyLong_AS_LONG(Int); Py_DECREF(Int); return r; }; diff --git a/weave/scxx/object.h b/weave/scxx/object.h index a7074ad..0fb069a 100644 --- a/weave/scxx/object.h +++ b/weave/scxx/object.h @@ -80,16 +80,16 @@ class object // Numeric constructors //------------------------------------------------------------------------- object(bool val) { - _obj = _own = PyInt_FromLong((int)val); + _obj = _own = PyLong_FromLong((int)val); }; object(int val) { - _obj = _own = PyInt_FromLong((int)val); + _obj = _own = PyLong_FromLong((int)val); }; object(unsigned int val) { _obj = _own = PyLong_FromUnsignedLong(val); }; object(long val) { - _obj = _own = PyInt_FromLong(val); + _obj = _own = PyLong_FromLong(val); }; object(unsigned long val) { _obj = _own = PyLong_FromUnsignedLong(val); @@ -105,10 +105,10 @@ class object // string constructors //------------------------------------------------------------------------- object(const char* val) { - _obj = _own = PyString_FromString((char*) val); + _obj = _own = PyUnicode_FromString((char*) val); }; object(const std::string& val) : _obj (0), _own (0) { - _obj = _own = PyString_FromString((char*)val.c_str()); + _obj = _own = PyUnicode_FromString((char*)val.c_str()); }; //------------------------------------------------------------------------- @@ -127,7 +127,7 @@ class object operator int () const { int _val; - _val = PyInt_AsLong(_obj); + _val = PyLong_AsLong(_obj); if (PyErr_Occurred()) { PyErr_Clear(); fail(PyExc_TypeError, "cannot convert value to integer"); @@ -163,14 +163,14 @@ class object return std::complex(real, imag); }; operator std::string () const { - if (!PyString_Check(_obj)) + if (!PyUnicode_Check(_obj)) fail(PyExc_TypeError, "cannot convert value to std::string"); - return std::string(PyString_AsString(_obj)); + return std::string(PyUnicode_AsUTF8(_obj)); }; operator char* () const { - if (!PyString_Check(_obj)) + if (!PyUnicode_Check(_obj)) fail(PyExc_TypeError, "cannot convert value to char*"); - return PyString_AsString(_obj); + return PyUnicode_AsUTF8(_obj); }; //------------------------------------------------------------------------- @@ -414,12 +414,8 @@ class object // !! NOT TESTED //------------------------------------------------------------------------- int cmp(const object& other) const { - int rslt = 0; - int rc = PyObject_Cmp(_obj, other, &rslt); - if (rc == -1) - fail(PyExc_TypeError, "cannot make the comparison"); - return rslt; - }; + return PyObject_RichCompareBool(_obj, other, Py_LT) ? -1 : PyObject_RichCompareBool(_obj, other, Py_GT) ? 1 : 0; + }; int cmp(int other) const { object _other = object(other); return cmp(_other); @@ -587,23 +583,14 @@ class object object result = PyObject_Repr(_obj); if (!(PyObject*)result) throw 1; - return std::string(PyString_AsString(result)); + return std::string(PyUnicode_AsUTF8(result)); }; std::string str() const { object result = PyObject_Str(_obj); if (!(PyObject*)result) throw 1; - return std::string(PyString_AsString(result)); - }; - - // !! Not Tested - object unicode() const { - object result = PyObject_Unicode(_obj); - if (!(PyObject*)result) - throw 1; - lose_ref(result); - return result; + return std::string(PyUnicode_AsUTF8(result)); }; //------------------------------------------------------------------------- @@ -726,7 +713,7 @@ class object }; object is_int() const { - return PyInt_Check(_obj) == 1; + return PyLong_Check(_obj) == 1; }; object is_float() const { @@ -750,7 +737,7 @@ class object }; object is_string() const { - return PyString_Check(_obj) == 1; + return PyUnicode_Check(_obj) == 1; }; //------------------------------------------------------------------------- diff --git a/weave/scxx/str.h b/weave/scxx/str.h index 837719a..1861280 100644 --- a/weave/scxx/str.h +++ b/weave/scxx/str.h @@ -18,9 +18,9 @@ class str : public sequence public: str() : sequence() {}; str(const char* s) - : sequence(PyString_FromString((char* )s)) { lose_ref(_obj); } + : sequence(PyUnicode_FromString((char* )s)) { lose_ref(_obj); } str(const char* s, int sz) - : sequence(PyString_FromStringAndSize((char* )s, sz)) { lose_ref(_obj); } + : sequence(PyUnicode_FromStringAndSize((char* )s, sz)) { lose_ref(_obj); } str(const str& other) : sequence(other) {}; str(PyObject* obj) @@ -39,13 +39,13 @@ class str : public sequence return *this; }; virtual void _violentTypeCheck() { - if (!PyString_Check(_obj)) { + if (!PyUnicode_Check(_obj)) { grab_ref(0); fail(PyExc_TypeError, "Not a Python String"); } }; operator const char* () const { - return PyString_AsString(_obj); + return PyUnicode_AsUTF8(_obj); }; /* static str format(const str& fmt, tuple& args){