Merge pull request #767 from rudi-c/ctypes

Import the ctypes module.

Author: kmod

from_cpython/CMakeLists.txt

@@ -118,6 +118,7 @@ add_custom_command(OUTPUT
                        ${CMAKE_BINARY_DIR}/lib_pyston/pyexpat.pyston.so
                        ${CMAKE_BINARY_DIR}/lib_pyston/_elementtree.pyston.so
                        ${CMAKE_BINARY_DIR}/lib_pyston/bz2.pyston.so
+                       ${CMAKE_BINARY_DIR}/lib_pyston/_ctypes.pyston.so
                        ${CMAKE_BINARY_DIR}/lib_pyston/grp.pyston.so
                        ${CMAKE_BINARY_DIR}/lib_pyston/termios.pyston.so
                        ${CMAKE_BINARY_DIR}/lib_pyston/_curses.pyston.so
@@ -129,6 +130,11 @@ add_custom_command(OUTPUT
                        Modules/_multiprocessing/multiprocessing.c
                        Modules/_multiprocessing/semaphore.c
                        Modules/_multiprocessing/socket_connection.c
+                       Modules/_ctypes/_ctypes.c
+                       Modules/_ctypes/callbacks.c
+                       Modules/_ctypes/callproc.c
+                       Modules/_ctypes/stgdict.c
+                       Modules/_ctypes/cfield.c
                        Modules/expat/xmlparse.c
                        Modules/expat/xmlrole.c
                        Modules/expat/xmltok.c

from_cpython/Include/compile.h

@@ -0,0 +1,40 @@
+
+#ifndef Py_COMPILE_H
+#define Py_COMPILE_H
+
+#include "code.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Public interface */
+struct _node; /* Declare the existence of this type */
+PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *);
+
+/* Future feature support */
+
+typedef struct {
+    int ff_features;      /* flags set by future statements */
+    int ff_lineno;        /* line number of last future statement */
+} PyFutureFeatures;
+
+#define FUTURE_NESTED_SCOPES "nested_scopes"
+#define FUTURE_GENERATORS "generators"
+#define FUTURE_DIVISION "division"
+#define FUTURE_ABSOLUTE_IMPORT "absolute_import"
+#define FUTURE_WITH_STATEMENT "with_statement"
+#define FUTURE_PRINT_FUNCTION "print_function"
+#define FUTURE_UNICODE_LITERALS "unicode_literals"
+
+
+struct _mod; /* Declare the existence of this type */
+PyAPI_FUNC(PyCodeObject *) PyAST_Compile(struct _mod *, const char *,
+					PyCompilerFlags *, PyArena *);
+PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_COMPILE_H */

from_cpython/Include/pyconfig.h.in

@@ -20,6 +20,7 @@
 #define _GNU_SOURCE 1
 
 #define PY_LONG_LONG long long
+#define SIZEOF__BOOL 1
 #define SIZEOF_VOID_P 8
 #define SIZEOF_SIZE_T 8
 #define SIZEOF_INT 4

from_cpython/Modules/_ctypes/_ctypes.c

@@ -2044,13 +2044,21 @@ PyCSimpleType_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
             break;
         }
 
-        if (ml) {
+        // TODO: Pyston change:
+        // For now, don't run this code path because we don't support some of
+        // the descriptor CAPI functions.
+        // We do this to be able to run `import ctypes`, but this will need to be
+        // enabled again once we want CType to actually work.
+        // if (ml) {
+        if (false) {
 #if (PYTHON_API_VERSION >= 1012)
             PyObject *meth;
             int x;
+            /*
             meth = PyDescr_NewClassMethod(result, ml);
             if (!meth)
                 return NULL;
+            */
 #else
 #error
             PyObject *meth, *func;

from_cpython/Modules/_ctypes/callbacks.c

@@ -4,7 +4,10 @@
 
 #include "Python.h"
 #include "compile.h" /* required only for 2.3, as it seems */
-#include "frameobject.h"
+
+// Pyston change: We don't have this file and commented out the function that needs it, but
+// we may want to support that function in the future.
+//#include "frameobject.h"
 
 #include <ffi.h>
 #ifdef MS_WIN32
@@ -149,6 +152,9 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
 /* after code that pyrex generates */
 void _ctypes_add_traceback(char *funcname, char *filename, int lineno)
 {
+    // TODO: Pyston change:
+    // Supporting this will require frameobject.h
+#if 0
     PyObject *py_globals = 0;
     PyCodeObject *py_code = 0;
     PyFrameObject *py_frame = 0;
@@ -170,6 +176,9 @@ void _ctypes_add_traceback(char *funcname, char *filename, int lineno)
     Py_XDECREF(py_globals);
     Py_XDECREF(py_code);
     Py_XDECREF(py_frame);
+#else
+    assert(false);
+#endif
 }
 
 #ifdef MS_WIN32

from_cpython/Modules/_ctypes/cfield.c

@@ -1291,7 +1291,9 @@ s_get(void *ptr, Py_ssize_t size)
      */
     slen = strlen(PyString_AS_STRING(result));
     size = min(size, (Py_ssize_t)slen);
-    if (result->ob_refcnt == 1) {
+
+    // Pyston change: no ob_refcnt
+    if (false /*result->ob_refcnt == 1*/) {
         /* shorten the result */
         _PyString_Resize(&result, size);
         return result;

from_cpython/setup.py

@@ -2,38 +2,81 @@
 
 from distutils.core import setup, Extension
 import os
+import sysconfig
 
 def relpath(fn):
     r =  os.path.join(os.path.dirname(__file__), fn)
     return r
 
+def unique(f):
+    # Use an array otherwise Python 2 gets confused about scoping.
+    cache = []
+    def wrapper(*args):
+        if len(cache) == 0:
+            cache.append(f(*args))
+        return cache[0]
+    return wrapper
+
+@unique
 def multiprocessing_ext():
     return Extension("_multiprocessing", sources = map(relpath, [
             "Modules/_multiprocessing/multiprocessing.c",
             "Modules/_multiprocessing/socket_connection.c",
             "Modules/_multiprocessing/semaphore.c",
             ]))
 
+@unique
 def bz2_ext():
     return Extension("bz2", sources = map(relpath, [
             "Modules/bz2module.c",
             ]), libraries = ['bz2'])
 
+@unique
+def ctypes_ext():
+    ext = Extension("_ctypes", sources = map(relpath, [
+            "Modules/_ctypes/_ctypes.c",
+            "Modules/_ctypes/callbacks.c",
+            "Modules/_ctypes/callproc.c",
+            "Modules/_ctypes/stgdict.c",
+            "Modules/_ctypes/cfield.c"
+            ]))
+
+    # Hack: Just copy the values of ffi_inc and ffi_lib from CPython's setup.py
+    # May want something more robust later.
+    ffi_inc = ['/usr/include/x86_64-linux-gnu']
+    ffi_lib = "ffi_pic"
+    ext.include_dirs.extend(ffi_inc)
+    ext.libraries.append(ffi_lib)
+
+    return ext
+
+@unique
+def ctypes_test_ext():
+    # TODO: I'm not sure how to use the ctypes tests, I just copied it over
+    # from CPython's setup.py. For now we're only importing ctypes, not passing
+    # all its tests.
+    return Extension('_ctypes_test',
+                     sources= map(relpath, ['Modules/_ctypes/_ctypes_test.c']))
+
+@unique
 def grp_ext():
     return Extension("grp", sources = map(relpath, [
             "Modules/grpmodule.c",
             ]))
 
+@unique
 def curses_ext():
     return Extension("_curses", sources = map(relpath, [
             "Modules/_cursesmodule.c",
             ]), libraries = ['curses'])
 
+@unique
 def termios_ext():
     return Extension("termios", sources = map(relpath, [
             "Modules/termios.c",
             ]))
 
+@unique
 def pyexpat_ext():
     define_macros = [('HAVE_EXPAT_CONFIG_H', '1'),]
     expat_sources = map(relpath, ['Modules/expat/xmlparse.c',
@@ -60,6 +103,7 @@ def pyexpat_ext():
                       depends = expat_depends,
                     )
 
+@unique
 def elementtree_ext():
     # elementtree depends on expat
     pyexpat = pyexpat_ext()
@@ -71,10 +115,15 @@ def elementtree_ext():
                         sources = [relpath('Modules/_elementtree.c')],
                         depends = pyexpat.depends,
                       )
+ext_modules = [multiprocessing_ext(),
+               pyexpat_ext(),
+               elementtree_ext(),
+               bz2_ext(),
+               ctypes_ext(),
+               ctypes_test_ext(),
+               grp_ext(),
+               curses_ext(),
+               termios_ext()]
 
 
-setup(name="Pyston",
-        version="1.0",
-        description="Pyston shared modules",
-        ext_modules=[multiprocessing_ext(), pyexpat_ext(), elementtree_ext(), bz2_ext(), grp_ext(), curses_ext(), termios_ext()]
-    )
+setup(name="Pyston", version="1.0", description="Pyston shared modules", ext_modules=ext_modules)

src/capi/typeobject.cpp

@@ -3235,11 +3235,7 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
                        | Py_TPFLAGS_TUPLE_SUBCLASS | Py_TPFLAGS_STRING_SUBCLASS | Py_TPFLAGS_UNICODE_SUBCLASS
                        | Py_TPFLAGS_DICT_SUBCLASS | Py_TPFLAGS_BASE_EXC_SUBCLASS | Py_TPFLAGS_TYPE_SUBCLASS;
     RELEASE_ASSERT((cls->tp_flags & ~ALLOWABLE_FLAGS) == 0, "");
-    if (cls->tp_as_number) {
-        RELEASE_ASSERT(cls->tp_flags & Py_TPFLAGS_CHECKTYPES, "Pyston doesn't yet support non-checktypes behavior");
-    }
 
-    RELEASE_ASSERT(cls->tp_descr_set == NULL, "");
     RELEASE_ASSERT(cls->tp_free == NULL || cls->tp_free == PyObject_Del || cls->tp_free == PyObject_GC_Del, "");
     RELEASE_ASSERT(cls->tp_is_gc == NULL, "");
     RELEASE_ASSERT(cls->tp_mro == NULL, "");
@@ -3317,8 +3313,24 @@ extern "C" int PyType_Ready(PyTypeObject* cls) noexcept {
     cls->gc_visit = &conservativeGCHandler;
     cls->is_user_defined = true;
 
-    // this should get automatically initialized to 0 on this path:
-    assert(cls->attrs_offset == 0);
+
+    if (!cls->instancesHaveHCAttrs() && cls->tp_base) {
+        // These doesn't get copied in inherit_slots like other slots do.
+        if (cls->tp_base->instancesHaveHCAttrs()) {
+            cls->attrs_offset = cls->tp_base->attrs_offset;
+        }
+
+        // Example of when this code path could be reached and needs to be:
+        // If this class is a metaclass defined in a C extension, chances are that some of its
+        // instances may be hardcoded in the C extension as well. Those instances will call
+        // PyType_Ready and expect their class (this metaclass) to have a place to put attributes.
+        // e.g. CTypes does this.
+        bool is_metaclass = PyType_IsSubtype(cls, type_cls);
+        assert(!is_metaclass || cls->instancesHaveHCAttrs() || cls->instancesHaveDictAttrs());
+    } else {
+        // this should get automatically initialized to 0 on this path:
+        assert(cls->attrs_offset == 0);
+    }
 
     if (Py_TPFLAGS_BASE_EXC_SUBCLASS & cls->tp_flags) {
         exception_types.push_back(cls);

src/runtime/int.cpp

@@ -184,6 +184,16 @@ extern "C" int _PyInt_AsInt(PyObject* obj) noexcept {
     return (int)result;
 }
 
+#ifdef HAVE_LONG_LONG
+extern "C" unsigned PY_LONG_LONG PyInt_AsUnsignedLongLongMask(register PyObject* op) noexcept {
+    Py_FatalError("unimplemented");
+
+    unsigned PY_LONG_LONG val = 0;
+
+    return val;
+}
+#endif
+
 extern "C" PyObject* PyInt_FromString(const char* s, char** pend, int base) noexcept {
     char* end;
     long x;