Merged in py3k-buffer branch to main line. All objects now use the buffer protocol in PEP 3118.

Author: teoliphant

Include/Python.h

@@ -77,6 +77,7 @@
 #include "rangeobject.h"
 #include "stringobject.h"
 #include "bufferobject.h"
+#include "memoryobject.h"
 #include "tupleobject.h"
 #include "listobject.h"
 #include "dictobject.h"

Include/abstract.h

@@ -475,6 +475,12 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
 	 This is the equivalent of the Python statement: del o[key].
        */
 
+	/* old buffer API
+	   FIXME:  usage of these should all be replaced in Python itself
+	   but for backwards compatibility we will implement them. 
+	   Their usage without a corresponding "unlock" mechansim
+	   may create issues (but they would already be there). */
+
      PyAPI_FUNC(int) PyObject_AsCharBuffer(PyObject *obj,
 					   const char **buffer,
 					   Py_ssize_t *buffer_len);
@@ -527,6 +533,110 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
 	  an exception set.
        */
 
+	/* new buffer API */
+
+#define PyObject_CheckBuffer(obj) \
+        (((obj)->ob_type->tp_as_buffer != NULL) &&  \
+         ((obj)->ob_type->tp_as_buffer->bf_getbuffer != NULL))
+
+	/* Return 1 if the getbuffer function is available, otherwise 
+	   return 0 */
+
+     PyAPI_FUNC(int) PyObject_GetBuffer(PyObject *obj, PyBuffer *view, 
+					int flags);
+
+	/* This is a C-API version of the getbuffer function call.  It checks
+       	   to make sure object has the required function pointer and issues the
+	   call.  Returns -1 and raises an error on failure and returns 0 on 
+	   success
+        */
+
+
+     PyAPI_FUNC(void) PyObject_ReleaseBuffer(PyObject *obj, PyBuffer *view);
+
+
+	/* C-API version of the releasebuffer function call.  It
+	   checks to make sure the object has the required function
+	   pointer and issues the call.  The obj must have the buffer
+	   interface or this function will cause a segfault (i.e. it
+	   is assumed to be called only after a corresponding
+	   getbuffer which already verified the existence of the
+	   tp_as_buffer pointer).
+           
+           Returns 0 on success and -1 (with an error raised) on
+           failure.  This function always succeeds (as a NO-OP) if
+           there is no releasebuffer function for the object so that
+           it can always be called when the consumer is done with the
+           buffer
+        */
+
+     PyAPI_FUNC(void *) PyBuffer_GetPointer(PyBuffer *view, Py_ssize_t *indices);
+        
+        /* Get the memory area pointed to by the indices for the buffer given. 
+           Note that view->ndim is the assumed size of indices 
+        */
+
+     PyAPI_FUNC(int) PyBuffer_SizeFromFormat(const char *);
+		
+	/* Return the implied itemsize of the data-format area from a 
+	   struct-style description */
+    
+
+	
+     PyAPI_FUNC(int) PyBuffer_ToContiguous(void *buf, PyBuffer *view,
+    					   Py_ssize_t len, char fort);
+
+     PyAPI_FUNC(int) PyBuffer_FromContiguous(PyBuffer *view, void *buf, 
+    					     Py_ssize_t len, char fort);
+
+
+	/* Copy len bytes of data from the contiguous chunk of memory
+	   pointed to by buf into the buffer exported by obj.  Return
+	   0 on success and return -1 and raise a PyBuffer_Error on
+	   error (i.e. the object does not have a buffer interface or
+	   it is not working).
+
+	   If fortran is 'F', then if the object is multi-dimensional,
+	   then the data will be copied into the array in
+	   Fortran-style (first dimension varies the fastest).  If
+	   fortran is 'C', then the data will be copied into the array
+	   in C-style (last dimension varies the fastest).  If fortran
+	   is 'A', then it does not matter and the copy will be made
+	   in whatever way is more efficient.
+
+        */
+
+     PyAPI_FUNC(int) PyObject_CopyData(PyObject *dest, PyObject *src);
+        
+        /* Copy the data from the src buffer to the buffer of destination
+         */
+
+     PyAPI_FUNC(int) PyBuffer_IsContiguous(PyBuffer *view, char fortran);
+
+
+     PyAPI_FUNC(void) PyBuffer_FillContiguousStrides(int ndims, 
+	  					    Py_ssize_t *shape, 
+						    Py_ssize_t *strides,
+	                                            int itemsize,
+	     					    char fort);
+
+       	/*  Fill the strides array with byte-strides of a contiguous
+            (Fortran-style if fortran is 'F' or C-style otherwise)
+            array of the given shape with the given number of bytes
+            per element.
+        */
+
+     PyAPI_FUNC(int) PyBuffer_FillInfo(PyBuffer *view, void *buf,
+		             	       Py_ssize_t len, int readonly,
+				       int flags);
+
+        /* Fills in a buffer-info structure correctly for an exporter
+           that can only share a contiguous chunk of memory of
+           "unsigned bytes" of the given length. Returns 0 on success
+           and -1 (with raising an error) on error.
+         */
+
+
 /* Iterators */
 
      PyAPI_FUNC(PyObject *) PyObject_GetIter(PyObject *);

Include/bytesobject.h

@@ -21,6 +21,7 @@ extern "C" {
 /* Object layout */
 typedef struct {
     PyObject_VAR_HEAD
+    int ob_exports; /* how many buffer exports */
     Py_ssize_t ob_alloc; /* How many bytes allocated */
     char *ob_bytes;
 } PyBytesObject;

Include/memoryobject.h

@@ -0,0 +1,67 @@
+
+/* Memory object interface */
+
+#ifndef Py_MEMORYOBJECT_H
+#define Py_MEMORYOBJECT_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+        PyObject_HEAD
+        PyObject *base;
+        PyBuffer view;
+} PyMemoryViewObject;
+
+
+PyAPI_DATA(PyTypeObject) PyMemoryView_Type;
+
+#define PyMemory_Check(op) (Py_Type(op) == &PyMemoryView_Type)
+#define PyMemoryView(op) (((PyMemoryViewObject *)(op))->view)
+
+#define Py_END_OF_MEMORY	(-1)
+
+PyAPI_FUNC(PyObject *) PyMemoryView_GetContiguous(PyObject *base, int buffertype, 
+                                                  char fort);
+
+	/* Return a contiguous chunk of memory representing the buffer
+	   from an object in a memory view object.  If a copy is made then the
+           base object for the memory view will be a *new* bytes object. 
+           
+           Otherwise, the base-object will be the object itself and no 
+           data-copying will be done. 
+
+           The buffertype argument can be PyBUF_READ, PyBUF_WRITE,
+           PyBUF_UPDATEIFCOPY to determine whether the returned buffer
+           should be READONLY, WRITEABLE, or set to update the
+           original buffer if a copy must be made.  If buffertype is
+           PyBUF_WRITE and the buffer is not contiguous an error will
+           be raised.  In this circumstance, the user can use
+           PyBUF_UPDATEIFCOPY to ensure that a a writeable temporary
+           contiguous buffer is returned.  The contents of this
+           contiguous buffer will be copied back into the original
+           object after the memoryview object is deleted as long as
+           the original object is writeable and allows setting its
+           memory to "readonly".  If this is not allowed by the
+           original object, then a BufferError is raised.
+	   
+           If the object is multi-dimensional and if fortran is 'F',
+           the first dimension of the underlying array will vary the
+           fastest in the buffer.  If fortran is 'C', then the last
+           dimension will vary the fastest (C-style contiguous).  If
+           fortran is 'A', then it does not matter and you will get
+           whatever the object decides is more efficient.  
+
+           A new reference is returned that must be DECREF'd when finished.
+        */
+
+PyAPI_FUNC(PyObject *) PyMemoryView_FromObject(PyObject *base);
+
+PyAPI_FUNC(PyObject *) PyMemoryView_FromMemory(PyBuffer *info);
+	/* create new if bufptr is NULL 
+	    will be a new bytesobject in base */
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_MEMORYOBJECT_H */

Include/object.h

@@ -140,11 +140,59 @@ typedef int(*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *);
 typedef int(*ssizessizeobjargproc)(PyObject *, Py_ssize_t, Py_ssize_t, PyObject *);
 typedef int(*objobjargproc)(PyObject *, PyObject *, PyObject *);
 
-/* ssize_t-based buffer interface */
-typedef Py_ssize_t (*readbufferproc)(PyObject *, Py_ssize_t, void **);
-typedef Py_ssize_t (*writebufferproc)(PyObject *, Py_ssize_t, void **);
-typedef Py_ssize_t (*segcountproc)(PyObject *, Py_ssize_t *);
-typedef Py_ssize_t (*charbufferproc)(PyObject *, Py_ssize_t, char **);
+
+/* buffer interface */
+typedef struct bufferinfo {
+	void *buf;         
+        Py_ssize_t len;
+        Py_ssize_t itemsize;  
+        int readonly;
+        int ndim; 
+        char *format;
+        Py_ssize_t *shape;
+        Py_ssize_t *strides;
+        Py_ssize_t *suboffsets;
+        void *internal;
+} PyBuffer;
+
+typedef int (*getbufferproc)(PyObject *, PyBuffer *, int);
+typedef void (*releasebufferproc)(PyObject *, PyBuffer *);
+
+        /* Flags for getting buffers */
+#define PyBUF_SIMPLE 0
+#define PyBUF_CHARACTER 1
+#define PyBUF_WRITEABLE 0x0002
+#define PyBUF_LOCKDATA 0x0004
+#define PyBUF_FORMAT 0x0008
+#define PyBUF_ND 0x0010
+#define PyBUF_STRIDES (0x0020 | PyBUF_ND)
+#define PyBUF_C_CONTIGUOUS (0x0040 | PyBUF_STRIDES)
+#define PyBUF_F_CONTIGUOUS (0x0080 | PyBUF_STRIDES)
+#define PyBUF_ANY_CONTIGUOUS (0x0100 | PyBUF_STRIDES)
+#define PyBUF_INDIRECT (0x0200 | PyBUF_STRIDES)
+
+#define PyBUF_CONTIG (PyBUF_ND | PyBUF_WRITEABLE)
+#define PyBUF_CONTIG_RO (PyBUF_ND)
+#define PyBUF_CONTIG_LCK (PyBUF_ND | PyBUF_LOCKDATA)
+
+#define PyBUF_STRIDED (PyBUF_STRIDES | PyBUF_WRITEABLE)
+#define PyBUF_STRIDED_RO (PyBUF_STRIDES)
+#define PyBUF_STRIDED_LCK (PyBUF_STRIDES | PyBUF_LOCKDATA)
+
+#define PyBUF_RECORDS (PyBUF_STRIDES | PyBUF_WRITEABLE | PyBUF_FORMAT)
+#define PyBUF_RECORDS_RO (PyBUF_STRIDES | PyBUF_FORMAT)
+#define PyBUF_RECORDS_LCK (PyBUF_STRIDES | PyBUF_LOCKDATA | PyBUF_FORMAT)
+
+#define PyBUF_FULL (PyBUF_INDIRECT | PyBUF_WRITEABLE | PyBUF_FORMAT)
+#define PyBUF_FULL_RO (PyBUF_INDIRECT | PyBUF_FORMAT)
+#define PyBUF_FULL_LCK (PyBUF_INDIRECT | PyBUF_LOCKDATA | PyBUF_FORMAT)
+
+
+#define PyBUF_READ  0x100
+#define PyBUF_WRITE 0x200
+#define PyBUF_SHADOW 0x400
+
+/* End buffer interface */
 
 typedef int (*objobjproc)(PyObject *, PyObject *);
 typedef int (*visitproc)(PyObject *, void *);
@@ -218,14 +266,13 @@ typedef struct {
 	objobjargproc mp_ass_subscript;
 } PyMappingMethods;
 
+
 typedef struct {
-	readbufferproc bf_getreadbuffer;
-	writebufferproc bf_getwritebuffer;
-	segcountproc bf_getsegcount;
-	charbufferproc bf_getcharbuffer;
+     getbufferproc bf_getbuffer;
+     releasebufferproc bf_releasebuffer;
+     inquiry bf_multisegment;
 } PyBufferProcs;
 
-
 typedef void (*freefunc)(void *);
 typedef void (*destructor)(PyObject *);
 typedef int (*printfunc)(PyObject *, FILE *, int);

Include/pyerrors.h

@@ -138,6 +138,7 @@ PyAPI_DATA(PyObject *) PyExc_UnicodeDecodeError;
 PyAPI_DATA(PyObject *) PyExc_UnicodeTranslateError;
 PyAPI_DATA(PyObject *) PyExc_ValueError;
 PyAPI_DATA(PyObject *) PyExc_ZeroDivisionError;
+PyAPI_DATA(PyObject *) PyExc_BufferError;
 #ifdef MS_WINDOWS
 PyAPI_DATA(PyObject *) PyExc_WindowsError;
 #endif

Makefile.pre.in

@@ -300,6 +300,7 @@ OBJECT_OBJS=	\
 		Objects/listobject.o \
 		Objects/longobject.o \
 		Objects/dictobject.o \
+		Objects/memoryobject.o \
 		Objects/methodobject.o \
 		Objects/moduleobject.o \
 		Objects/object.o \
@@ -533,6 +534,7 @@ PYTHON_HEADERS= \
 		Include/iterobject.h \
 		Include/listobject.h \
 		Include/longobject.h \
+		Include/memoryobject.h \
 		Include/methodobject.h \
 		Include/modsupport.h \
 		Include/moduleobject.h \

Modules/_ctypes/_ctypes.c

@@ -739,22 +739,33 @@ CharArray_set_raw(CDataObject *self, PyObject *value)
 {
 	char *ptr;
 	Py_ssize_t size;
+        int rel = 0;
+        PyBuffer view;
+
 	if (PyBuffer_Check(value)) {
-		size = Py_Type(value)->tp_as_buffer->bf_getreadbuffer(value, 0, (void *)&ptr);
-		if (size < 0)
-			return -1;
+                if (PyObject_GetBuffer(value, &view, PyBUF_SIMPLE) < 0)
+                        return -1;
+                size = view.len;
+                ptr = view.buf;
+                rel = 1;
 	} else if (-1 == PyString_AsStringAndSize(value, &ptr, &size)) {
 		return -1;
 	}
 	if (size > self->b_size) {
 		PyErr_SetString(PyExc_ValueError,
 				"string too long");
-		return -1;
+		goto fail;
 	}
 
 	memcpy(self->b_ptr, ptr, size);
 
+        if (rel)
+                PyObject_ReleaseBuffer(value, &view);
 	return 0;
+ fail:
+        if (rel) 
+                PyObject_ReleaseBuffer(value, &view);
+        return -1;
 }
 
 static PyObject *
@@ -2072,29 +2083,15 @@ static PyMemberDef CData_members[] = {
 	{ NULL },
 };
 
-static Py_ssize_t CData_GetBuffer(PyObject *_self, Py_ssize_t seg, void **pptr)
+static int CData_GetBuffer(PyObject *_self, PyBuffer *view, int flags)
 {
 	CDataObject *self = (CDataObject *)_self;
-	if (seg != 0) {
-		/* Hm. Must this set an exception? */
-		return -1;
-	}
-	*pptr = self->b_ptr;
-	return self->b_size;
-}
-
-static Py_ssize_t CData_GetSegcount(PyObject *_self, Py_ssize_t *lenp)
-{
-	if (lenp)
-		*lenp = 1;
-	return 1;
+        return PyBuffer_FillInfo(view, self->b_ptr, self->b_size, 0, flags);
 }
 
 static PyBufferProcs CData_as_buffer = {
 	CData_GetBuffer,
-	CData_GetBuffer,
-	CData_GetSegcount,
-	NULL,
+        NULL,
 };
 
 /*