Skip to content

Commit 7d12d9d

Browse files
author
Stefan Krah
committed
Issue #12834: Fix PyBuffer_ToContiguous() for non-contiguous arrays.
1 parent 6c779ea commit 7d12d9d

8 files changed

Lines changed: 410 additions & 68 deletions

File tree

‎Include/abstract.h‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -535,11 +535,12 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx*/
535535

536536

537537

538+
/* Implementation in memoryobject.c */
538539
PyAPI_FUNC(int) PyBuffer_ToContiguous(void *buf, Py_buffer *view,
539-
Py_ssize_t len, char fort);
540+
Py_ssize_t len, char order);
540541

541542
PyAPI_FUNC(int) PyBuffer_FromContiguous(Py_buffer *view, void *buf,
542-
Py_ssize_t len, char fort);
543+
Py_ssize_t len, char order);
543544

544545

545546
/* Copy len bytes of data from the contiguous chunk of memory

‎Lib/test/test_buffer.py‎

Lines changed: 290 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,11 @@
5353
'f':0, 'd':0, 'P':0
5454
}
5555

56+
# NumPy does not have 'n' or 'N':
57+
if numpy_array:
58+
del NATIVE['n']
59+
del NATIVE['N']
60+
5661
if struct:
5762
try:
5863
# Add "qQ" if present in native mode.
@@ -855,11 +860,49 @@ def verify(self, result, obj=-1,
855860
is_contiguous(result, 'F') and order == 'C':
856861
# The flattened list is already in C-order.
857862
expected = ndarray(flattened, shape=shape, format=ff)
858-
contig = get_contiguous(result, PyBUF_READ, order)
863+
859864
contig = get_contiguous(result, PyBUF_READ, order)
860865
self.assertEqual(contig.tobytes(), b)
861866
self.assertTrue(cmp_contig(contig, expected))
862867

868+
if ndim == 0:
869+
continue
870+
871+
nmemb = len(flattened)
872+
ro = 0 if readonly else ND_WRITABLE
873+
874+
### See comment in test_py_buffer_to_contiguous for an
875+
### explanation why these tests are valid.
876+
877+
# To 'C'
878+
contig = py_buffer_to_contiguous(result, 'C', PyBUF_FULL_RO)
879+
self.assertEqual(len(contig), nmemb * itemsize)
880+
initlst = [struct.unpack_from(fmt, contig, n*itemsize)[0]
881+
for n in range(nmemb)]
882+
883+
y = ndarray(initlst, shape=shape, flags=ro, format=fmt)
884+
self.assertEqual(memoryview(y), memoryview(result))
885+
886+
# To 'F'
887+
contig = py_buffer_to_contiguous(result, 'F', PyBUF_FULL_RO)
888+
self.assertEqual(len(contig), nmemb * itemsize)
889+
initlst = [struct.unpack_from(fmt, contig, n*itemsize)[0]
890+
for n in range(nmemb)]
891+
892+
y = ndarray(initlst, shape=shape, flags=ro|ND_FORTRAN,
893+
format=fmt)
894+
self.assertEqual(memoryview(y), memoryview(result))
895+
896+
# To 'A'
897+
contig = py_buffer_to_contiguous(result, 'A', PyBUF_FULL_RO)
898+
self.assertEqual(len(contig), nmemb * itemsize)
899+
initlst = [struct.unpack_from(fmt, contig, n*itemsize)[0]
900+
for n in range(nmemb)]
901+
902+
f = ND_FORTRAN if is_contiguous(result, 'F') else 0
903+
y = ndarray(initlst, shape=shape, flags=f|ro, format=fmt)
904+
self.assertEqual(memoryview(y), memoryview(result))
905+
863906
if is_memoryview_format(fmt):
864907
try:
865908
m = memoryview(result)
@@ -1805,6 +1848,9 @@ def test_ndarray_random(self):
18051848
self.assertEqual(mvlist, ylist)
18061849

18071850
if numpy_array:
1851+
# XXX NumPy (as far as it compiles with 3.3) currently
1852+
# segfaults here. Wait for a stable 3.3 compatible version.
1853+
continue
18081854
shape = t[3]
18091855
if 0 in shape:
18101856
continue # http://projects.scipy.org/numpy/ticket/1910
@@ -1884,6 +1930,9 @@ def test_ndarray_random_slice_assign(self):
18841930
self.assertEqual(mr.tolist(), yrlist)
18851931

18861932
if numpy_array:
1933+
# XXX NumPy (as far as it compiles with 3.3) currently
1934+
# segfaults here. Wait for a stable 3.3 compatible version.
1935+
continue
18871936
if 0 in lshape or 0 in rshape:
18881937
continue # http://projects.scipy.org/numpy/ticket/1910
18891938

@@ -2020,6 +2069,246 @@ def test_ndarray_hash(self):
20202069
nd = ndarray(list(range(12)), shape=[2,2,3], format='L')
20212070
self.assertEqual(hash(nd), hash(nd.tobytes()))
20222071

2072+
def test_py_buffer_to_contiguous(self):
2073+
2074+
# The requests are used in _testbuffer.c:py_buffer_to_contiguous
2075+
# to generate buffers without full information for testing.
2076+
requests = (
2077+
# distinct flags
2078+
PyBUF_INDIRECT, PyBUF_STRIDES, PyBUF_ND, PyBUF_SIMPLE,
2079+
# compound requests
2080+
PyBUF_FULL, PyBUF_FULL_RO,
2081+
PyBUF_RECORDS, PyBUF_RECORDS_RO,
2082+
PyBUF_STRIDED, PyBUF_STRIDED_RO,
2083+
PyBUF_CONTIG, PyBUF_CONTIG_RO,
2084+
)
2085+
2086+
# no buffer interface
2087+
self.assertRaises(TypeError, py_buffer_to_contiguous, {}, 'F',
2088+
PyBUF_FULL_RO)
2089+
2090+
# scalar, read-only request
2091+
nd = ndarray(9, shape=(), format="L", flags=ND_WRITABLE)
2092+
for order in ['C', 'F', 'A']:
2093+
for request in requests:
2094+
b = py_buffer_to_contiguous(nd, order, request)
2095+
self.assertEqual(b, nd.tobytes())
2096+
2097+
# zeros in shape
2098+
nd = ndarray([1], shape=[0], format="L", flags=ND_WRITABLE)
2099+
for order in ['C', 'F', 'A']:
2100+
for request in requests:
2101+
b = py_buffer_to_contiguous(nd, order, request)
2102+
self.assertEqual(b, b'')
2103+
2104+
nd = ndarray(list(range(8)), shape=[2, 0, 7], format="L",
2105+
flags=ND_WRITABLE)
2106+
for order in ['C', 'F', 'A']:
2107+
for request in requests:
2108+
b = py_buffer_to_contiguous(nd, order, request)
2109+
self.assertEqual(b, b'')
2110+
2111+
### One-dimensional arrays are trivial, since Fortran and C order
2112+
### are the same.
2113+
2114+
# one-dimensional
2115+
for f in [0, ND_FORTRAN]:
2116+
nd = ndarray([1], shape=[1], format="h", flags=f|ND_WRITABLE)
2117+
ndbytes = nd.tobytes()
2118+
for order in ['C', 'F', 'A']:
2119+
for request in requests:
2120+
b = py_buffer_to_contiguous(nd, order, request)
2121+
self.assertEqual(b, ndbytes)
2122+
2123+
nd = ndarray([1, 2, 3], shape=[3], format="b", flags=f|ND_WRITABLE)
2124+
ndbytes = nd.tobytes()
2125+
for order in ['C', 'F', 'A']:
2126+
for request in requests:
2127+
b = py_buffer_to_contiguous(nd, order, request)
2128+
self.assertEqual(b, ndbytes)
2129+
2130+
# one-dimensional, non-contiguous input
2131+
nd = ndarray([1, 2, 3], shape=[2], strides=[2], flags=ND_WRITABLE)
2132+
ndbytes = nd.tobytes()
2133+
for order in ['C', 'F', 'A']:
2134+
for request in [PyBUF_STRIDES, PyBUF_FULL]:
2135+
b = py_buffer_to_contiguous(nd, order, request)
2136+
self.assertEqual(b, ndbytes)
2137+
2138+
nd = nd[::-1]
2139+
ndbytes = nd.tobytes()
2140+
for order in ['C', 'F', 'A']:
2141+
for request in requests:
2142+
try:
2143+
b = py_buffer_to_contiguous(nd, order, request)
2144+
except BufferError:
2145+
continue
2146+
self.assertEqual(b, ndbytes)
2147+
2148+
###
2149+
### Multi-dimensional arrays:
2150+
###
2151+
### The goal here is to preserve the logical representation of the
2152+
### input array but change the physical representation if necessary.
2153+
###
2154+
### _testbuffer example:
2155+
### ====================
2156+
###
2157+
### C input array:
2158+
### --------------
2159+
### >>> nd = ndarray(list(range(12)), shape=[3, 4])
2160+
### >>> nd.tolist()
2161+
### [[0, 1, 2, 3],
2162+
### [4, 5, 6, 7],
2163+
### [8, 9, 10, 11]]
2164+
###
2165+
### Fortran output:
2166+
### ---------------
2167+
### >>> py_buffer_to_contiguous(nd, 'F', PyBUF_FULL_RO)
2168+
### >>> b'\x00\x04\x08\x01\x05\t\x02\x06\n\x03\x07\x0b'
2169+
###
2170+
### The return value corresponds to this input list for
2171+
### _testbuffer's ndarray:
2172+
### >>> nd = ndarray([0,4,8,1,5,9,2,6,10,3,7,11], shape=[3,4],
2173+
### flags=ND_FORTRAN)
2174+
### >>> nd.tolist()
2175+
### [[0, 1, 2, 3],
2176+
### [4, 5, 6, 7],
2177+
### [8, 9, 10, 11]]
2178+
###
2179+
### The logical array is the same, but the values in memory are now
2180+
### in Fortran order.
2181+
###
2182+
### NumPy example:
2183+
### ==============
2184+
### _testbuffer's ndarray takes lists to initialize the memory.
2185+
### Here's the same sequence in NumPy:
2186+
###
2187+
### C input:
2188+
### --------
2189+
### >>> nd = ndarray(buffer=bytearray(list(range(12))),
2190+
### shape=[3, 4], dtype='B')
2191+
### >>> nd
2192+
### array([[ 0, 1, 2, 3],
2193+
### [ 4, 5, 6, 7],
2194+
### [ 8, 9, 10, 11]], dtype=uint8)
2195+
###
2196+
### Fortran output:
2197+
### ---------------
2198+
### >>> fortran_buf = nd.tostring(order='F')
2199+
### >>> fortran_buf
2200+
### b'\x00\x04\x08\x01\x05\t\x02\x06\n\x03\x07\x0b'
2201+
###
2202+
### >>> nd = ndarray(buffer=fortran_buf, shape=[3, 4],
2203+
### dtype='B', order='F')
2204+
###
2205+
### >>> nd
2206+
### array([[ 0, 1, 2, 3],
2207+
### [ 4, 5, 6, 7],
2208+
### [ 8, 9, 10, 11]], dtype=uint8)
2209+
###
2210+
2211+
# multi-dimensional, contiguous input
2212+
lst = list(range(12))
2213+
for f in [0, ND_FORTRAN]:
2214+
nd = ndarray(lst, shape=[3, 4], flags=f|ND_WRITABLE)
2215+
if numpy_array:
2216+
na = numpy_array(buffer=bytearray(lst),
2217+
shape=[3, 4], dtype='B',
2218+
order='C' if f == 0 else 'F')
2219+
2220+
# 'C' request
2221+
if f == ND_FORTRAN: # 'F' to 'C'
2222+
x = ndarray(transpose(lst, [4, 3]), shape=[3, 4],
2223+
flags=ND_WRITABLE)
2224+
expected = x.tobytes()
2225+
else:
2226+
expected = nd.tobytes()
2227+
for request in requests:
2228+
try:
2229+
b = py_buffer_to_contiguous(nd, 'C', request)
2230+
except BufferError:
2231+
continue
2232+
2233+
self.assertEqual(b, expected)
2234+
2235+
# Check that output can be used as the basis for constructing
2236+
# a C array that is logically identical to the input array.
2237+
y = ndarray([v for v in b], shape=[3, 4], flags=ND_WRITABLE)
2238+
self.assertEqual(memoryview(y), memoryview(nd))
2239+
2240+
if numpy_array:
2241+
self.assertEqual(b, na.tostring(order='C'))
2242+
2243+
# 'F' request
2244+
if f == 0: # 'C' to 'F'
2245+
x = ndarray(transpose(lst, [3, 4]), shape=[4, 3],
2246+
flags=ND_WRITABLE)
2247+
else:
2248+
x = ndarray(lst, shape=[3, 4], flags=ND_WRITABLE)
2249+
expected = x.tobytes()
2250+
for request in [PyBUF_FULL, PyBUF_FULL_RO, PyBUF_INDIRECT,
2251+
PyBUF_STRIDES, PyBUF_ND]:
2252+
try:
2253+
b = py_buffer_to_contiguous(nd, 'F', request)
2254+
except BufferError:
2255+
continue
2256+
self.assertEqual(b, expected)
2257+
2258+
# Check that output can be used as the basis for constructing
2259+
# a Fortran array that is logically identical to the input array.
2260+
y = ndarray([v for v in b], shape=[3, 4], flags=ND_FORTRAN|ND_WRITABLE)
2261+
self.assertEqual(memoryview(y), memoryview(nd))
2262+
2263+
if numpy_array:
2264+
self.assertEqual(b, na.tostring(order='F'))
2265+
2266+
# 'A' request
2267+
if f == ND_FORTRAN:
2268+
x = ndarray(lst, shape=[3, 4], flags=ND_WRITABLE)
2269+
expected = x.tobytes()
2270+
else:
2271+
expected = nd.tobytes()
2272+
for request in [PyBUF_FULL, PyBUF_FULL_RO, PyBUF_INDIRECT,
2273+
PyBUF_STRIDES, PyBUF_ND]:
2274+
try:
2275+
b = py_buffer_to_contiguous(nd, 'A', request)
2276+
except BufferError:
2277+
continue
2278+
2279+
self.assertEqual(b, expected)
2280+
2281+
# Check that output can be used as the basis for constructing
2282+
# an array with order=f that is logically identical to the input
2283+
# array.
2284+
y = ndarray([v for v in b], shape=[3, 4], flags=f|ND_WRITABLE)
2285+
self.assertEqual(memoryview(y), memoryview(nd))
2286+
2287+
if numpy_array:
2288+
self.assertEqual(b, na.tostring(order='A'))
2289+
2290+
# multi-dimensional, non-contiguous input
2291+
nd = ndarray(list(range(12)), shape=[3, 4], flags=ND_WRITABLE|ND_PIL)
2292+
2293+
# 'C'
2294+
b = py_buffer_to_contiguous(nd, 'C', PyBUF_FULL_RO)
2295+
self.assertEqual(b, nd.tobytes())
2296+
y = ndarray([v for v in b], shape=[3, 4], flags=ND_WRITABLE)
2297+
self.assertEqual(memoryview(y), memoryview(nd))
2298+
2299+
# 'F'
2300+
b = py_buffer_to_contiguous(nd, 'F', PyBUF_FULL_RO)
2301+
x = ndarray(transpose(lst, [3, 4]), shape=[4, 3], flags=ND_WRITABLE)
2302+
self.assertEqual(b, x.tobytes())
2303+
y = ndarray([v for v in b], shape=[3, 4], flags=ND_FORTRAN|ND_WRITABLE)
2304+
self.assertEqual(memoryview(y), memoryview(nd))
2305+
2306+
# 'A'
2307+
b = py_buffer_to_contiguous(nd, 'A', PyBUF_FULL_RO)
2308+
self.assertEqual(b, nd.tobytes())
2309+
y = ndarray([v for v in b], shape=[3, 4], flags=ND_WRITABLE)
2310+
self.assertEqual(memoryview(y), memoryview(nd))
2311+
20232312
def test_memoryview_construction(self):
20242313

20252314
items_shape = [(9, []), ([1,2,3], [3]), (list(range(2*3*5)), [2,3,5])]

‎Misc/NEWS‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ What's New in Python 3.3.0 Beta 2?
1010
Core and Builtins
1111
-----------------
1212

13+
- Issue #12834: Fix PyBuffer_ToContiguous() for non-contiguous arrays.
14+
1315
- Issue #15456: Fix code __sizeof__ after #12399 change.
1416
Patch by Serhiy Storchaka.
1517

0 commit comments

Comments
 (0)