Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

gh-92888: Fix memoryview bad __index__ use after free #92946

Merged
merged 12 commits into from
Jun 17, 2022
Merged

gh-92888: Fix memoryview bad __index__ use after free #92946

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
73ffb29
Fix memoryview bad `__index__` use after free
Fidget-Spinner May 19, 2022
3581de7
📜🤖 Added by blurb_it.
blurb-it[bot] May 19, 2022
43c66d6
stylistic nits
Fidget-Spinner May 19, 2022
18b921f
Merge branch 'fix_memoryview_auf' of https://github.com/Fidget-Spinne…
Fidget-Spinner May 20, 2022
2f3c04f
Fix whitespace
Fidget-Spinner May 21, 2022
a250b02
Use more comprehensive tests by Serhiy
Fidget-Spinner May 23, 2022
c818990
Add more tests by Serhiy
Fidget-Spinner May 23, 2022
42a706c
Fix newline
Fidget-Spinner May 23, 2022
b650764
Fix for tuple index
Fidget-Spinner May 23, 2022
47bd91f
Address review
Fidget-Spinner May 23, 2022
6890713
Address Victor's review
Fidget-Spinner May 29, 2022
d3edf74
re-add ba=None
Fidget-Spinner May 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 77 additions & 0 deletions Lib/test/test_memoryview.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -545,6 +545,83 @@ def test_pickle(self):
with self.assertRaises(TypeError):
pickle.dumps(m, proto)

def test_use_released_memory(self):
Fidget-Spinner marked this conversation as resolved.
Show resolved Hide resolved
size = 128
def release():
m.release()
nonlocal ba
ba = bytearray(size)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's useless, no?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, we need it for tests below that tests indexing into ba[].

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We allocate a bytearray of the same size as the bytearray just released in memoryview in hope that it will be allocated at the same memory. It helps to check that we do nor read/write a freed memory.

class MyIndex:
def __index__(self):
release()
return 4
class MyFloat:
def __float__(self):
release()
return 4.25
class MyBool:
def __bool__(self):
release()
return True

ba = None
m = memoryview(bytearray(b'\xff'*size))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In my PR, I tried to make the code more generic to test more cases: https://github.com/python/cpython/pull/93127/files#diff-d41c6bb40a1e03fea5a20d15c4077413e0ddde65651147922b625b03a66a2f16R399:

        tp = self.rw_type
        b = self.rw_type(self._source)
        view = self._view(b)

with self.assertRaises(ValueError):
m[MyIndex()]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This test is very long. Can you try to factorize similar code and use loop with subTest(), and put pack operations in one test method and unpack in another test method?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Then we will need to duplicate the definitions of internal classes.

The tested code is so different, that it is difficult to use a loop. And I think that the result will be more complicated.


ba = None
m = memoryview(bytearray(b'\xff'*size))
self.assertEqual(list(m[:MyIndex()]), [255] * 4)

ba = None
m = memoryview(bytearray(b'\xff'*size))
self.assertEqual(list(m[MyIndex():8]), [255] * 4)

Fidget-Spinner marked this conversation as resolved.
Show resolved Hide resolved
ba = None
m = memoryview(bytearray(b'\xff'*size))
with self.assertRaisesRegex(ValueError, "operation forbidden"):
m[MyIndex()] = 42
self.assertEqual(ba[:8], b'\0'*8)

ba = None
m = memoryview(bytearray(b'\xff'*size))
with self.assertRaisesRegex(ValueError, "operation forbidden"):
Fidget-Spinner marked this conversation as resolved.
Show resolved Hide resolved
m[:MyIndex()] = b'spam'
self.assertEqual(ba[:8], b'\0'*8)

ba = None
m = memoryview(bytearray(b'\xff'*size))
with self.assertRaisesRegex(ValueError, "operation forbidden"):
Fidget-Spinner marked this conversation as resolved.
Show resolved Hide resolved
m[MyIndex():8] = b'spam'
self.assertEqual(ba[:8], b'\0'*8)

Fidget-Spinner marked this conversation as resolved.
Show resolved Hide resolved
ba = None
m = memoryview(bytearray(b'\xff'*size))
with self.assertRaisesRegex(ValueError, "operation forbidden"):
m[0] = MyIndex()
self.assertEqual(ba[:8], b'\0'*8)

for fmt in 'bhilqnBHILQN':
with self.subTest(fmt=fmt):
ba = None
m = memoryview(bytearray(b'\xff'*size)).cast(fmt)
with self.assertRaisesRegex(ValueError, "operation forbidden"):
m[0] = MyIndex()
self.assertEqual(ba[:8], b'\0'*8)

for fmt in 'fd':
with self.subTest(fmt=fmt):
ba = None
m = memoryview(bytearray(b'\xff'*size)).cast(fmt)
with self.assertRaisesRegex(ValueError, "operation forbidden"):
m[0] = MyFloat()
self.assertEqual(ba[:8], b'\0'*8)

ba = None
m = memoryview(bytearray(b'\xff'*size)).cast('?')
with self.assertRaisesRegex(ValueError, "operation forbidden"):
m[0] = MyBool()
self.assertEqual(ba[:8], b'\0'*8)

if __name__ == "__main__":
unittest.main()
1 change: 1 addition & 0 deletions Misc/NEWS.d/next/Security/2022-05-19-08-53-07.gh-issue-92888.TLtR9W.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Fix ``memoryview`` use after free when accessing the backing buffer in certain cases.
23 changes: 17 additions & 6 deletions Objects/memoryobject.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -381,8 +381,9 @@ copy_rec(const Py_ssize_t *shape, Py_ssize_t ndim, Py_ssize_t itemsize,

/* Faster copying of one-dimensional arrays. */
static int
copy_single(const Py_buffer *dest, const Py_buffer *src)
copy_single(PyMemoryViewObject *self, const Py_buffer *dest, const Py_buffer *src)
{
CHECK_RELEASED_INT(self); /* See gh-92888 for why we need this here */
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you move the check after the equiv_structure() test?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Would not access to Py_buffer of the released object cause reading after free?

char *mem = NULL;

assert(dest->ndim == 1);
Expand Down Expand Up @@ -1767,7 +1768,7 @@ unpack_single(const char *ptr, const char *fmt)
/* Pack a single item. 'fmt' can be any native format character in
struct module syntax. */
static int
pack_single(char *ptr, PyObject *item, const char *fmt)
pack_single(PyMemoryViewObject *self, char *ptr, PyObject *item, const char *fmt)
{
unsigned long long llu;
unsigned long lu;
Expand All @@ -1784,6 +1785,7 @@ pack_single(char *ptr, PyObject *item, const char *fmt)
ld = pylong_as_ld(item);
if (ld == -1 && PyErr_Occurred())
goto err_occurred;
CHECK_RELEASED_INT(self); /* See gh-92888 for why we need this here */
switch (fmt[0]) {
case 'b':
if (ld < SCHAR_MIN || ld > SCHAR_MAX) goto err_range;
Expand All @@ -1804,6 +1806,7 @@ pack_single(char *ptr, PyObject *item, const char *fmt)
lu = pylong_as_lu(item);
if (lu == (unsigned long)-1 && PyErr_Occurred())
goto err_occurred;
CHECK_RELEASED_INT(self); /* See gh-92888 for why we need this here */
switch (fmt[0]) {
case 'B':
if (lu > UCHAR_MAX) goto err_range;
Expand All @@ -1824,12 +1827,14 @@ pack_single(char *ptr, PyObject *item, const char *fmt)
lld = pylong_as_lld(item);
if (lld == -1 && PyErr_Occurred())
goto err_occurred;
CHECK_RELEASED_INT(self); /* See gh-92888 for why we need this here */
PACK_SINGLE(ptr, lld, long long);
break;
case 'Q':
llu = pylong_as_llu(item);
if (llu == (unsigned long long)-1 && PyErr_Occurred())
goto err_occurred;
CHECK_RELEASED_INT(self); /* See gh-92888 for why we need this here */
PACK_SINGLE(ptr, llu, unsigned long long);
break;

Expand All @@ -1838,12 +1843,14 @@ pack_single(char *ptr, PyObject *item, const char *fmt)
zd = pylong_as_zd(item);
if (zd == -1 && PyErr_Occurred())
goto err_occurred;
CHECK_RELEASED_INT(self); /* See gh-92888 for why we need this here */
PACK_SINGLE(ptr, zd, Py_ssize_t);
break;
case 'N':
zu = pylong_as_zu(item);
if (zu == (size_t)-1 && PyErr_Occurred())
goto err_occurred;
CHECK_RELEASED_INT(self); /* See gh-92888 for why we need this here */
PACK_SINGLE(ptr, zu, size_t);
break;

Expand All @@ -1852,6 +1859,7 @@ pack_single(char *ptr, PyObject *item, const char *fmt)
d = PyFloat_AsDouble(item);
if (d == -1.0 && PyErr_Occurred())
goto err_occurred;
CHECK_RELEASED_INT(self); /* See gh-92888 for why we need this here */
if (fmt[0] == 'f') {
PACK_SINGLE(ptr, d, float);
}
Expand All @@ -1865,6 +1873,7 @@ pack_single(char *ptr, PyObject *item, const char *fmt)
ld = PyObject_IsTrue(item);
if (ld < 0)
return -1; /* preserve original error */
CHECK_RELEASED_INT(self); /* See gh-92888 for why we need this here */
PACK_SINGLE(ptr, ld, _Bool);
break;

Expand All @@ -1882,6 +1891,7 @@ pack_single(char *ptr, PyObject *item, const char *fmt)
p = PyLong_AsVoidPtr(item);
if (p == NULL && PyErr_Occurred())
goto err_occurred;
CHECK_RELEASED_INT(self); /* See gh-92888 for why we need this here */
PACK_SINGLE(ptr, p, void *);
break;

Expand Down Expand Up @@ -2538,7 +2548,7 @@ memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value)
if (key == Py_Ellipsis ||
(PyTuple_Check(key) && PyTuple_GET_SIZE(key)==0)) {
ptr = (char *)view->buf;
return pack_single(ptr, value, fmt);
return pack_single(self, ptr, value, fmt);
}
else {
PyErr_SetString(PyExc_TypeError,
Expand All @@ -2557,10 +2567,11 @@ memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value)
index = PyNumber_AsSsize_t(key, PyExc_IndexError);
if (index == -1 && PyErr_Occurred())
return -1;
CHECK_RELEASED_INT(self); /* See gh-92888 for why we need this here */
ptr = ptr_from_index(view, index);
if (ptr == NULL)
return -1;
return pack_single(ptr, value, fmt);
return pack_single(self, ptr, value, fmt);
}
/* one-dimensional: fast path */
if (PySlice_Check(key) && view->ndim == 1) {
Expand All @@ -2583,7 +2594,7 @@ memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value)
goto end_block;
dest.len = dest.shape[0] * dest.itemsize;

ret = copy_single(&dest, &src);
ret = copy_single(self, &dest, &src);

end_block:
PyBuffer_Release(&src);
Expand All @@ -2599,7 +2610,7 @@ memory_ass_sub(PyMemoryViewObject *self, PyObject *key, PyObject *value)
ptr = ptr_from_tuple(view, key);
if (ptr == NULL)
return -1;
return pack_single(ptr, value, fmt);
return pack_single(self, ptr, value, fmt);
}
if (PySlice_Check(key) || is_multislice(key)) {
/* Call memory_subscript() to produce a sliced lvalue, then copy
Expand Down