bpo-35436: Add missing PyErr_NoMemory() calls and other minor bug fixes. (GH-11015)

Set MemoryError when appropriate, add missing failure checks,
and fix some potential leaks.

Author: ZackerySpytz

Misc/NEWS.d/next/Core and Builtins/2018-12-07-02-38-01.bpo-35436.0VW7p9.rst

@@ -0,0 +1,2 @@
+Fix various issues with memory allocation error handling.  Patch by Zackery
+Spytz.

Modules/_abc.c

@@ -728,6 +728,10 @@ subclasscheck_check_registry(_abc_data *impl, PyObject *subclass,
     // Weakref callback may remove entry from set.
     // So we take snapshot of registry first.
     PyObject **copy = PyMem_Malloc(sizeof(PyObject*) * registry_size);
+    if (copy == NULL) {
+        PyErr_NoMemory();
+        return -1;
+    }
     PyObject *key;
     Py_ssize_t pos = 0;
     Py_hash_t hash;

Modules/_ctypes/_ctypes.c

@@ -305,8 +305,10 @@ _ctypes_alloc_format_string_for_type(char code, int big_endian)
     }
 
     result = PyMem_Malloc(3);
-    if (result == NULL)
+    if (result == NULL) {
+        PyErr_NoMemory();
         return NULL;
+    }
 
     result[0] = big_endian ? '>' : '<';
     result[1] = pep_code;
@@ -366,8 +368,10 @@ _ctypes_alloc_format_string_with_shape(int ndim, const Py_ssize_t *shape,
     if (prefix)
         prefix_len += strlen(prefix);
     new_prefix = PyMem_Malloc(prefix_len);
-    if (new_prefix == NULL)
+    if (new_prefix == NULL) {
+        PyErr_NoMemory();
         return NULL;
+    }
     new_prefix[0] = '\0';
     if (prefix)
         strcpy(new_prefix, prefix);
@@ -1899,6 +1903,10 @@ static PyObject *CreateSwappedType(PyTypeObject *type, PyObject *args, PyObject
 #else
         suffix = PyUnicode_InternFromString("_be");
 #endif
+    if (suffix == NULL) {
+        Py_DECREF(swapped_args);
+        return NULL;
+    }
 
     newname = PyUnicode_Concat(name, suffix);
     if (newname == NULL) {

Modules/_ctypes/callbacks.c

@@ -310,7 +310,6 @@ static CThunkObject* CThunkObject_new(Py_ssize_t nArgs)
 
     p = PyObject_GC_NewVar(CThunkObject, &PyCThunk_Type, nArgs);
     if (p == NULL) {
-        PyErr_NoMemory();
         return NULL;
     }
 

Modules/_io/winconsoleio.c

@@ -816,11 +816,13 @@ _io__WindowsConsoleIO_readall_impl(winconsoleio *self)
             }
             bufsize = newsize;
 
-            buf = PyMem_Realloc(buf, (bufsize + 1) * sizeof(wchar_t));
-            if (!buf) {
+            wchar_t *tmp = PyMem_Realloc(buf,
+                                         (bufsize + 1) * sizeof(wchar_t));
+            if (tmp == NULL) {
                 PyMem_Free(buf);
                 return NULL;
             }
+            buf = tmp;
         }
 
         subbuf = read_console_w(self->handle, bufsize - len, &n);

Modules/_multiprocessing/semaphore.c

@@ -449,8 +449,9 @@ semlock_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
 
     if (!unlink) {
         name_copy = PyMem_Malloc(strlen(name) + 1);
-        if (name_copy == NULL)
-            goto failure;
+        if (name_copy == NULL) {
+            return PyErr_NoMemory();
+        }
         strcpy(name_copy, name);
     }
 
@@ -473,7 +474,9 @@ semlock_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
     if (handle != SEM_FAILED)
         SEM_CLOSE(handle);
     PyMem_Free(name_copy);
-    _PyMp_SetError(NULL, MP_STANDARD_ERROR);
+    if (!PyErr_Occurred()) {
+        _PyMp_SetError(NULL, MP_STANDARD_ERROR);
+    }
     return NULL;
 }
 

Modules/_ssl.c

@@ -911,6 +911,11 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock,
     PySSL_BEGIN_ALLOW_THREADS
     self->ssl = SSL_new(ctx);
     PySSL_END_ALLOW_THREADS
+    if (self->ssl == NULL) {
+        Py_DECREF(self);
+        _setSSLError(NULL, 0, __FILE__, __LINE__);
+        return NULL;
+    }
     SSL_set_app_data(self->ssl, self);
     if (sock) {
         SSL_set_fd(self->ssl, Py_SAFE_DOWNCAST(sock->sock_fd, SOCKET_T, int));
@@ -1240,6 +1245,10 @@ _get_peer_alt_names (X509 *certificate) {
 
     /* get a memory buffer */
     biobuf = BIO_new(BIO_s_mem());
+    if (biobuf == NULL) {
+        PyErr_SetString(PySSLErrorObject, "failed to allocate BIO");
+        return NULL;
+    }
 
     names = (GENERAL_NAMES *)X509_get_ext_d2i(
         certificate, NID_subject_alt_name, NULL, NULL);
@@ -1592,6 +1601,10 @@ _decode_certificate(X509 *certificate) {
 
     /* get a memory buffer */
     biobuf = BIO_new(BIO_s_mem());
+    if (biobuf == NULL) {
+        PyErr_SetString(PySSLErrorObject, "failed to allocate BIO");
+        goto fail0;
+    }
 
     (void) BIO_reset(biobuf);
     serialNumber = X509_get_serialNumber(certificate);

Modules/mathmodule.c

@@ -2142,7 +2142,7 @@ math_dist_impl(PyObject *module, PyObject *p, PyObject *q)
     if (n > NUM_STACK_ELEMS) {
         diffs = (double *) PyObject_Malloc(n * sizeof(double));
         if (diffs == NULL) {
-            return NULL;
+            return PyErr_NoMemory();
         }
     }
     for (i=0 ; i<n ; i++) {
@@ -2199,8 +2199,9 @@ math_hypot(PyObject *self, PyObject *args)
     n = PyTuple_GET_SIZE(args);
     if (n > NUM_STACK_ELEMS) {
         coordinates = (double *) PyObject_Malloc(n * sizeof(double));
-        if (coordinates == NULL)
-            return NULL;
+        if (coordinates == NULL) {
+            return PyErr_NoMemory();
+        }
     }
     for (i=0 ; i<n ; i++) {
         item = PyTuple_GET_ITEM(args, i);

Modules/posixmodule.c

@@ -6651,8 +6651,7 @@ os_getgroups_impl(PyObject *module)
     } else {
         alt_grouplist = PyMem_New(gid_t, n);
         if (alt_grouplist == NULL) {
-            errno = EINVAL;
-            return posix_error();
+            return PyErr_NoMemory();
         }
     }
 
@@ -6677,8 +6676,7 @@ os_getgroups_impl(PyObject *module)
             } else {
                 alt_grouplist = PyMem_New(gid_t, n);
                 if (alt_grouplist == NULL) {
-                    errno = EINVAL;
-                    return posix_error();
+                    return PyErr_NoMemory();
                 }
                 n = getgroups(n, alt_grouplist);
                 if (n == -1) {

Objects/capsule.c

@@ -201,7 +201,7 @@ PyCapsule_Import(const char *name, int no_block)
     char *name_dup = (char *)PyMem_MALLOC(name_length);
 
     if (!name_dup) {
-        return NULL;
+        return PyErr_NoMemory();
     }
 
     memcpy(name_dup, name, name_length);