[3.6] bpo-34922: Fix integer overflow in the digest() and hexdigest() methods (GH-9751) (GH-9798) (GH-9801)

for the SHAKE algorithm in the hashlib module.
(cherry picked from commit 9b8c2e7)
(cherry picked from commit 8b040e5)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>

Author: miss-islington

Lib/test/test_hashlib.py

@@ -233,6 +233,20 @@ def test_hexdigest(self):
                 self.assertIsInstance(h.digest(), bytes)
                 self.assertEqual(hexstr(h.digest()), h.hexdigest())
 
+    def test_digest_length_overflow(self):
+        # See issue #34922
+        large_sizes = (2**29, 2**32-10, 2**32+10, 2**61, 2**64-10, 2**64+10)
+        for cons in self.hash_constructors:
+            h = cons()
+            if h.name not in self.shakes:
+                continue
+            for digest in h.digest, h.hexdigest:
+                with self.assertRaises((ValueError, OverflowError)):
+                    digest(-10)
+                for length in large_sizes:
+                    with self.assertRaises((ValueError, OverflowError)):
+                        digest(length)
+
     def test_name_attribute(self):
         for cons in self.hash_constructors:
             h = cons()

Misc/NEWS.d/next/Library/2018-10-07-21-18-52.bpo-34922.37IdsA.rst

@@ -0,0 +1,3 @@
+Fixed integer overflow in the :meth:`~hashlib.shake.digest()` and
+:meth:`~hashlib.shake.hexdigest()` methods for the SHAKE algorithm
+in the :mod:`hashlib` module.

Modules/_sha3/sha3module.c

@@ -609,7 +609,10 @@ _SHAKE_digest(SHA3object *self, PyObject *digestlen_obj, int hex)
     if (digestlen == (unsigned long) -1 && PyErr_Occurred()) {
         return NULL;
     }
-
+    if (digestlen >= (1 << 29)) {
+        PyErr_SetString(PyExc_ValueError, "length is too large");
+        return NULL;
+    }
     /* ExtractLane needs at least SHA3_MAX_DIGESTSIZE + SHA3_LANESIZE and
      * SHA3_LANESIZE extra space.
      */