Verify MD5 is available

On some systems, such as a FIPS enabled system, MD5 hashing is
unavailable. This commit will detect when that is the case, avoiding it
where possible and raising a better error otherwise.

botocore/compat.py

@@ -237,3 +237,17 @@ def total_seconds(delta):
     day_in_seconds = delta.days * 24 * 3600.0
     micro_in_seconds = delta.microseconds / 10.0**6
     return day_in_seconds + delta.seconds + micro_in_seconds
+
+
+def md5_available():
+    """
+    Checks to see if md5 is available on this system. A given system might not
+    have access to it for various reasons, such as FIPS mode being enabled.
+    :return: True if md5 is available. False if not.
+    """
+    try:
+        import hashlib
+        hashlib.md5()
+        return True
+    except:
+        return False

botocore/exceptions.py

@@ -367,3 +367,7 @@ class InvalidConfigError(BotoCoreError):
 
 class RefreshWithMFAUnsupportedError(BotoCoreError):
     fmt = 'Cannot refresh credentials: MFA token required.'
+
+
+class MD5UnavailableError(BotoCoreError):
+    fmt = "This system does not support MD5 generation."

botocore/handlers.py

@@ -25,13 +25,13 @@
 import warnings
 
 from botocore.compat import urlsplit, urlunsplit, unquote, \
-    json, quote, six, unquote_str, ensure_bytes
+    json, quote, six, unquote_str, ensure_bytes, md5_available
 from botocore.docs.utils import AutoPopulatedParam
 from botocore.docs.utils import HideParamFromOperations
 from botocore.docs.utils import AppendParamDocumentation
 from botocore.signers import add_generate_presigned_url
 from botocore.signers import add_generate_presigned_post
-from botocore.exceptions import ParamValidationError
+from botocore.exceptions import ParamValidationError, MD5UnavailableError
 from botocore.exceptions import UnsupportedTLSVersionWarning
 from botocore.utils import percent_encode, SAFE_CHARS
 
@@ -122,6 +122,9 @@ def json_decode_template_body(parsed, **kwargs):
 
 
 def calculate_md5(params, **kwargs):
+    if not md5_available():
+        raise MD5UnavailableError()
+
     request_dict = params
     if request_dict['body'] and 'Content-MD5' not in params['headers']:
         body = request_dict['body']
@@ -150,7 +153,7 @@ def _calculate_md5_from_file(fileobj):
 def conditionally_calculate_md5(params, **kwargs):
     """Only add a Content-MD5 when not using sigv4"""
     signer = kwargs['request_signer']
-    if signer.signature_version not in ['v4', 's3v4']:
+    if signer.signature_version not in ['v4', 's3v4'] and md5_available():
         calculate_md5(params, **kwargs)
 
 
@@ -188,6 +191,9 @@ def copy_source_sse_md5(params, **kwargs):
 def _sse_md5(params, sse_member_prefix='SSECustomer'):
     if not _needs_s3_sse_customization(params, sse_member_prefix):
         return
+    if not md5_available():
+        raise MD5UnavailableError()
+
     sse_key_member = sse_member_prefix + 'Key'
     sse_md5_member = sse_member_prefix + 'KeyMD5'
     key_as_bytes = params[sse_key_member]

tests/unit/test_compat.py

@@ -12,8 +12,11 @@
 # language governing permissions and limitations under the License.
 
 import datetime
+import mock
 
-from botocore.compat import total_seconds, unquote_str, six, ensure_bytes
+from botocore.compat import (
+    total_seconds, unquote_str, six, ensure_bytes, md5_available
+)
 
 from tests import BaseEnvVar, unittest
 
@@ -78,3 +81,14 @@ def test_non_string_or_bytes_raises_error(self):
         value = 500
         with self.assertRaises(ValueError):
             ensure_bytes(value)
+
+
+class TestMD5Available(unittest.TestCase):
+    def test_available(self):
+        with mock.patch('hashlib.md5'):
+            self.assertTrue(md5_available())
+
+    def test_unavailable(self):
+        with mock.patch('hashlib.md5') as md5:
+            md5.side_effect = ValueError()
+            self.assertFalse(md5_available())

tests/unit/test_handlers.py

@@ -20,7 +20,7 @@
 
 import botocore
 import botocore.session
-from botocore.exceptions import ParamValidationError
+from botocore.exceptions import ParamValidationError, MD5UnavailableError
 from botocore.awsrequest import AWSRequest
 from botocore.compat import quote, six
 from botocore.model import OperationModel, ServiceModel
@@ -509,60 +509,7 @@ def test_switch_host_with_param(self):
         handlers.switch_host_with_param(request, 'PredictEndpoint')
         self.assertEqual(request.url, new_endpoint)
 
-    def test_does_not_add_md5_when_v4(self):
-        credentials = Credentials('key', 'secret')
-        request_signer = RequestSigner(
-            's3', 'us-east-1', 's3', 'v4', credentials, mock.Mock())
-        request_dict = {'body': b'bar',
-                        'url': 'https://s3.us-east-1.amazonaws.com',
-                        'method': 'PUT',
-                        'headers': {}}
-        handlers.conditionally_calculate_md5(request_dict,
-                                             request_signer=request_signer)
-        self.assertTrue('Content-MD5' not in request_dict['headers'])
-
-    def test_does_not_add_md5_when_s3v4(self):
-        credentials = Credentials('key', 'secret')
-        request_signer = RequestSigner(
-            's3', 'us-east-1', 's3', 's3v4', credentials, mock.Mock())
-        request_dict = {'body': b'bar',
-                        'url': 'https://s3.us-east-1.amazonaws.com',
-                        'method': 'PUT',
-                        'headers': {}}
-        handlers.conditionally_calculate_md5(request_dict,
-                                             request_signer=request_signer)
-        self.assertTrue('Content-MD5' not in request_dict['headers'])
-
-    def test_adds_md5_when_not_v4(self):
-        credentials = Credentials('key', 'secret')
-        request_signer = RequestSigner(
-            's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
-        request_dict = {'body': b'bar',
-                        'url': 'https://s3.us-east-1.amazonaws.com',
-                        'method': 'PUT',
-                        'headers': {}}
-        handlers.conditionally_calculate_md5(request_dict,
-                                             request_signer=request_signer)
-        self.assertTrue('Content-MD5' in request_dict['headers'])
 
-    def test_adds_md5_with_file_like_body(self):
-        request_dict = {
-            'body': six.BytesIO(b'foobar'),
-            'headers': {}
-        }
-        handlers.calculate_md5(request_dict)
-        self.assertEqual(request_dict['headers']['Content-MD5'],
-                         'OFj2IjCsPJFfMAxmQxLGPw==')
-
-    def test_adds_md5_with_bytes_object(self):
-        request_dict = {
-            'body': b'foobar',
-            'headers': {}
-        }
-        handlers.calculate_md5(request_dict)
-        self.assertEqual(
-            request_dict['headers']['Content-MD5'],
-            'OFj2IjCsPJFfMAxmQxLGPw==')
 
     def test_invalid_char_in_bucket_raises_exception(self):
         params = {
@@ -735,3 +682,127 @@ def test_s3_special_case_is_before_other_retry(self):
         self.assertTrue(s3_200_handler < general_retry_handler,
                         "S3 200 error handler was supposed to be before "
                         "the general retry handler, but it was not.")
+
+
+class BaseMD5Test(BaseSessionTest):
+    def setUp(self, **environ):
+        super(BaseMD5Test, self).setUp(**environ)
+        self.md5_object = mock.Mock()
+        self.md5_digest = mock.Mock(return_value=b'foo')
+        self.md5_object.digest = self.md5_digest
+        self.md5_builder = mock.Mock(return_value=self.md5_object)
+        self.md5_patch = mock.patch('hashlib.md5', self.md5_builder)
+        self.md5_patch.start()
+
+    def tearDown(self):
+        super(BaseMD5Test, self).tearDown()
+        self.md5_patch.stop()
+
+
+class TestSSEMD5(BaseMD5Test):
+    def test_sse_md5(self):
+        key = 'SSECustomerKey'
+        params = {key: b'foo'}
+        handlers.sse_md5(params)
+        self.assertEqual(params[key + 'MD5'], 'Zm9v')
+
+        key = 'CopySourceSSECustomerKey'
+        params = {key: b'foo'}
+        handlers.copy_source_sse_md5(params)
+        self.assertEqual(params[key + 'MD5'], 'Zm9v')
+
+    def test_raises_error_when_md5_unavailable(self):
+        self.md5_builder.side_effect = ValueError()
+
+        with self.assertRaises(MD5UnavailableError):
+            handlers.sse_md5({'SSECustomerKey': b'foo'})
+
+        with self.assertRaises(MD5UnavailableError):
+            handlers.copy_source_sse_md5({'CopySourceSSECustomerKey': b'foo'})
+
+
+class TestAddMD5(BaseMD5Test):
+    def test_does_not_add_md5_when_v4(self):
+        credentials = Credentials('key', 'secret')
+        request_signer = RequestSigner(
+            's3', 'us-east-1', 's3', 'v4', credentials, mock.Mock())
+        request_dict = {'body': b'bar',
+                        'url': 'https://s3.us-east-1.amazonaws.com',
+                        'method': 'PUT',
+                        'headers': {}}
+        handlers.conditionally_calculate_md5(request_dict,
+                                             request_signer=request_signer)
+        self.assertTrue('Content-MD5' not in request_dict['headers'])
+
+    def test_does_not_add_md5_when_s3v4(self):
+        credentials = Credentials('key', 'secret')
+        request_signer = RequestSigner(
+            's3', 'us-east-1', 's3', 's3v4', credentials, mock.Mock())
+        request_dict = {'body': b'bar',
+                        'url': 'https://s3.us-east-1.amazonaws.com',
+                        'method': 'PUT',
+                        'headers': {}}
+        handlers.conditionally_calculate_md5(request_dict,
+                                             request_signer=request_signer)
+        self.assertTrue('Content-MD5' not in request_dict['headers'])
+
+    def test_conditional_does_not_add_when_md5_unavailable(self):
+        credentials = Credentials('key', 'secret')
+        request_signer = RequestSigner(
+            's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
+        request_dict = {'body': b'bar',
+                        'url': 'https://s3.us-east-1.amazonaws.com',
+                        'method': 'PUT',
+                        'headers': {}}
+
+        self.md5_builder.side_effect = ValueError()
+        handlers.conditionally_calculate_md5(
+            request_dict, request_signer=request_signer)
+        self.assertFalse('Content-MD5' in request_dict['headers'])
+
+    def test_add_md5_raises_error_when_md5_unavailable(self):
+        credentials = Credentials('key', 'secret')
+        request_signer = RequestSigner(
+            's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
+        request_dict = {'body': b'bar',
+                        'url': 'https://s3.us-east-1.amazonaws.com',
+                        'method': 'PUT',
+                        'headers': {}}
+
+        self.md5_builder.side_effect = ValueError()
+        with self.assertRaises(MD5UnavailableError):
+            handlers.calculate_md5(
+                request_dict, request_signer=request_signer)
+
+    def test_adds_md5_when_not_v4(self):
+        credentials = Credentials('key', 'secret')
+        request_signer = RequestSigner(
+            's3', 'us-east-1', 's3', 's3', credentials, mock.Mock())
+        request_dict = {'body': b'bar',
+                        'url': 'https://s3.us-east-1.amazonaws.com',
+                        'method': 'PUT',
+                        'headers': {}}
+        handlers.conditionally_calculate_md5(request_dict,
+                                             request_signer=request_signer)
+        self.assertTrue('Content-MD5' in request_dict['headers'])
+
+    def test_add_md5_with_file_like_body(self):
+        request_dict = {
+            'body': six.BytesIO(b'foobar'),
+            'headers': {}
+        }
+        self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?'
+        handlers.calculate_md5(request_dict)
+        self.assertEqual(request_dict['headers']['Content-MD5'],
+                         'OFj2IjCsPJFfMAxmQxLGPw==')
+
+    def test_add_md5_with_bytes_object(self):
+        request_dict = {
+            'body': b'foobar',
+            'headers': {}
+        }
+        self.md5_digest.return_value = b'8X\xf6"0\xac<\x91_0\x0cfC\x12\xc6?'
+        handlers.calculate_md5(request_dict)
+        self.assertEqual(
+            request_dict['headers']['Content-MD5'],
+            'OFj2IjCsPJFfMAxmQxLGPw==')