#34265 add handling for Python versions incompatible with SHA-2

The hosted Shotgun server certificates are being upgraded to more secure ones signed with SHA-2. Some older versions of Python will have issues with this change as they do not support SHA-2 encryption. In order to try and prevent scripts from breaking, when the API encounters a version of Python that is incompatible with SHA-2, it will automatically turn off certificate verification and try the request again. If the validation still fails for some reason, the error will be raised, otherwise the request succeeds and validation will remain off for the remaining life of the connection.

There is also support for the `SHOTGUN_FORCE_CERTIFICATE_VALIDATION` environment variable which when set (the value does not matter), will prevent disabling certificate verification and will instead raise an exception.

This behavior of having certificate validation off, is actually the default in Python versions < v2.7.9. Up to this point we have been electing to enhance the default level of security. Your connection is still encrypted when certificate validation is off, but the server identity cannot be verified.

Adds info showing the OpenSSL version (if available) and whether certificate validation is enabled or not, to the user-agent string

shotgun_api3/sg_24.py

@@ -2,7 +2,7 @@
 import sys
 import logging
 
-from shotgun_api3.lib.httplib2 import Http, ProxyInfo, socks
+from shotgun_api3.lib.httplib2 import Http, ProxyInfo, socks, SSLHandshakeError
 from shotgun_api3.lib.sgtimezone import SgTimezone
 from shotgun_api3.lib.xmlrpclib import Error, ProtocolError, ResponseError
 import mimetypes    # used for attachment upload

shotgun_api3/sg_25.py

@@ -2,7 +2,7 @@
 import os
 import logging
 
-from .lib.httplib2 import Http, ProxyInfo, socks
+from .lib.httplib2 import Http, ProxyInfo, socks, SSLHandshakeError
 from .lib.sgtimezone import SgTimezone
 from .lib.xmlrpclib import Error, ProtocolError, ResponseError
 import mimetypes    # used for attachment upload

shotgun_api3/sg_26.py

@@ -2,7 +2,7 @@
 import os
 import logging
 
-from .lib.httplib2 import Http, ProxyInfo, socks
+from .lib.httplib2 import Http, ProxyInfo, socks, SSLHandshakeError
 from .lib.sgtimezone import SgTimezone
 from .lib.xmlrpclib import Error, ProtocolError, ResponseError
 

shotgun_api3/shotgun.py

@@ -66,19 +66,19 @@
 
 SG_TIMEZONE = SgTimezone()
 
-
 NO_SSL_VALIDATION = False
 try:
-    import ssl
-    if os.environ.get("SHOTGUN_DISABLE_SSL_VALIDATION", False):
-        NO_SSL_VALIDATION = True
-except ImportError:
+    import ssl        
+except ImportError, e:
+    if "SHOTGUN_FORCE_CERTIFICATE_VALIDATION" in os.environ:
+        raise ImportError("%s. SHOTGUN_FORCE_CERTIFICATE_VALIDATION environment variable prevents "
+                          "disabling SSL certificate validation." % e)
     LOG.debug("ssl not found, disabling certificate validation")
     NO_SSL_VALIDATION = True
 
 # ----------------------------------------------------------------------------
 # Version
-__version__ = "3.0.24"
+__version__ = "3.0.25.Dev"
 
 # ----------------------------------------------------------------------------
 # Errors
@@ -217,10 +217,18 @@ def __init__(self):
 
         self.py_version = ".".join(str(x) for x in sys.version_info[:2])
 
+        # extract the OpenSSL version if we can. The version is only available in Python 2.7 and
+        # only if we successfully imported ssl
+        self.ssl_version = "unknown"
+        try:
+            self.ssl_version = ssl.OPENSSL_VERSION
+        except (AttributeError, NameError):
+            pass
+
     def __str__(self):
         return "ClientCapabilities: platform %s, local_path_field %s, "\
-            "py_verison %s" % (self.platform, self.local_path_field,
-            self.py_version)
+            "py_verison %s, ssl version %s" % (self.platform, self.local_path_field,
+            self.py_version, self.ssl_version)
 
 class _Config(object):
     """Container for the client configuration."""
@@ -1259,13 +1267,22 @@ def add_user_agent(self, agent):
     def reset_user_agent(self):
         """Reset user agent to the default.
 
-        Eg. shotgun-json (3.0.17); Python 2.6 (Mac)
+        Eg. "shotgun-json (3.0.17); Python 2.6 (Mac); ssl OpenSSL 1.0.2d 9 Jul 2015 (validate)"
         """
         ua_platform = "Unknown"
         if self.client_caps.platform is not None:
             ua_platform = self.client_caps.platform.capitalize()
+
+
+        # create ssl validation string based on settings
+        validation_str = "validate"
+        if self.config.no_ssl_validation:
+            validation_str = "no-validate"
+
         self._user_agents = ["shotgun-json (%s)" % __version__,
-                             "Python %s (%s)" % (self.client_caps.py_version, ua_platform)]
+                             "Python %s (%s)" % (self.client_caps.py_version, ua_platform),
+                             "ssl %s (%s)" % (self.client_caps.ssl_version, validation_str)]
+
 
     def set_session_uuid(self, session_uuid):
         """Sets the browser session_uuid for this API session.
@@ -2000,6 +2017,16 @@ def _build_opener(self, handler):
             opener = urllib2.build_opener(handler)
         return opener
 
+    def _turn_off_ssl_validation(self):
+        """Turn off SSL certificate validation."""
+        global NO_SSL_VALIDATION
+        self.config.no_ssl_validation = True
+        NO_SSL_VALIDATION = True
+        # reset ssl-validation in user-agents
+        self._user_agents = ["ssl %s (no-validate)" % self.client_caps.ssl_version 
+                             if ua.startswith("ssl ") else ua 
+                             for ua in self._user_agents] 
+
     # Deprecated methods from old wrapper
     def schema(self, entity_type):
         raise ShotgunError("Deprecated: use schema_field_read('type':'%s') "
@@ -2149,9 +2176,8 @@ def _make_call(self, verb, path, body, headers):
         """
 
         attempt = 0
-        req_headers = {
-            "user-agent": "; ".join(self._user_agents),
-        }
+        req_headers = {}
+        req_headers["user-agent"] = "; ".join(self._user_agents)
         if self.config.authorization:
             req_headers["Authorization"] = self.config.authorization
 
@@ -2164,6 +2190,38 @@ def _make_call(self, verb, path, body, headers):
             attempt += 1
             try:
                 return self._http_request(verb, path, body, req_headers)
+            except SSLHandshakeError, e:
+                # Test whether the exception is due to the fact that this is an older version of
+                # Python that cannot validate certificates encrypted with SHA-2. If it is, then 
+                # fall back on disabling the certificate validation and try again - unless the
+                # SHOTGUN_FORCE_CERTIFICATE_VALIDATION environment variable has been set by the 
+                # user. In that case we simply raise the exception. Any other exceptions simply 
+                # get raised as well. 
+                #
+                # For more info see:
+                # http://blog.shotgunsoftware.com/2016/01/important-ssl-certificate-renewal-and.html
+                #
+                # SHA-2 errors look like this: 
+                #   [Errno 1] _ssl.c:480: error:0D0C50A1:asn1 encoding routines:ASN1_item_verify:
+                #   unknown message digest algorithm
+                # 
+                # Any other exceptions simply get raised.
+                if not str(e).endswith("unknown message digest algorithm") or \
+                   "SHOTGUN_FORCE_CERTIFICATE_VALIDATION" in os.environ:
+                    raise
+
+                if self.config.no_ssl_validation is False:
+                    LOG.warning("SSLHandshakeError: this Python installation is incompatible with "
+                                "certificates signed with SHA-2. Disabling certificate validation. "
+                                "For more information, see http://blog.shotgunsoftware.com/2016/01/"
+                                "important-ssl-certificate-renewal-and.html")
+                    self._turn_off_ssl_validation()
+                    # reload user agent to reflect that we have turned off ssl validation
+                    req_headers["user-agent"] = "; ".join(self._user_agents)
+
+                self._close_connection()
+                if attempt == max_rpc_attempts:
+                    raise
             except Exception:
                 #TODO: LOG ?
                 self._close_connection()

tests/test_api.py

@@ -15,7 +15,7 @@
 import urllib2
 
 import shotgun_api3
-from shotgun_api3.lib.httplib2 import Http
+from shotgun_api3.lib.httplib2 import Http, SSLHandshakeError
 
 import base
 
@@ -1470,6 +1470,65 @@ def test_status_not_200(self, mock_request):
         mock_request.return_value = (response, {})
         self.assertRaises(shotgun_api3.ProtocolError, self.sg.find_one, 'Shot', [])
 
+    @patch('shotgun_api3.shotgun.Http.request')
+    def test_sha2_error(self, mock_request):
+        # Simulate the SSLHandshakeError raised with SHA-2 errors
+        mock_request.side_effect = SSLHandshakeError("[Errno 1] _ssl.c:480: error:0D0C50A1:asn1 "
+                                    "encoding routines:ASN1_item_verify: unknown message digest "
+                                    "algorithm")
+
+        # save the original state
+        original_env_val = os.environ.pop("SHOTGUN_FORCE_CERTIFICATE_VALIDATION", None)
+
+        # ensure we're starting with the right values
+        self.sg.reset_user_agent()
+
+        # ensure the initial settings are correct
+        self.assertFalse(self.sg.config.no_ssl_validation)
+        self.assertFalse(shotgun_api3.shotgun.NO_SSL_VALIDATION)
+        self.assertTrue("(validate)" in " ".join(self.sg._user_agents))
+        self.assertFalse("(no-validate)" in " ".join(self.sg._user_agents))
+        try:
+            result = self.sg.info()
+        except SSLHandshakeError:
+            # ensure the api has reset the values in the correct fallback behavior
+            self.assertTrue(self.sg.config.no_ssl_validation)
+            self.assertTrue(shotgun_api3.shotgun.NO_SSL_VALIDATION)
+            self.assertFalse("(validate)" in " ".join(self.sg._user_agents))
+            self.assertTrue("(no-validate)" in " ".join(self.sg._user_agents))
+
+        if original_env_val is not None:
+            os.environ["SHOTGUN_FORCE_CERTIFICATE_VALIDATION"] = original_env_val
+
+    @patch('shotgun_api3.shotgun.Http.request')
+    def test_sha2_error_with_strict(self, mock_request):
+        # Simulate the SSLHandshakeError raised with SHA-2 errors
+        mock_request.side_effect = SSLHandshakeError("[Errno 1] _ssl.c:480: error:0D0C50A1:asn1 "
+                                    "encoding routines:ASN1_item_verify: unknown message digest "
+                                    "algorithm")
+
+        # save the original state
+        original_env_val = os.environ.pop("SHOTGUN_FORCE_CERTIFICATE_VALIDATION", None)
+        os.environ["SHOTGUN_FORCE_CERTIFICATE_VALIDATION"] = "1"
+
+        # ensure we're starting with the right values
+        self.sg.config.no_ssl_validation = False
+        shotgun_api3.shotgun.NO_SSL_VALIDATION = False
+        self.sg.reset_user_agent()
+
+        try:
+            result = self.sg.info()
+        except SSLHandshakeError:
+            # ensure the api has NOT reset the values in the fallback behavior because we have
+            # set the env variable to force validation
+            self.assertFalse(self.sg.config.no_ssl_validation)
+            self.assertFalse(shotgun_api3.shotgun.NO_SSL_VALIDATION)
+            self.assertFalse("(no-validate)" in " ".join(self.sg._user_agents))
+            self.assertTrue("(validate)" in " ".join(self.sg._user_agents))
+
+        if original_env_val is not None:
+            os.environ["SHOTGUN_FORCE_CERTIFICATE_VALIDATION"] = original_env_val
+
     @patch.object(urllib2.OpenerDirector, 'open')
     def test_sanitized_auth_params(self, mock_open):
         # Simulate the server blowing up and giving us a 500 error

tests/test_client.py

@@ -48,6 +48,8 @@ def test_detect_client_caps(self):
         self.assertTrue(str(client_caps).startswith("ClientCapabilities"))
         self.assertTrue(client_caps.py_version.startswith(str(sys.version_info[0])))
         self.assertTrue(client_caps.py_version.endswith(str(sys.version_info[1])))
+        self.assertTrue(client_caps.ssl_version is not None)
+        # todo test for version string (eg. "1.2.3ng") or "unknown"
 
     def test_detect_server_caps(self):
         '''test_detect_server_caps tests that ServerCapabilities object is made
@@ -158,23 +160,30 @@ def test_user_agent(self):
         # test default user agent
         self.sg.info()
         client_caps = self.sg.client_caps
+        config = self.sg.config
         args, _ = self.sg._http_request.call_args
         (_, _, _, headers) = args
-        expected = "shotgun-json (%s); Python %s (%s)" % (api.__version__, 
-                                                          client_caps.py_version,
-                                                          client_caps.platform.capitalize()
-                                                          )
+        ssl_validate_lut = {True: "no-validate", False: "validate"}
+        expected = "shotgun-json (%s); Python %s (%s); ssl %s (%s)" % (
+                        api.__version__, 
+                        client_caps.py_version,
+                        client_caps.platform.capitalize(),
+                        client_caps.ssl_version,
+                        ssl_validate_lut[config.no_ssl_validation]
+                        )
         self.assertEqual(expected, headers.get("user-agent"))
 
         # test adding to user agent
         self.sg.add_user_agent("test-agent")
         self.sg.info()
         args, _ = self.sg._http_request.call_args
         (_, _, _, headers) = args
-        expected = "shotgun-json (%s); Python %s (%s); test-agent" % (
+        expected = "shotgun-json (%s); Python %s (%s); ssl %s (%s); test-agent" % (
                         api.__version__, 
                         client_caps.py_version,
-                        client_caps.platform.capitalize()
+                        client_caps.platform.capitalize(),
+                        client_caps.ssl_version,
+                        ssl_validate_lut[config.no_ssl_validation]
                         )
         self.assertEqual(expected, headers.get("user-agent"))
 
@@ -183,10 +192,13 @@ def test_user_agent(self):
         self.sg.info()
         args, _ = self.sg._http_request.call_args
         (_, _, _, headers) = args
-        expected = "shotgun-json (%s); Python %s (%s)" % (api.__version__, 
-                                                          client_caps.py_version,
-                                                          client_caps.platform.capitalize(),
-                                                          )
+        expected = "shotgun-json (%s); Python %s (%s); ssl %s (%s)" % (
+                        api.__version__, 
+                        client_caps.py_version,
+                        client_caps.platform.capitalize(),
+                        client_caps.ssl_version,
+                        ssl_validate_lut[config.no_ssl_validation]
+                        )
         self.assertEqual(expected, headers.get("user-agent"))
 
     def test_connect_close(self):