BigQuery: replace httplib2 with Requests

Author: Jon Wayne Parrott

bigquery/google/cloud/bigquery/client.py

@@ -64,10 +64,10 @@ class Client(ClientWithProject):
                         passed), falls back to the default inferred from the
                         environment.
 
-    :type _http: :class:`~httplib2.Http`
+    :type _http: :class:`~requests.Session`
     :param _http: (Optional) HTTP object to make requests. Can be any object
                   that defines ``request()`` with the same interface as
-                  :meth:`~httplib2.Http.request`. If not passed, an
+                  :meth:`requests.Session.request`. If not passed, an
                   ``_http`` object is created that is bound to the
                   ``credentials`` for the current object.
                   This parameter should be considered private, and could

bigquery/google/cloud/bigquery/dataset.py

@@ -364,7 +364,7 @@ def _parse_access_grants(access):
     def _set_properties(self, api_response):
         """Update properties from resource in body of ``api_response``
 
-        :type api_response: httplib2.Response
+        :type api_response: dict
         :param api_response: response returned from an API call.
         """
         self._properties.clear()

bigquery/google/cloud/bigquery/job.py

@@ -14,7 +14,6 @@
 
 """Define API Jobs."""
 
-import collections
 import threading
 
 import six
@@ -58,8 +57,6 @@
     'tableUnavailable': http_client.BAD_REQUEST,
 }
 
-_FakeResponse = collections.namedtuple('_FakeResponse', ['status'])
-
 
 def _error_result_to_exception(error_result):
     """Maps BigQuery error reasons to an exception.
@@ -80,12 +77,8 @@ def _error_result_to_exception(error_result):
     status_code = _ERROR_REASON_TO_EXCEPTION.get(
         reason, http_client.INTERNAL_SERVER_ERROR)
     # make_exception expects an httplib2 response object.
-    fake_response = _FakeResponse(status=status_code)
-    return exceptions.make_exception(
-        fake_response,
-        error_result.get('message', ''),
-        error_info=error_result,
-        use_json=False)
+    return exceptions.from_http_status(
+        status_code, error_result.get('message', ''), errors=[error_result])
 
 
 class Compression(_EnumProperty):
@@ -307,7 +300,7 @@ def _scrub_local_properties(self, cleaned):
     def _set_properties(self, api_response):
         """Update properties from resource in body of ``api_response``
 
-        :type api_response: httplib2.Response
+        :type api_response: dict
         :param api_response: response returned from an API call
         """
         cleaned = api_response.copy()

bigquery/google/cloud/bigquery/query.py

@@ -310,7 +310,7 @@ def schema(self):
     def _set_properties(self, api_response):
         """Update properties from resource in body of ``api_response``
 
-        :type api_response: httplib2.Response
+        :type api_response: dict
         :param api_response: response returned from an API call
         """
         self._properties.clear()

bigquery/google/cloud/bigquery/table.py

@@ -17,18 +17,16 @@
 import datetime
 import os
 
-import httplib2
 import six
 
 import google.auth.transport.requests
 from google import resumable_media
 from google.resumable_media.requests import MultipartUpload
 from google.resumable_media.requests import ResumableUpload
 
+from google.cloud import exceptions
 from google.cloud._helpers import _datetime_from_microseconds
 from google.cloud._helpers import _millis_from_datetime
-from google.cloud.exceptions import NotFound
-from google.cloud.exceptions import make_exception
 from google.cloud.iterator import HTTPIterator
 from google.cloud.bigquery.schema import SchemaField
 from google.cloud.bigquery._helpers import _item_to_row
@@ -474,7 +472,7 @@ def _require_client(self, client):
     def _set_properties(self, api_response):
         """Update properties from resource in body of ``api_response``
 
-        :type api_response: httplib2.Response
+        :type api_response: dict
         :param api_response: response returned from an API call
         """
         self._properties.clear()
@@ -563,7 +561,7 @@ def exists(self, client=None):
         try:
             client._connection.api_request(method='GET', path=self.path,
                                            query_params={'fields': 'id'})
-        except NotFound:
+        except exceptions.NotFound:
             return False
         else:
             return True
@@ -1113,7 +1111,7 @@ def upload_from_file(self,
                 client, file_obj, metadata, size, num_retries)
             return client.job_from_resource(created_json)
         except resumable_media.InvalidResponse as exc:
-            _raise_from_invalid_response(exc)
+            raise exceptions.from_http_response(exc.response)
     # pylint: enable=too-many-arguments,too-many-locals
 
 
@@ -1298,22 +1296,3 @@ def _get_upload_metadata(source_format, schema, dataset, name):
             'load': load_config,
         },
     }
-
-
-def _raise_from_invalid_response(error, error_info=None):
-    """Re-wrap and raise an ``InvalidResponse`` exception.
-
-    :type error: :exc:`google.resumable_media.InvalidResponse`
-    :param error: A caught exception from the ``google-resumable-media``
-                  library.
-
-    :type error_info: str
-    :param error_info: (Optional) Extra information about the failed request.
-
-    :raises: :class:`~google.cloud.exceptions.GoogleCloudError` corresponding
-             to the failed status code
-    """
-    response = error.response
-    faux_response = httplib2.Response({'status': response.status_code})
-    raise make_exception(faux_response, response.content,
-                         error_info=error_info, use_json=False)

bigquery/tests/unit/test__http.py

@@ -15,6 +15,7 @@
 import unittest
 
 import mock
+import requests
 
 
 class TestConnection(unittest.TestCase):
@@ -55,10 +56,12 @@ def test_extra_headers(self):
         from google.cloud import _http as base_http
         from google.cloud.bigquery import _http as MUT
 
-        http = mock.Mock(spec=['request'])
-        response = mock.Mock(status=200, spec=['status'])
+        http = mock.create_autospec(requests.Session, instance=True)
+        response = requests.Response()
+        response.status_code = 200
         data = b'brent-spiner'
-        http.request.return_value = response, data
+        response._content = data
+        http.request.return_value = response
         client = mock.Mock(_http=http, spec=['_http'])
 
         conn = self._make_one(client)
@@ -68,15 +71,14 @@ def test_extra_headers(self):
         self.assertEqual(result, data)
 
         expected_headers = {
-            'Content-Length': str(len(req_data)),
             'Accept-Encoding': 'gzip',
             base_http.CLIENT_INFO_HEADER: MUT._CLIENT_INFO,
             'User-Agent': conn.USER_AGENT,
         }
         expected_uri = conn.build_api_url('/rainbow')
         http.request.assert_called_once_with(
-            body=req_data,
+            data=req_data,
             headers=expected_headers,
             method='GET',
-            uri=expected_uri,
+            url=expected_uri,
         )

bigquery/tests/unit/test_job.py

@@ -31,7 +31,7 @@ def test_simple(self):
         exception = self._call_fut(error_result)
         self.assertEqual(exception.code, http_client.BAD_REQUEST)
         self.assertTrue(exception.message.startswith('bad request'))
-        self.assertIn("'reason': 'invalid'", exception.message)
+        self.assertIn(error_result, exception.errors)
 
     def test_missing_reason(self):
         error_result = {}

bigquery/tests/unit/test_table.py

@@ -1744,9 +1744,11 @@ def _make_table():
     def _make_response(status_code, content='', headers={}):
         """Make a mock HTTP response."""
         import requests
-        response = mock.create_autospec(requests.Response, instance=True)
-        response.content = content.encode('utf-8')
-        response.headers = headers
+        response = requests.Response()
+        response.request = requests.Request(
+            'POST', 'http://example.com').prepare()
+        response._content = content.encode('utf-8')
+        response.headers.update(headers)
         response.status_code = status_code
         return response
 
@@ -1921,7 +1923,7 @@ def test_upload_from_file_failure(self):
             table.upload_from_file(
                 file_obj, source_format='CSV', rewind=True)
 
-        assert exc_info.value.message == response.content.decode('utf-8')
+        assert response.text in exc_info.value.message
         assert exc_info.value.errors == []
 
     def test_upload_from_file_bad_mode(self):