Cloudml Branch Merge From Master

datalab/bigquery/_table.py

@@ -401,7 +401,7 @@ def insert_data(self, data, include_index=False, index_name=None):
       self._info = self._api.tables_get(self._name_parts)
       if 'streamingBuffer' not in self._info or \
               'estimatedRows' not in self._info['streamingBuffer'] or \
-              self._info['streamingBuffer']['estimatedRows'] > 0:
+              int(self._info['streamingBuffer']['estimatedRows']) > 0:
         break
       time.sleep(2)
 

datalab/context/_utils.py

@@ -116,7 +116,7 @@ def get_project_id():
     proc = subprocess.Popen(['gcloud', 'config', 'list', '--format', 'value(core.project)'],
                             stdout=subprocess.PIPE)
     stdout, _ = proc.communicate()
-    value = stdout.strip()
+    value = stdout.decode().strip()
     if proc.poll() == 0 and value:
       return value
   except:

datalab/utils/_http.py

@@ -53,6 +53,16 @@ class Http(object):
   """A helper class for making HTTP requests.
   """
 
+  # Reuse one Http object across requests to take advantage of Keep-Alive, e.g.
+  # for BigQuery queries that requires at least ~5 sequential http requests.
+  #
+  # TODO(nikhilko):
+  # SSL cert validation seemingly fails, and workarounds are not amenable
+  # to implementing in library code. So configure the Http object to skip
+  # doing so, in the interim.
+  http = httplib2.Http()
+  http.disable_ssl_certificate_validation = True
+
   def __init__(self):
     pass
 
@@ -109,15 +119,8 @@ def request(url, args=None, data=None, headers=None, method=None,
     if method is None:
       method = 'GET'
 
-    # Create an Http object to issue requests. Associate the credentials
-    # with it if specified to perform authorization.
-    #
-    # TODO(nikhilko):
-    # SSL cert validation seemingly fails, and workarounds are not amenable
-    # to implementing in library code. So configure the Http object to skip
-    # doing so, in the interim.
-    http = httplib2.Http()
-    http.disable_ssl_certificate_validation = True
+    # Authorize with credentials if given.
+    http = Http.http
     if credentials is not None:
       http = credentials.authorize(http)
     if stats is not None:

tests/_util/http_tests.py

@@ -23,7 +23,7 @@
 class TestCases(unittest.TestCase):
 
   @mock.patch('httplib2.Response')
-  @mock.patch('httplib2.Http.request')
+  @mock.patch('datalab.utils._http.Http.http.request')
   def test_get_request_is_invoked(self, mock_request, mock_response):
     TestCases._setup_mocks(mock_request, mock_response, '{}')
 
@@ -32,23 +32,23 @@ def test_get_request_is_invoked(self, mock_request, mock_response):
     self.assertEqual(mock_request.call_args[1]['method'], 'GET')
 
   @mock.patch('httplib2.Response')
-  @mock.patch('httplib2.Http.request')
+  @mock.patch('datalab.utils._http.Http.http.request')
   def test_post_request_is_invoked(self, mock_request, mock_response):
     TestCases._setup_mocks(mock_request, mock_response, '{}')
 
     Http.request('http://www.example.org', data={})
     self.assertEqual(mock_request.call_args[1]['method'], 'POST')
 
   @mock.patch('httplib2.Response')
-  @mock.patch('httplib2.Http.request')
+  @mock.patch('datalab.utils._http.Http.http.request')
   def test_explicit_post_request_is_invoked(self, mock_request, mock_response):
     TestCases._setup_mocks(mock_request, mock_response, '{}')
 
     Http.request('http://www.example.org', method='POST')
     self.assertEqual(mock_request.call_args[1]['method'], 'POST')
 
   @mock.patch('httplib2.Response')
-  @mock.patch('httplib2.Http.request')
+  @mock.patch('datalab.utils._http.Http.http.request')
   def test_query_string_format(self, mock_request, mock_response):
     TestCases._setup_mocks(mock_request, mock_response, '{}')
 
@@ -59,7 +59,7 @@ def test_query_string_format(self, mock_request, mock_response):
     self.assertTrue('b=a+b+c' in parts[1:])
 
   @mock.patch('httplib2.Response')
-  @mock.patch('httplib2.Http.request')
+  @mock.patch('datalab.utils._http.Http.http.request')
   def test_formats_json_request(self, mock_request, mock_response):
     TestCases._setup_mocks(mock_request, mock_response, '{}')
 
@@ -71,7 +71,7 @@ def test_formats_json_request(self, mock_request, mock_response):
                      'application/json')
 
   @mock.patch('httplib2.Response')
-  @mock.patch('httplib2.Http.request')
+  @mock.patch('datalab.utils._http.Http.http.request')
   def test_supports_custom_content(self, mock_request, mock_response):
     TestCases._setup_mocks(mock_request, mock_response, '{}')
 
@@ -83,15 +83,15 @@ def test_supports_custom_content(self, mock_request, mock_response):
     self.assertEqual(mock_request.call_args[1]['headers']['Content-Type'], 'text/plain')
 
   @mock.patch('httplib2.Response')
-  @mock.patch('httplib2.Http.request')
+  @mock.patch('datalab.utils._http.Http.http.request')
   def test_parses_json_response(self, mock_request, mock_response):
     TestCases._setup_mocks(mock_request, mock_response, '{"abc":123}')
 
     data = Http.request('http://www.example.org')
     self.assertEqual(data['abc'], 123)
 
   @mock.patch('httplib2.Response')
-  @mock.patch('httplib2.Http.request')
+  @mock.patch('datalab.utils._http.Http.http.request')
   def test_raises_http_error(self, mock_request, mock_response):
     TestCases._setup_mocks(mock_request, mock_response, 'Not Found', 404)