Use 'Upload.ConfigureRequest' to decide 'simple' vs. 'resumable'.

Author: tseaver

gcloud/storage/key.py

@@ -6,10 +6,8 @@
 from StringIO import StringIO
 import urllib
 
-from _gcloud_vendor.apitools.base.py.http_wrapper import Request
-from _gcloud_vendor.apitools.base.py.transfer import Upload
-from _gcloud_vendor.apitools.base.py.transfer import Download
-from _gcloud_vendor.apitools.base.py.transfer import _RESUMABLE_UPLOAD
+from _gcloud_vendor.apitools.base.py import http_wrapper
+from _gcloud_vendor.apitools.base.py import transfer
 
 from gcloud.storage._helpers import _PropertyMixin
 from gcloud.storage._helpers import _scalar_property
@@ -212,12 +210,12 @@ def download_to_file(self, file_obj):
         :raises: :class:`gcloud.storage.exceptions.NotFound`
         """
         # Use apitools 'Download' facility.
-        download = Download.FromStream(file_obj, auto_transfer=False)
+        download = transfer.Download.FromStream(file_obj, auto_transfer=False)
         download.chunksize = self.CHUNK_SIZE
         download_url = self.connection.build_api_url(
             path=self.path, query_params={'alt': 'media'})
         headers = {'Range': 'bytes=0-%d' % (self.CHUNK_SIZE - 1)}
-        request = Request(download_url, 'POST', headers)
+        request = http_wrapper.Request(download_url, 'POST', headers)
 
         download.InitializeDownload(request, self.connection.http)
 
@@ -259,7 +257,7 @@ def download_as_string(self):
     get_contents_as_string = download_as_string
 
     def upload_from_file(self, file_obj, rewind=False, size=None,
-                         content_type=None):
+                         content_type=None, num_retries=6):
         """Upload the contents of this key from a file-like object.
 
         .. note::
@@ -292,41 +290,43 @@ def upload_from_file(self, file_obj, rewind=False, size=None,
 
         # Get the basic stats about the file.
         total_bytes = size or os.fstat(file_obj.fileno()).st_size
+        conn = self.connection
         headers = {
-            # Base headers
             'Accept': 'application/json',
             'Accept-Encoding': 'gzip, deflate',
-            'User-Agent': self.connection.USER_AGENT,
-            # resumable upload headers.
-            'X-Upload-Content-Type': content_type or 'application/unknown',
-            'X-Upload-Content-Length': total_bytes,
+            'User-Agent': conn.USER_AGENT,
         }
 
-        query_params = {
-            'uploadType': 'resumable',
-            'name': self.name,
-        }
+        upload = transfer.Upload(file_obj,
+                                 content_type or 'application/unknown',
+                                 total_bytes, auto_transfer=False,
+                                 chunksize=self.CHUNK_SIZE)
 
-        upload_url = self.connection.build_api_url(
-            path=self.bucket.path + '/o',
-            query_params=query_params,
-            api_base_url=self.connection.API_BASE_URL + '/upload')
+        url_builder = _UrlBuilder(bucket_name=self.bucket.name,
+                                  object_name=self.name)
+        upload_config = _UploadConfig()
 
-        # Use apitools 'Upload' facility.
-        request = Request(upload_url, 'POST', headers)
+        # Temporary URL, until we know simple vs. resumable.
+        upload_url = conn.build_api_url(
+            path=self.bucket.path + '/o')
 
-        upload = Upload(file_obj, content_type or 'application/unknown',
-                        total_bytes, auto_transfer=False,
-                        chunksize=self.CHUNK_SIZE)
-        upload.strategy = _RESUMABLE_UPLOAD
+        # Use apitools 'Upload' facility.
+        request = http_wrapper.Request(upload_url, 'POST', headers)
 
-        upload.InitializeUpload(request, self.connection.http)
+        upload.ConfigureRequest(upload_config, request, url_builder)
+        path = url_builder.relative_path.format(bucket=self.bucket.name)
+        query_params = url_builder.query_params
+        request.url = conn.build_api_url(path=path, query_params=query_params)
+        upload.InitializeUpload(request, conn.http)
 
         # Should we be passing callbacks through from caller?  We can't
         # pass them as None, because apitools wants to print to the console
         # by default.
-        upload.StreamInChunks(callback=lambda *args: None,
-                              finish_callback=lambda *args: None)
+        if upload.strategy == transfer._RESUMABLE_UPLOAD:
+            upload.StreamInChunks(callback=lambda *args: None,
+                                  finish_callback=lambda *args: None)
+        else:
+            http_wrapper.MakeRequest(conn.http, request, retries=num_retries)
 
     # NOTE: Alias for boto-like API.
     set_contents_from_file = upload_from_file
@@ -606,3 +606,34 @@ def updated(self):
         :returns: timestamp in RFC 3339 format.
         """
         return self.properties['updated']
+
+
+class _UploadConfig(object):
+    """ Faux message FBO apitools' 'ConfigureRequest'.
+
+    Values extracted from apitools
+    'samples/storage_sample/storage/storage_v1_client.py'
+    """
+    accept = ['*/*']
+    max_size = None
+    resumable_multipart = True
+    resumable_path = u'/resumable/upload/storage/v1/b/{bucket}/o'
+    simple_multipart = True
+    simple_path = u'/upload/storage/v1/b/{bucket}/o'
+
+
+class _UrlBuilder(object):
+    """Faux builder FBO apitools' 'ConfigureRequest'
+    """
+    def __init__(self, bucket_name, object_name):
+        self.query_params = {'name': object_name}
+        self._bucket_name = bucket_name
+        self._relative_path = ''
+
+    @property
+    def relative_path(self):
+        return self._relative_path.format(bucket=self._bucket_name)
+
+    @relative_path.setter
+    def relative_path(self, value):
+        self._relative_path = value

gcloud/storage/test_key.py

@@ -236,12 +236,47 @@ def test_download_as_string(self):
         fetched = key.download_as_string()
         self.assertEqual(fetched, 'abcdef')
 
-    def test_upload_from_file(self):
+    def test_upload_from_file_simple(self):
         import httplib
         from tempfile import NamedTemporaryFile
         from urlparse import parse_qsl
         from urlparse import urlsplit
+        KEY = 'key'
+        DATA = 'ABCDEF'
+        response = {'status': httplib.OK}
+        connection = _Connection(
+            (response, ''),
+        )
+        bucket = _Bucket(connection)
+        key = self._makeOne(bucket, KEY)
+        key.CHUNK_SIZE = 5
+        with NamedTemporaryFile() as fh:
+            fh.write(DATA)
+            fh.flush()
+            key.upload_from_file(fh, rewind=True)
+        rq = connection.http._requested
+        self.assertEqual(len(rq), 1)
+        self.assertEqual(rq[0]['method'], 'POST')
+        uri = rq[0]['uri']
+        scheme, netloc, path, qs, _ = urlsplit(uri)
+        self.assertEqual(scheme, 'http')
+        self.assertEqual(netloc, 'example.com')
+        self.assertEqual(path, '/upload/storage/v1/b/name/o')
+        self.assertEqual(dict(parse_qsl(qs)),
+                         {'uploadType': 'media', 'name': 'key'})
+        headers = dict(
+            [(x.title(), str(y)) for x, y in rq[0]['headers'].items()])
+        self.assertEqual(headers['Content-Length'], '6')
+        self.assertEqual(headers['Content-Type'], 'application/unknown')
+
+    def test_upload_from_file_resumable(self):
+        import httplib
+        from tempfile import NamedTemporaryFile
+        from urlparse import parse_qsl
+        from urlparse import urlsplit
+        from gcloud._testing import _Monkey
         from _gcloud_vendor.apitools.base.py import http_wrapper
+        from _gcloud_vendor.apitools.base.py import transfer
         KEY = 'key'
         UPLOAD_URL = 'http://example.com/upload/name/key'
         DATA = 'ABCDEF'
@@ -257,18 +292,20 @@ def test_upload_from_file(self):
         bucket = _Bucket(connection)
         key = self._makeOne(bucket, KEY)
         key.CHUNK_SIZE = 5
-        with NamedTemporaryFile() as fh:
-            fh.write(DATA)
-            fh.flush()
-            key.upload_from_file(fh, rewind=True)
+        # Set the threshhold low enough that we force a resumable uploada.
+        with _Monkey(transfer, _RESUMABLE_UPLOAD_THRESHOLD=5):
+            with NamedTemporaryFile() as fh:
+                fh.write(DATA)
+                fh.flush()
+                key.upload_from_file(fh, rewind=True)
         rq = connection.http._requested
         self.assertEqual(len(rq), 3)
         self.assertEqual(rq[0]['method'], 'POST')
         uri = rq[0]['uri']
         scheme, netloc, path, qs, _ = urlsplit(uri)
         self.assertEqual(scheme, 'http')
         self.assertEqual(netloc, 'example.com')
-        self.assertEqual(path, '/b/name/o')
+        self.assertEqual(path, '/resumable/upload/storage/v1/b/name/o')
         self.assertEqual(dict(parse_qsl(qs)),
                          {'uploadType': 'resumable', 'name': 'key'})
         headers = dict(
@@ -317,32 +354,19 @@ def test_upload_from_file_w_slash_in_name(self):
             fh.flush()
             key.upload_from_file(fh, rewind=True)
         rq = connection.http._requested
-        self.assertEqual(len(rq), 3)
+        self.assertEqual(len(rq), 1)
         self.assertEqual(rq[0]['method'], 'POST')
         uri = rq[0]['uri']
         scheme, netloc, path, qs, _ = urlsplit(uri)
         self.assertEqual(scheme, 'http')
         self.assertEqual(netloc, 'example.com')
-        self.assertEqual(path, '/b/name/o')
+        self.assertEqual(path, '/upload/storage/v1/b/name/o')
         self.assertEqual(dict(parse_qsl(qs)),
-                         {'uploadType': 'resumable', 'name': 'parent/child'})
+                         {'uploadType': 'media', 'name': 'parent/child'})
         headers = dict(
             [(x.title(), str(y)) for x, y in rq[0]['headers'].items()])
-        self.assertEqual(headers['X-Upload-Content-Length'], '6')
-        self.assertEqual(headers['X-Upload-Content-Type'],
-                         'application/unknown')
-        self.assertEqual(rq[1]['method'], 'PUT')
-        self.assertEqual(rq[1]['uri'], UPLOAD_URL)
-        self.assertEqual(rq[1]['body'], DATA[:5])
-        headers = dict(
-            [(x.title(), str(y)) for x, y in rq[1]['headers'].items()])
-        self.assertEqual(headers['Content-Range'], 'bytes 0-4/6')
-        self.assertEqual(rq[2]['method'], 'PUT')
-        self.assertEqual(rq[2]['uri'], UPLOAD_URL)
-        self.assertEqual(rq[2]['body'], DATA[5:])
-        headers = dict(
-            [(x.title(), str(y)) for x, y in rq[2]['headers'].items()])
-        self.assertEqual(headers['Content-Range'], 'bytes 5-5/6')
+        self.assertEqual(headers['Content-Length'], '6')
+        self.assertEqual(headers['Content-Type'], 'application/unknown')
 
     def test_upload_from_filename(self):
         import httplib
@@ -370,32 +394,19 @@ def test_upload_from_filename(self):
             fh.flush()
             key.upload_from_filename(fh.name)
         rq = connection.http._requested
-        self.assertEqual(len(rq), 3)
+        self.assertEqual(len(rq), 1)
         self.assertEqual(rq[0]['method'], 'POST')
         uri = rq[0]['uri']
         scheme, netloc, path, qs, _ = urlsplit(uri)
         self.assertEqual(scheme, 'http')
         self.assertEqual(netloc, 'example.com')
-        self.assertEqual(path, '/b/name/o')
+        self.assertEqual(path, '/upload/storage/v1/b/name/o')
         self.assertEqual(dict(parse_qsl(qs)),
-                         {'uploadType': 'resumable', 'name': 'key'})
+                         {'uploadType': 'media', 'name': 'key'})
         headers = dict(
             [(x.title(), str(y)) for x, y in rq[0]['headers'].items()])
-        self.assertEqual(headers['X-Upload-Content-Length'], '6')
-        self.assertEqual(headers['X-Upload-Content-Type'],
-                         'image/jpeg')
-        self.assertEqual(rq[1]['method'], 'PUT')
-        self.assertEqual(rq[1]['uri'], UPLOAD_URL)
-        self.assertEqual(rq[1]['body'], DATA[:5])
-        headers = dict(
-            [(x.title(), str(y)) for x, y in rq[1]['headers'].items()])
-        self.assertEqual(headers['Content-Range'], 'bytes 0-4/6')
-        self.assertEqual(rq[2]['method'], 'PUT')
-        self.assertEqual(rq[2]['uri'], UPLOAD_URL)
-        self.assertEqual(rq[2]['body'], DATA[5:])
-        headers = dict(
-            [(x.title(), str(y)) for x, y in rq[2]['headers'].items()])
-        self.assertEqual(headers['Content-Range'], 'bytes 5-5/6')
+        self.assertEqual(headers['Content-Length'], '6')
+        self.assertEqual(headers['Content-Type'], 'image/jpeg')
 
     def test_upload_from_string(self):
         import httplib
@@ -419,32 +430,19 @@ def test_upload_from_string(self):
         key.CHUNK_SIZE = 5
         key.upload_from_string(DATA)
         rq = connection.http._requested
-        self.assertEqual(len(rq), 3)
+        self.assertEqual(len(rq), 1)
         self.assertEqual(rq[0]['method'], 'POST')
         uri = rq[0]['uri']
         scheme, netloc, path, qs, _ = urlsplit(uri)
         self.assertEqual(scheme, 'http')
         self.assertEqual(netloc, 'example.com')
-        self.assertEqual(path, '/b/name/o')
+        self.assertEqual(path, '/upload/storage/v1/b/name/o')
         self.assertEqual(dict(parse_qsl(qs)),
-                         {'uploadType': 'resumable', 'name': 'key'})
+                         {'uploadType': 'media', 'name': 'key'})
         headers = dict(
             [(x.title(), str(y)) for x, y in rq[0]['headers'].items()])
-        self.assertEqual(headers['X-Upload-Content-Length'], '6')
-        self.assertEqual(headers['X-Upload-Content-Type'],
-                         'text/plain')
-        self.assertEqual(rq[1]['method'], 'PUT')
-        self.assertEqual(rq[1]['uri'], UPLOAD_URL)
-        self.assertEqual(rq[1]['body'], DATA[:5])
-        headers = dict(
-            [(x.title(), str(y)) for x, y in rq[1]['headers'].items()])
-        self.assertEqual(headers['Content-Range'], 'bytes 0-4/6')
-        self.assertEqual(rq[2]['method'], 'PUT')
-        self.assertEqual(rq[2]['uri'], UPLOAD_URL)
-        self.assertEqual(rq[2]['body'], DATA[5:])
-        headers = dict(
-            [(x.title(), str(y)) for x, y in rq[2]['headers'].items()])
-        self.assertEqual(headers['Content-Range'], 'bytes 5-5/6')
+        self.assertEqual(headers['Content-Length'], '6')
+        self.assertEqual(headers['Content-Type'], 'text/plain')
 
     def test_make_public(self):
         from gcloud.storage.acl import _ACLEntity