From 942c5d05cab57de052630cda69a0a3ae28ae193a Mon Sep 17 00:00:00 2001 From: linar-jether Date: Mon, 27 Jun 2016 01:31:20 -0500 Subject: [PATCH] Support blob streaming for file-like objects When uploading file-like objects without a specified size, upload in chunks instead of exhausting the stream and keeping the data in-memory --- gcloud/storage/blob.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/gcloud/storage/blob.py b/gcloud/storage/blob.py index 6b1280a31289a..07c5cfca53e65 100644 --- a/gcloud/storage/blob.py +++ b/gcloud/storage/blob.py @@ -492,9 +492,7 @@ def upload_from_file(self, file_obj, rewind=False, size=None, if total_bytes is None: if hasattr(file_obj, 'fileno'): total_bytes = os.fstat(file_obj.fileno()).st_size - else: - raise ValueError('total bytes could not be determined. Please ' - 'pass an explicit size.') + headers = { 'Accept': 'application/json', 'Accept-Encoding': 'gzip, deflate', @@ -510,6 +508,12 @@ def upload_from_file(self, file_obj, rewind=False, size=None, if self.chunk_size is not None: upload.chunksize = self.chunk_size + if total_bytes is None: + upload.strategy = RESUMABLE_UPLOAD + elif self.chunk_size is None and total_bytes is None: + raise ValueError('total bytes could not be determined. Please ' + 'pass an explicit size, or supply a chunk size for a streaming transfer.') + url_builder = _UrlBuilder(bucket_name=self.bucket.name, object_name=self.name) upload_config = _UploadConfig() @@ -522,6 +526,7 @@ def upload_from_file(self, file_obj, rewind=False, size=None, # Use apitools 'Upload' facility. request = Request(upload_url, 'POST', headers) + upload.configure_request(upload_config, request, url_builder) query_params = url_builder.query_params base_url = connection.API_BASE_URL + '/upload'