Release 0.0.48 version.

.travis.yml

@@ -10,6 +10,7 @@ matrix:
     - python: 3.4
     - python: 3.5
     - python: 3.6
+    - python: 3.7
 
 install:
   # Install dependencies
@@ -39,5 +40,5 @@ deploy:
   distributions: "bdist_wheel sdist"   # Parameter order for distributions is important.
   on:
     tags: true
-    python: '3.6'
+    python: '3.7'
     repo: Azure/azure-data-lake-store-python

HISTORY.rst

@@ -3,6 +3,11 @@
 Release History
 ===============
 
+0.0.48 (2019-10-18)
++++++++++++++++++++
+* Buffer writes to prevent out of memory problems
+* Add Python 3.7 in travis configuration
+
 0.0.47 (2019-08-14)
 +++++++++++++++++++
 * Remove logging of bearer token

Readme.md

@@ -124,6 +124,8 @@ ADLDownloader(adl, '', 'my_temp_dir', 5, 2**24)
 ```
 # API
 
+<<<<<<< HEAD
+=======
 |
 |
 |
@@ -173,6 +175,7 @@ ADLDownloader(adl, '', 'my_temp_dir', 5, 2**24)
 |
 |
 
+>>>>>>> master
 #### class azure.datalake.store.core.AzureDLFileSystem(token=None, per_call_timeout_seconds=60, \*\*kwargs)
 Access Azure DataLake Store as if it were a file-system
 
@@ -211,6 +214,28 @@ Access Azure DataLake Store as if it were a file-system
 
 ### Methods
 
+<<<<<<< HEAD
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+=======
 |
 |
 |
@@ -251,6 +276,7 @@ Access Azure DataLake Store as if it were a file-system
 |
 |
 |
+>>>>>>> master
 <!-- !! processed by numpydoc !! -->
 
 #### access(self, path, invalidate_cache=True)
@@ -1235,12 +1261,17 @@ of files or a glob pattern.
 
 ### Methods
 
+<<<<<<< HEAD
+
+
+=======
 |
 |
 |
 |
 |
 |
+>>>>>>> master
 <!-- !! processed by numpydoc !! -->
 
 #### active(self)
@@ -1397,12 +1428,18 @@ of files or a glob pattern.
 
 ### Methods
 
+<<<<<<< HEAD
+
+
+
+=======
 |
 |
 |
 |
 |
 |
+>>>>>>> master
 <!-- !! processed by numpydoc !! -->
 
 #### active(self)

azure/datalake/store/__init__.py

@@ -6,7 +6,7 @@
 # license information.
 # --------------------------------------------------------------------------
 
-__version__ = "0.0.47"
+__version__ = "0.0.48"
 
 from .core import AzureDLFileSystem
 from .multithread import ADLDownloader

azure/datalake/store/core.py

@@ -1213,10 +1213,15 @@ def write(self, data):
         if self.closed:
             raise ValueError('I/O operation on closed file.')
 
-        out = self.buffer.write(ensure_writable(data))
-        self.loc += out
-        self.flush(syncFlag='DATA')
-        return out
+        # TODO Flush may be simplified
+        # Buffered writes so a very large buffer is not copied leading to very large memory consumption
+        bytes_written = 0
+        for i in range(0, len(data), self.blocksize):
+            out = self.buffer.write(ensure_writable(data[i:i + self.blocksize]))
+            self.loc += out
+            bytes_written += out
+            self.flush(syncFlag='DATA')
+        return bytes_written
 
     def flush(self, syncFlag='METADATA', force=False):
         """