diff --git a/.travis.yml b/.travis.yml index c783e0b..c41e2f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ matrix: - python: 3.4 - python: 3.5 - python: 3.6 + - python: 3.7 install: # Install dependencies @@ -39,5 +40,5 @@ deploy: distributions: "bdist_wheel sdist" # Parameter order for distributions is important. on: tags: true - python: '3.6' + python: '3.7' repo: Azure/azure-data-lake-store-python \ No newline at end of file diff --git a/HISTORY.rst b/HISTORY.rst index 89f88dd..df4b725 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,11 @@ Release History =============== +0.0.48 (2019-10-18) ++++++++++++++++++++ +* Buffer writes to prevent out of memory problems +* Add Python 3.7 in travis configuration + 0.0.47 (2019-08-14) +++++++++++++++++++ * Remove logging of bearer token diff --git a/Readme.md b/Readme.md index 10d7034..748f204 100644 --- a/Readme.md +++ b/Readme.md @@ -124,6 +124,8 @@ ADLDownloader(adl, '', 'my_temp_dir', 5, 2**24) ``` # API +<<<<<<< HEAD +======= | | | @@ -173,6 +175,7 @@ ADLDownloader(adl, '', 'my_temp_dir', 5, 2**24) | | +>>>>>>> master #### class azure.datalake.store.core.AzureDLFileSystem(token=None, per_call_timeout_seconds=60, \*\*kwargs) Access Azure DataLake Store as if it were a file-system @@ -211,6 +214,28 @@ Access Azure DataLake Store as if it were a file-system ### Methods +<<<<<<< HEAD + + + + + + + + + + + + + + + + + + + + +======= | | | @@ -251,6 +276,7 @@ Access Azure DataLake Store as if it were a file-system | | | +>>>>>>> master #### access(self, path, invalidate_cache=True) @@ -1235,12 +1261,17 @@ of files or a glob pattern. ### Methods +<<<<<<< HEAD + + +======= | | | | | | +>>>>>>> master #### active(self) @@ -1397,12 +1428,18 @@ of files or a glob pattern. ### Methods +<<<<<<< HEAD + + + +======= | | | | | | +>>>>>>> master #### active(self) diff --git a/azure/datalake/store/__init__.py b/azure/datalake/store/__init__.py index 500b880..bda664f 100644 --- a/azure/datalake/store/__init__.py +++ b/azure/datalake/store/__init__.py @@ -6,7 +6,7 @@ # license information. # -------------------------------------------------------------------------- -__version__ = "0.0.47" +__version__ = "0.0.48" from .core import AzureDLFileSystem from .multithread import ADLDownloader diff --git a/azure/datalake/store/core.py b/azure/datalake/store/core.py index 77581f0..6a262e1 100644 --- a/azure/datalake/store/core.py +++ b/azure/datalake/store/core.py @@ -1213,10 +1213,15 @@ def write(self, data): if self.closed: raise ValueError('I/O operation on closed file.') - out = self.buffer.write(ensure_writable(data)) - self.loc += out - self.flush(syncFlag='DATA') - return out + # TODO Flush may be simplified + # Buffered writes so a very large buffer is not copied leading to very large memory consumption + bytes_written = 0 + for i in range(0, len(data), self.blocksize): + out = self.buffer.write(ensure_writable(data[i:i + self.blocksize])) + self.loc += out + bytes_written += out + self.flush(syncFlag='DATA') + return bytes_written def flush(self, syncFlag='METADATA', force=False): """