Fsspec buffered reading (#156)

Kimahriman · Nov 6, 2024 · af34ed5 · af34ed5
1 parent ffd719a
commit af34ed5
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 1 deletion.
diff --git a/python/hdfs_native/fsspec.py b/python/hdfs_native/fsspec.py
@@ -1,3 +1,4 @@
+import io
 import secrets
 import shutil
 import time
@@ -165,7 +166,10 @@ def _open(
     ):
         path = self._strip_protocol(path)
         if mode == "rb":
-            return self.client.read(path)
+            reader = self.client.read(path)
+            if not block_size:
+                return reader
+            return io.BufferedReader(reader, buffer_size=block_size)
         elif mode == "wb":
             write_options = WriteOptions()
             write_options.overwrite = overwrite

diff --git a/python/tests/test_fsspec.py b/python/tests/test_fsspec.py
@@ -37,6 +37,10 @@ def test_io(fs: HdfsFileSystem):
         data = file.read()
         assert data == b"hello there"
 
+    with fs.open("/test", mode="rb", block_size=1024) as file:
+        data = file.read()
+        assert data == b"hello there"
+
     fs.write_bytes("/test2", b"hello again")
     assert fs.read_bytes("/test2") == b"hello again"
     assert fs.read_bytes("/test2", start=1) == b"ello again"