Implement UTF-16 encoding/decoding routines

- Add identity tests - Add benchmarks - Add Word16 to Word8 helpers
composewell · Feb 8, 2024 · 1f6ac70 · 1f6ac70
1 parent d86db30
commit 1f6ac70
Show file tree

Hide file tree

Showing 4 changed files with 338 additions and 20 deletions.
diff --git a/benchmark/Streamly/Benchmark/Unicode/Stream.hs b/benchmark/Streamly/Benchmark/Unicode/Stream.hs
@@ -258,6 +258,17 @@ _copyStreamUtf8' inh outh =
      $ Unicode.decodeUtf8'
      $ Stream.unfold Handle.reader inh
 
+-- | Copy file
+{-# NOINLINE copyStreamUtf16 #-}
+copyStreamUtf16 :: Handle -> Handle -> IO ()
+copyStreamUtf16 inh outh =
+   Stream.fold (Handle.writeChunks outh)
+     $ Unicode.splitW16ToChunks
+     $ Unicode.encodeUtf16
+     $ Unicode.decodeUtf16
+     $ Unicode.scanW16FromChunks Unicode.LittleEndian Unicode.LittleEndian
+     $ Handle.readChunks inh
+
 #ifdef INSPECTION
 inspect $ hasNoTypeClasses '_copyStreamUtf8'
 -- inspect $ '_copyStreamUtf8 `hasNoType` ''Step
@@ -319,6 +330,8 @@ o_1_space_decode_encode_read env =
               $ \inh outh -> _copyStreamUtf8Parser inh outh
         , mkBenchSmall "encodeUtf8 . decodeUtf8" env $ \inh outh ->
             copyStreamUtf8 inh outh
+        , mkBenchSmall "encodeUtf16 . decodeUtf16" env $ \inh outh ->
+            copyStreamUtf16 inh outh
         ]
     ]
 

diff --git a/core/src/Streamly/Internal/FileSystem/PosixPath.hs b/core/src/Streamly/Internal/FileSystem/PosixPath.hs
@@ -6,8 +6,8 @@
 #define OS_NAME Windows
 #define OS_PATH WindowsPath
 #define WORD_TYPE Word16
-#define UNICODE_ENCODER encodeUtf16le'
-#define UNICODE_DECODER decodeUtf16le'
+#define UNICODE_ENCODER encodeUtf16'
+#define UNICODE_DECODER decodeUtf16'
 #define CODEC_NAME UTF-16LE
 #define SEPARATORS @/, \\@
 #else