Add System.Text.Rune support

alexanderkozlenko · Dec 2, 2024 · 7284a82 · 7284a82
1 parent aa03d61
commit 7284a82
Show file tree

Hide file tree

Showing 12 changed files with 181 additions and 59 deletions.
diff --git a/src/Addax.Formats.Tabular.Analyzers.CSharp.UnitTests/Assets/TypeMemberHasSupportedType.cs b/src/Addax.Formats.Tabular.Analyzers.CSharp.UnitTests/Assets/TypeMemberHasSupportedType.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Numerics;
+using System.Text;
 using Addax.Formats.Tabular;
 
 namespace MyNamespace;
@@ -81,4 +82,7 @@ public struct MyType
 
     [TabularFieldOrder(24)]
     public Uri? MyValue24;
+
+    [TabularFieldOrder(25)]
+    public Rune MyValue25;
 }
diff --git a/...ormats.Tabular.Analyzers.CSharp.UnitTests/Assets/TypeMemberHasSupportedTypeAsNullableT.cs b/...ormats.Tabular.Analyzers.CSharp.UnitTests/Assets/TypeMemberHasSupportedTypeAsNullableT.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Numerics;
+using System.Text;
 using Addax.Formats.Tabular;
 
 namespace MyNamespace;
@@ -81,4 +82,7 @@ public struct MyType
 
     [TabularFieldOrder(24)]
     public Uri? MyValue24;
+
+    [TabularFieldOrder(25)]
+    public Rune? MyValue25;
 }
diff --git a/src/Addax.Formats.Tabular.Analyzers.CSharp/TabularHandlerEmitter.cs b/src/Addax.Formats.Tabular.Analyzers.CSharp/TabularHandlerEmitter.cs
@@ -223,6 +223,10 @@ private static string GetValueTypeCode(string valueTypeName)
                     {
                         return "BigInteger";
                     }
+                case "global::System.Text.Rune":
+                    {
+                        return "Rune";
+                    }
                 default:
                     {
                         throw new NotSupportedException();

diff --git a/src/Addax.Formats.Tabular.Analyzers.CSharp/TabularHandlerParser.cs b/src/Addax.Formats.Tabular.Analyzers.CSharp/TabularHandlerParser.cs
@@ -415,7 +415,8 @@ private static (ITypeSymbol Type, string Name, bool IsSupported, bool IsNullable
                             (valueTypeName == "global::System.TimeSpan") ||
                             (valueTypeName == "global::System.UInt128") ||
                             (valueTypeName == "global::System.Uri") ||
-                            (valueTypeName == "global::System.Numerics.BigInteger");
+                            (valueTypeName == "global::System.Numerics.BigInteger") ||
+                            (valueTypeName == "global::System.Text.Rune");
                     }
                     break;
             }

diff --git a/src/Addax.Formats.Tabular.UnitTests/TabularReaderTests.T.cs b/src/Addax.Formats.Tabular.UnitTests/TabularReaderTests.T.cs
@@ -13,10 +13,9 @@ public partial class TabularReaderTests
     private static void TryGet<T>(Func<TabularReader, TryGetFunc<T>> selector, string content, T? expected)
     {
         var dialect = new TabularDialect("\u000a", '\u000b', '\u000c', '\u000d');
-        var options = new TabularOptions { Encoding = Encoding.ASCII };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
-        using var reader = new TabularReader(stream, dialect, options);
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
+        using var reader = new TabularReader(stream, dialect);
 
         var method = selector.Invoke(reader);
 
@@ -37,10 +36,9 @@ private static void TryGet<T>(Func<TabularReader, TryGetFunc<T>> selector, strin
     private static void Get<T>(Func<TabularReader, Func<T?>> selector, string content, T? expected)
     {
         var dialect = new TabularDialect("\u000a", '\u000b', '\u000c', '\u000d');
-        var options = new TabularOptions { Encoding = Encoding.ASCII };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
-        using var reader = new TabularReader(stream, dialect, options);
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
+        using var reader = new TabularReader(stream, dialect);
 
         var method = selector.Invoke(reader);
 
@@ -336,6 +334,14 @@ public void TryGetBase64Binary(string content, string expected)
         TryGet(x => x.TryGetBase64Binary, content, Convert.FromBase64String(expected));
     }
 
+    [TestMethod]
+    [DataRow("v", "v")]
+    [DataRow("\ud83d\udd2e", "\ud83d\udd2e")]
+    public void TryGetRune(string content, string expected)
+    {
+        TryGet(x => x.TryGetRune, content, Rune.GetRuneAt(expected, 0));
+    }
+
     [TestMethod]
     [DataRow("", "")]
     [DataRow(" v ", " v ")]
@@ -613,4 +619,12 @@ public void GetBase64Binary(string content, string expected)
     {
         Get(x => x.GetBase64Binary, content, Convert.FromBase64String(expected));
     }
+
+    [TestMethod]
+    [DataRow("v", "v")]
+    [DataRow("\ud83d\udd2e", "\ud83d\udd2e")]
+    public void GetRune(string content, string expected)
+    {
+        Get(x => x.GetRune, content, Rune.GetRuneAt(expected, 0));
+    }
 }
diff --git a/src/Addax.Formats.Tabular.UnitTests/TabularReaderTests.cs b/src/Addax.Formats.Tabular.UnitTests/TabularReaderTests.cs
@@ -126,9 +126,9 @@ public sealed partial class TabularReaderTests
     public void ConsumePositive(string dialectScript, string content, string structureScript)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         var structureBuilder = new StringBuilder();
@@ -227,9 +227,9 @@ public void ConsumePositive(string dialectScript, string content, string structu
     public void ConsumeNegative(string dialectScript, string content)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         Assert.ThrowsException<TabularContentException>(() => ReadStream(reader));
@@ -253,9 +253,9 @@ static void ReadStream(TabularReader reader)
     public void TryPickRecordIfStreamStart(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         Assert.AreEqual(expected, reader.TryPickRecord());
@@ -269,9 +269,9 @@ public void TryPickRecordIfStreamStart(string dialectScript, string content, boo
     public void TryPickRecordIfRecordStart(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         reader.TryPickRecord();
@@ -287,9 +287,9 @@ public void TryPickRecordIfRecordStart(string dialectScript, string content, boo
     public void TryPickRecordIfDelimiter(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         reader.TryReadField();
@@ -305,9 +305,9 @@ public void TryPickRecordIfDelimiter(string dialectScript, string content, bool
     public void TryPickRecordIfRecordEnd(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         reader.TryReadField();
@@ -323,9 +323,9 @@ public void TryPickRecordIfRecordEnd(string dialectScript, string content, bool
     public void TryPickRecordIfStreamEnd(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         reader.TryReadField();
@@ -341,9 +341,9 @@ public void TryPickRecordIfStreamEnd(string dialectScript, string content, bool
     public void TryReadFieldIfStreamStart(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         Assert.AreEqual(expected, reader.TryReadField());
@@ -357,9 +357,9 @@ public void TryReadFieldIfStreamStart(string dialectScript, string content, bool
     public void TryReadFieldIfRecordStart(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         reader.TryPickRecord();
@@ -375,9 +375,9 @@ public void TryReadFieldIfRecordStart(string dialectScript, string content, bool
     public void TryReadFieldIfDelimiter(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         reader.TryReadField();
@@ -393,9 +393,9 @@ public void TryReadFieldIfDelimiter(string dialectScript, string content, bool e
     public void TryReadFieldIfRecordEnd(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         reader.TryReadField();
@@ -411,9 +411,9 @@ public void TryReadFieldIfRecordEnd(string dialectScript, string content, bool e
     public void TryReadFieldIfStreamEnd(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         reader.TryReadField();
@@ -429,9 +429,9 @@ public void TryReadFieldIfStreamEnd(string dialectScript, string content, bool e
     public void TrySkipFieldIfStreamStart(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         Assert.AreEqual(expected, reader.TrySkipField());
@@ -445,9 +445,9 @@ public void TrySkipFieldIfStreamStart(string dialectScript, string content, bool
     public void TrySkipFieldIfRecordStart(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         reader.TryPickRecord();
@@ -463,9 +463,9 @@ public void TrySkipFieldIfRecordStart(string dialectScript, string content, bool
     public void TrySkipFieldIfDelimiter(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         reader.TryReadField();
@@ -481,9 +481,9 @@ public void TrySkipFieldIfDelimiter(string dialectScript, string content, bool e
     public void TrySkipFieldIfRecordEnd(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         reader.TryReadField();
@@ -499,9 +499,9 @@ public void TrySkipFieldIfRecordEnd(string dialectScript, string content, bool e
     public void TrySkipFieldIfStreamEnd(string dialectScript, string content, bool expected)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader(stream, dialect, options);
 
         reader.TryReadField();
@@ -521,9 +521,9 @@ public void TrySkipFieldIfStreamEnd(string dialectScript, string content, bool e
     public void TryReadRecord(string dialectScript, string content)
     {
         var dialect = CreateDialect(dialectScript);
-        var options = new TabularOptions { Encoding = Encoding.ASCII, BufferSize = 1 };
+        var options = new TabularOptions { BufferSize = 1 };
 
-        using var stream = new MemoryStream(Encoding.ASCII.GetBytes(content));
+        using var stream = new MemoryStream(Encoding.UTF8.GetBytes(content));
         using var reader = new TabularReader<string?[]>(stream, dialect, options);
 
         Assert.IsTrue(reader.TrySkipRecord());

diff --git a/src/Addax.Formats.Tabular.UnitTests/TabularWriterTests.T.cs b/src/Addax.Formats.Tabular.UnitTests/TabularWriterTests.T.cs
@@ -12,17 +12,16 @@ public partial class TabularWriterTests
     private static void Write<T>(Func<TabularWriter, WriteValue<T>> selector, T value, string expected)
     {
         var dialect = new TabularDialect("\u000a", '\u000b', '\u000c', '\u000d');
-        var options = new TabularOptions { Encoding = Encoding.ASCII };
 
         using var stream = new MemoryStream();
-        using var writer = new TabularWriter(stream, dialect, options);
+        using var writer = new TabularWriter(stream, dialect);
 
         var method = selector.Invoke(writer);
 
         method.Invoke(value);
         writer.Flush();
 
-        Assert.AreEqual(expected, Encoding.ASCII.GetString(stream.ToArray()));
+        Assert.AreEqual(expected, Encoding.UTF8.GetString(stream.ToArray()));
     }
 
     [TestMethod]
@@ -265,6 +264,14 @@ public void WriteGuid(string value, string expected)
         Write(x => x.WriteGuid, Guid.Parse(value, CultureInfo.InvariantCulture), expected);
     }
 
+    [TestMethod]
+    [DataRow("urn:com.example", "urn:com.example")]
+    [DataRow("https://example.com/", "https://example.com/")]
+    public void WriteUri(string value, string expected)
+    {
+        Write(x => x.WriteUri, new Uri(value, UriKind.RelativeOrAbsolute), expected);
+    }
+
     [TestMethod]
     [DataRow("", "")]
     [DataRow("123e4567e89b12d3a456426614174000", "123e4567e89b12d3a456426614174000")]
@@ -282,10 +289,10 @@ public void WriteBase64Binary(string value, string expected)
     }
 
     [TestMethod]
-    [DataRow("urn:com.example", "urn:com.example")]
-    [DataRow("https://example.com/", "https://example.com/")]
-    public void WriteUri(string value, string expected)
+    [DataRow("v", "v")]
+    [DataRow("\ud83d\udd2e", "\ud83d\udd2e")]
+    public void WriteRune(string value, string expected)
     {
-        Write(x => x.WriteUri, new Uri(value, UriKind.RelativeOrAbsolute), expected);
+        Write(x => x.WriteRune, Rune.GetRuneAt(value, 0), expected);
     }
 }