diff --git a/.gitattributes b/.gitattributes index 1ff0c42..da9167f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,6 +2,9 @@ # Set default behavior to automatically normalize line endings. ############################################################################### * text=auto +# Bash shell script requires LF on POSIX (CRLF is also supported on MSYS2) +native/**/posix/*.sh text eol=lf +native/**/windows/*.sh text eol=lf ############################################################################### # Set default behavior for command prompt diff. diff --git a/.gitignore b/.gitignore index 82bf9a1..907cc78 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ # Joveler Custom res/ +native/**/posix/build*/ +native/**/windows/build*/ # User-specific files *.suo diff --git a/Benchmark/BenchProgram.cs b/Benchmark/BenchProgram.cs index de48488..8d898ba 100644 --- a/Benchmark/BenchProgram.cs +++ b/Benchmark/BenchProgram.cs @@ -1,16 +1,15 @@ -using BenchmarkDotNet.Running; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Running; using CommandLine; using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; -using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -// ReSharper disable InconsistentNaming namespace Benchmark { - #region Parameter + #region Parameters [Flags] public enum AlgorithmFlags { @@ -26,20 +25,6 @@ public abstract class ParamOptions { [Option("algo", Default = AlgorithmFlags.All, HelpText = "Choose algorithms to benchmark | zlib,xz,lz4,all")] public AlgorithmFlags Algorithms { get; set; } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public T Cast() where T : ParamOptions - { - T cast = this as T; - Debug.Assert(cast != null); - return cast; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static T Cast(ParamOptions opts) where T : ParamOptions - { - return opts.Cast(); - } } [Verb("all", HelpText = "Benchmark all")] @@ -51,6 +36,9 @@ public class CompBenchOptions : ParamOptions { } [Verb("decomp", HelpText = "Benchmark decompression")] public class DecompBenchOptions : ParamOptions { } + [Verb("xzmulti", HelpText = "Benchmark multithread options (XZ only)")] + public class XZMultiOptionBenchOptions : ParamOptions { } + [Verb("hash", HelpText = "Benchmark hash and checksums")] public class HashBenchOptions : ParamOptions { } @@ -61,6 +49,11 @@ public class BufferSizeBenchOptions : ParamOptions { } #region Program public static class Program { + #region Directories + public static string BaseDir => Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..")); + public static string SampleDir => Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..", "..", "..", "..", "Samples")); + #endregion + #region PrintErrorAndExit internal static void PrintErrorAndExit(IEnumerable errs) { @@ -76,8 +69,6 @@ public static void NativeGlobalInit() const string runtimes = "runtimes"; const string native = "native"; - string baseDir = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..", "..", "..", "..")); - string zlibPath = null; string xzPath = null; string lz4Path = null; @@ -86,9 +77,9 @@ public static void NativeGlobalInit() { string libDir = RuntimeInformation.ProcessArchitecture switch { - Architecture.X86 => Path.Combine(baseDir, runtimes, "win-x86", native), - Architecture.X64 => Path.Combine(baseDir, runtimes, "win-x64", native), - Architecture.Arm64 => Path.Combine(baseDir, runtimes, "win-arm64", native), + Architecture.X86 => Path.Combine(BaseDir, runtimes, "win-x86", native), + Architecture.X64 => Path.Combine(BaseDir, runtimes, "win-x64", native), + Architecture.Arm64 => Path.Combine(BaseDir, runtimes, "win-arm64", native), _ => throw new PlatformNotSupportedException(), }; zlibPath = Path.Combine(libDir, "zlibwapi.dll"); @@ -100,9 +91,9 @@ public static void NativeGlobalInit() { string libDir = RuntimeInformation.ProcessArchitecture switch { - Architecture.X64 => Path.Combine(baseDir, runtimes, "linux-x64", native), - Architecture.Arm => Path.Combine(baseDir, runtimes, "linux-arm", native), - Architecture.Arm64 => Path.Combine(baseDir, runtimes, "linux-arm64", native), + Architecture.X64 => Path.Combine(BaseDir, runtimes, "linux-x64", native), + Architecture.Arm => Path.Combine(BaseDir, runtimes, "linux-arm", native), + Architecture.Arm64 => Path.Combine(BaseDir, runtimes, "linux-arm64", native), _ => throw new PlatformNotSupportedException(), }; zlibPath = Path.Combine(libDir, "libz.so"); @@ -114,8 +105,8 @@ public static void NativeGlobalInit() { string libDir = RuntimeInformation.ProcessArchitecture switch { - Architecture.X64 => Path.Combine(baseDir, runtimes, "osx-x64", native), - Architecture.Arm64 => throw new PlatformNotSupportedException("TODO"), + Architecture.X64 => Path.Combine(BaseDir, runtimes, "osx-x64", native), + Architecture.Arm64 => Path.Combine(BaseDir, runtimes, "osx-arm64", native), _ => throw new PlatformNotSupportedException(), }; zlibPath = Path.Combine(libDir, "libz.dylib"); @@ -157,34 +148,47 @@ public static void Main(string[] args) }); argParser.ParseArguments(args) + CompBenchOptions, DecompBenchOptions, XZMultiOptionBenchOptions, HashBenchOptions, BufferSizeBenchOptions>(args) .WithParsed(x => Opts = x) .WithParsed(x => Opts = x) .WithParsed(x => Opts = x) + .WithParsed(x => Opts = x) .WithParsed(x => Opts = x) .WithParsed(x => Opts = x) .WithNotParsed(PrintErrorAndExit); Debug.Assert(Opts != null, $"{nameof(Opts)} != null"); + // InvertedTomato.Crc is the slowest, and ships unoptimized binaries. + // Disable for awhile to avoid BenchmarkDotNet's unoptimized run error. +#if INVERTEDTOMATO_CRC_EANBLE + ManualConfig config = DefaultConfig.Instance.WithOptions(ConfigOptions.DisableOptimizationsValidator); +#else + ManualConfig config = DefaultConfig.Instance.WithOptions(ConfigOptions.Default); +#endif + switch (Opts) { case AllBenchOptions _: - BenchmarkRunner.Run(); - BenchmarkRunner.Run(); - BenchmarkRunner.Run(); - BenchmarkRunner.Run(); + BenchmarkRunner.Run(config); + BenchmarkRunner.Run(config); + BenchmarkRunner.Run(config); + BenchmarkRunner.Run(config); + BenchmarkRunner.Run(config); break; case CompBenchOptions _: - BenchmarkRunner.Run(); + BenchmarkRunner.Run(config); break; case DecompBenchOptions _: - BenchmarkRunner.Run(); + BenchmarkRunner.Run(config); + break; + case XZMultiOptionBenchOptions _: + BenchmarkRunner.Run(config); break; case HashBenchOptions _: - BenchmarkRunner.Run(); + BenchmarkRunner.Run(config); break; case BufferSizeBenchOptions _: - BenchmarkRunner.Run(); + BenchmarkRunner.Run(config); break; } } diff --git a/Benchmark/Benchmark.csproj b/Benchmark/Benchmark.csproj index f60cdce..4010b3c 100644 --- a/Benchmark/Benchmark.csproj +++ b/Benchmark/Benchmark.csproj @@ -7,14 +7,13 @@ - + - - + - + diff --git a/Benchmark/CompBench.cs b/Benchmark/CompBench.cs index 672899a..f063c77 100644 --- a/Benchmark/CompBench.cs +++ b/Benchmark/CompBench.cs @@ -218,7 +218,7 @@ public double ZLib_Managed() [Benchmark] [BenchmarkCategory(BenchConfig.XZ)] - public double XZ_Native() + public double XZ_Native_Single() { long compLen; byte[] rawData = SrcFiles[SrcFileName]; @@ -244,6 +244,44 @@ public double XZ_Native() return CompRatio; } + [Benchmark] + [BenchmarkCategory(BenchConfig.XZ)] + public double XZ_Native_Multi() + { + // LZMA2 threaded compression with -9 option takes a lot of memory. + // To prevent memory starvation, skip threaded -9 compression. + if (Level.Equals("Best", StringComparison.OrdinalIgnoreCase)) + return 0; + + long compLen; + byte[] rawData = SrcFiles[SrcFileName]; + using (MemoryStream ms = new MemoryStream()) + { + Joveler.Compression.XZ.XZCompressOptions compOpts = new Joveler.Compression.XZ.XZCompressOptions + { + Level = XZLevelDict[Level], + LeaveOpen = true, + }; + + Joveler.Compression.XZ.XZThreadedCompressOptions threadOpts = new Joveler.Compression.XZ.XZThreadedCompressOptions + { + Threads = Environment.ProcessorCount, + }; + + using (MemoryStream rms = new MemoryStream(rawData)) + using (Joveler.Compression.XZ.XZStream xzs = new Joveler.Compression.XZ.XZStream(ms, compOpts, threadOpts)) + { + rms.CopyTo(xzs); + } + + ms.Flush(); + compLen = ms.Position; + } + + CompRatio = (double)compLen / rawData.Length; + return CompRatio; + } + [Benchmark] [BenchmarkCategory(BenchConfig.ZSTD)] public double ZSTD_Native() diff --git a/Benchmark/DecompBench.cs b/Benchmark/DecompBench.cs index df71e66..b96cdba 100644 --- a/Benchmark/DecompBench.cs +++ b/Benchmark/DecompBench.cs @@ -141,7 +141,7 @@ public long ZLib_Managed() [Benchmark] [BenchmarkCategory(BenchConfig.XZ)] - public long XZ_Native() + public long XZ_Native_Single() { byte[] compData = SrcFiles[$"{Level}_{SrcFileName}.xz"]; using MemoryStream ms = new MemoryStream(); @@ -156,6 +156,27 @@ public long XZ_Native() return ms.Length; } + [Benchmark] + [BenchmarkCategory(BenchConfig.XZ)] + public long XZ_Native_Multi() + { + byte[] compData = SrcFiles[$"{Level}_{SrcFileName}.xz"]; + using MemoryStream ms = new MemoryStream(); + Joveler.Compression.XZ.XZDecompressOptions decompOpts = new Joveler.Compression.XZ.XZDecompressOptions(); + Joveler.Compression.XZ.XZThreadedDecompressOptions threadOpts = new Joveler.Compression.XZ.XZThreadedDecompressOptions() + { + Threads = Environment.ProcessorCount, + }; + using (MemoryStream rms = new MemoryStream(compData)) + using (Joveler.Compression.XZ.XZStream zs = new Joveler.Compression.XZ.XZStream(rms, decompOpts, threadOpts)) + { + zs.CopyTo(ms); + } + + ms.Flush(); + return ms.Length; + } + [Benchmark] [BenchmarkCategory(BenchConfig.XZ)] public long XZ_Managed() diff --git a/Benchmark/HashBench.cs b/Benchmark/HashBench.cs index 845a78e..dfac0db 100644 --- a/Benchmark/HashBench.cs +++ b/Benchmark/HashBench.cs @@ -117,7 +117,7 @@ public uint CRC32_XZNative() } [Benchmark] - [BenchmarkCategory("ZLib", "XZ")] + [BenchmarkCategory(BenchConfig.ZLib, BenchConfig.XZ)] public byte[] CRC32_ForceManaged() { byte[] compData = SrcFiles[SrcFileName]; @@ -126,7 +126,7 @@ public byte[] CRC32_ForceManaged() } [Benchmark] - [BenchmarkCategory("ZLib", "XZ")] + [BenchmarkCategory(BenchConfig.ZLib, BenchConfig.XZ)] public uint CRC32_K4osManaged() { byte[] compData = SrcFiles[SrcFileName]; @@ -134,9 +134,12 @@ public uint CRC32_K4osManaged() crc32.Update(compData); return crc32.Digest(); } - + + // InvertedTomato.Crc is the slowest, and ships unoptimized binaries. + // Disable for awhile to avoid BenchmarkDotNet's unoptimized run error. +#if INVERTEDTOMATO_CRC_EANBLE [Benchmark] - [BenchmarkCategory("ZLib", "XZ")] + [BenchmarkCategory(BenchConfig.ZLib, BenchConfig.XZ)] public ulong CRC32_TomatoManaged() { byte[] compData = SrcFiles[SrcFileName]; @@ -144,6 +147,7 @@ public ulong CRC32_TomatoManaged() crc32.Append(compData); return crc32.Check; } +#endif #endregion #region CRC64 diff --git a/Benchmark/Samples/Best/Banner.bmp.xz b/Benchmark/Samples/Best/Banner.bmp.xz index 0e28d85..004b586 100644 Binary files a/Benchmark/Samples/Best/Banner.bmp.xz and b/Benchmark/Samples/Best/Banner.bmp.xz differ diff --git a/Benchmark/Samples/Best/Banner.svg.xz b/Benchmark/Samples/Best/Banner.svg.xz index cbb3ce9..9fe4b97 100644 Binary files a/Benchmark/Samples/Best/Banner.svg.xz and b/Benchmark/Samples/Best/Banner.svg.xz differ diff --git a/Benchmark/Samples/Best/Type4.txt.xz b/Benchmark/Samples/Best/Type4.txt.xz index 3f5c4cf..031db4e 100644 Binary files a/Benchmark/Samples/Best/Type4.txt.xz and b/Benchmark/Samples/Best/Type4.txt.xz differ diff --git a/Benchmark/Samples/Default/Banner.bmp.xz b/Benchmark/Samples/Default/Banner.bmp.xz index 9749d4e..f100816 100644 Binary files a/Benchmark/Samples/Default/Banner.bmp.xz and b/Benchmark/Samples/Default/Banner.bmp.xz differ diff --git a/Benchmark/Samples/Default/Banner.svg.xz b/Benchmark/Samples/Default/Banner.svg.xz index f4e5ee0..c925dc7 100644 Binary files a/Benchmark/Samples/Default/Banner.svg.xz and b/Benchmark/Samples/Default/Banner.svg.xz differ diff --git a/Benchmark/Samples/Default/Type4.txt.xz b/Benchmark/Samples/Default/Type4.txt.xz index deb312e..416011a 100644 Binary files a/Benchmark/Samples/Default/Type4.txt.xz and b/Benchmark/Samples/Default/Type4.txt.xz differ diff --git a/Benchmark/Samples/Fastest/Banner.bmp.xz b/Benchmark/Samples/Fastest/Banner.bmp.xz index 3d9521b..32a275e 100644 Binary files a/Benchmark/Samples/Fastest/Banner.bmp.xz and b/Benchmark/Samples/Fastest/Banner.bmp.xz differ diff --git a/Benchmark/Samples/Fastest/Banner.svg.xz b/Benchmark/Samples/Fastest/Banner.svg.xz index 4889924..893b7f4 100644 Binary files a/Benchmark/Samples/Fastest/Banner.svg.xz and b/Benchmark/Samples/Fastest/Banner.svg.xz differ diff --git a/Benchmark/Samples/Fastest/Type4.txt.xz b/Benchmark/Samples/Fastest/Type4.txt.xz index c21fcc3..5a05cc4 100644 Binary files a/Benchmark/Samples/Fastest/Type4.txt.xz and b/Benchmark/Samples/Fastest/Type4.txt.xz differ diff --git a/Benchmark/XZMultiOptionBench.cs b/Benchmark/XZMultiOptionBench.cs new file mode 100644 index 0000000..4ea2699 --- /dev/null +++ b/Benchmark/XZMultiOptionBench.cs @@ -0,0 +1,149 @@ +using BenchmarkDotNet.Attributes; +using System; +using System.Collections.Generic; +using System.IO; +using static System.Net.Mime.MediaTypeNames; + +namespace Benchmark +{ + #region XZMultiCompBench + [Config(typeof(BenchConfig))] + public class XZMultiOptionBench + { + private string _sampleDir; + private string _destDir; + + public double CompRatio { get; set; } + + // SrcFiles + [ParamsSource(nameof(SrcFileNames))] + public string SrcFileName { get; set; } + public IReadOnlyList SrcFileNames { get; set; } = new string[] + { + "Banner.bmp", + "Banner.svg", + "Type4.txt", + }; + public Dictionary SrcCompFiles = new Dictionary(StringComparer.Ordinal); + public Dictionary SrcDecompFiles = new Dictionary(StringComparer.Ordinal); + + // TimeOut (in milliseconds) + [ParamsSource(nameof(TimeOutValues))] + public uint TimeOutValue { get; set; } + public IReadOnlyList TimeOutValues { get; set; } = new uint[] + { + 0, + 100, + 300, + 1000, + }; + + [GlobalSetup] + public void GlobalSetup() + { + Program.NativeGlobalInit(); + + _sampleDir = Program.SampleDir; + + _destDir = Path.GetTempFileName(); + File.Delete(_destDir); + Directory.CreateDirectory(_destDir); + + foreach (string srcFileName in SrcFileNames) + { + // Compress + { + string srcFile = Path.Combine(_sampleDir, "Raw", srcFileName); + using MemoryStream ms = new MemoryStream(); + using (FileStream fs = new FileStream(srcFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + fs.CopyTo(ms); + } + + SrcCompFiles[srcFileName] = ms.ToArray(); + } + + // Decompress + { + string srcFile = Path.Combine(_sampleDir, "Default", $"{srcFileName}.xz"); + using MemoryStream ms = new MemoryStream(); + using (FileStream fs = new FileStream(srcFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + fs.CopyTo(ms); + } + + SrcDecompFiles[$"{srcFileName}.xz"] = ms.ToArray(); + } + + } + } + + [GlobalCleanup] + public void GlobalCleanup() + { + if (Directory.Exists(_destDir)) + Directory.Delete(_destDir); + Program.NativeGlobalCleanup(); + } + + [Benchmark] + [BenchmarkCategory(BenchConfig.XZ)] + public double XZ_Native_Comp() + { + long compLen; + byte[] rawData = SrcCompFiles[SrcFileName]; + using (MemoryStream ms = new MemoryStream()) + { + Joveler.Compression.XZ.XZCompressOptions compOpts = new Joveler.Compression.XZ.XZCompressOptions + { + Level = Joveler.Compression.XZ.LzmaCompLevel.Default, + LeaveOpen = true, + }; + + Joveler.Compression.XZ.XZThreadedCompressOptions threadOpts = new Joveler.Compression.XZ.XZThreadedCompressOptions + { + Threads = 2, + TimeOut = TimeOutValue, + }; + + using (MemoryStream rms = new MemoryStream(rawData)) + using (Joveler.Compression.XZ.XZStream xzs = new Joveler.Compression.XZ.XZStream(ms, compOpts)) + { + rms.CopyTo(xzs); + } + + ms.Flush(); + compLen = ms.Position; + } + + CompRatio = (double)compLen / rawData.Length; + return CompRatio; + } + + [Benchmark] + [BenchmarkCategory(BenchConfig.XZ)] + public long XZ_Native_Decomp() + { + byte[] compData = SrcDecompFiles[$"{SrcFileName}.xz"]; + using MemoryStream ms = new MemoryStream(); + + Joveler.Compression.XZ.XZDecompressOptions decompOpts = new Joveler.Compression.XZ.XZDecompressOptions(); + + Joveler.Compression.XZ.XZThreadedDecompressOptions threadOpts = new Joveler.Compression.XZ.XZThreadedDecompressOptions + { + Threads = 2, + TimeOut = TimeOutValue, + }; + + using (MemoryStream rms = new MemoryStream(compData)) + using (Joveler.Compression.XZ.XZStream zs = new Joveler.Compression.XZ.XZStream(rms, decompOpts)) + { + zs.CopyTo(ms); + } + + ms.Flush(); + return ms.Length; + } + } + #endregion +} diff --git a/Joveler.Compression.LZ4.Tests/Joveler.Compression.LZ4.Tests.csproj b/Joveler.Compression.LZ4.Tests/Joveler.Compression.LZ4.Tests.csproj index 8fa05a5..7987e0e 100644 --- a/Joveler.Compression.LZ4.Tests/Joveler.Compression.LZ4.Tests.csproj +++ b/Joveler.Compression.LZ4.Tests/Joveler.Compression.LZ4.Tests.csproj @@ -1,11 +1,9 @@  - - - net48;net6.0 - - net6.0 + + + net481;net6.0 net6.0 @@ -19,8 +17,8 @@ - - + + diff --git a/Joveler.Compression.LZ4/Joveler.Compression.LZ4.csproj b/Joveler.Compression.LZ4/Joveler.Compression.LZ4.csproj index 7a89d8f..eaf15c6 100644 --- a/Joveler.Compression.LZ4/Joveler.Compression.LZ4.csproj +++ b/Joveler.Compression.LZ4/Joveler.Compression.LZ4.csproj @@ -27,7 +27,7 @@ Supports Windows, Linux and macOS. - + x86\%(FileName)%(Extension) @@ -37,8 +37,12 @@ Supports Windows, Linux and macOS. x64\%(FileName)%(Extension) PreserveNewest + + arm64\%(FileName)%(Extension) + PreserveNewest + - + runtimes\win-x86\native\%(FileName)%(Extension) diff --git a/Joveler.Compression.LZ4/Joveler.Compression.LZ4.netfx.targets b/Joveler.Compression.LZ4/Joveler.Compression.LZ4.netfx.targets index 46683f1..68354b7 100644 --- a/Joveler.Compression.LZ4/Joveler.Compression.LZ4.netfx.targets +++ b/Joveler.Compression.LZ4/Joveler.Compression.LZ4.netfx.targets @@ -32,6 +32,7 @@ HasTrailingSlash('$(MSBuildThisFileDirectory)')"> + - + CollectInteropLibFiles_x86; CollectInteropLibFiles_x64; + CollectInteropLibFiles_arm64; $(PipelineCollectFilesPhaseDependsOn); diff --git a/Joveler.Compression.LZ4/LZ4FrameException.cs b/Joveler.Compression.LZ4/LZ4FrameException.cs index 94981d2..14eb49d 100644 --- a/Joveler.Compression.LZ4/LZ4FrameException.cs +++ b/Joveler.Compression.LZ4/LZ4FrameException.cs @@ -28,14 +28,12 @@ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR */ using System; -using System.Diagnostics.CodeAnalysis; using System.Runtime.InteropServices; using System.Runtime.Serialization; namespace Joveler.Compression.LZ4 { [Serializable] - [SuppressMessage("ReSharper", "InconsistentNaming")] public class LZ4FrameException : Exception { public ulong ReturnCode { get; set; } diff --git a/Joveler.Compression.LZ4/LZ4FrameStream.cs b/Joveler.Compression.LZ4/LZ4FrameStream.cs index 5eb831b..fd49385 100644 --- a/Joveler.Compression.LZ4/LZ4FrameStream.cs +++ b/Joveler.Compression.LZ4/LZ4FrameStream.cs @@ -99,6 +99,11 @@ public class LZ4FrameCompressOptions /// public class LZ4FrameDecompressOptions { + /// + /// disable checksum calculation and verification, even when one is present in frame, to save CPU time. + /// Setting this option to 1 once disables all checksums for the rest of the frame. + /// + public bool SkipChecksums { get; set; } = false; /// /// Size of the internal buffer. /// @@ -147,11 +152,23 @@ private enum Mode public long TotalIn { get; private set; } = 0; public long TotalOut { get; private set; } = 0; + // LZ4F_compressOptions_t, LZ4F_decompressOptions_t + private FrameCompressOptions _compOpts = new FrameCompressOptions() + { + StableSrc = 0, + }; + private FrameDecompressOptions _decompOpts = new FrameDecompressOptions() + { + StableDst = 0, + SkipChecksums = 0, + }; + // Const private const int DecompressComplete = -1; // https://github.com/lz4/lz4/blob/master/doc/lz4_Frame_format.md internal const uint FrameVersion = 100; private static readonly byte[] FrameMagicNumber = { 0x04, 0x22, 0x4D, 0x18 }; // 0x184D2204 (LE) + private static readonly byte[] FrameMagicSkippableStart = { 0x50, 0x2A, 0x4D, 0x18 }; // 0x184D2A50 (LE) /* private const int FrameSizeToKnowHeaderLength = 5; /// @@ -243,7 +260,7 @@ public unsafe LZ4FrameStream(Stream baseStream, LZ4FrameCompressOptions compOpts /// /// Create decompressing LZ4FrameStream. /// - public unsafe LZ4FrameStream(Stream baseStream, LZ4FrameDecompressOptions compOpts) + public unsafe LZ4FrameStream(Stream baseStream, LZ4FrameDecompressOptions decompOpts) { LZ4Init.Manager.EnsureLoaded(); @@ -252,13 +269,17 @@ public unsafe LZ4FrameStream(Stream baseStream, LZ4FrameDecompressOptions compOp _disposed = false; // Check and set compress options - _leaveOpen = compOpts.LeaveOpen; - _bufferSize = CheckBufferSize(compOpts.BufferSize); + _leaveOpen = decompOpts.LeaveOpen; + _bufferSize = CheckBufferSize(decompOpts.BufferSize); // Prepare dctx UIntPtr ret = LZ4Init.Lib.CreateFrameDecompressContext(ref _dctx, FrameVersion); LZ4FrameException.CheckReturnValue(ret); + // Prepare LZ4F_decompressOptions_t* + if (decompOpts.SkipChecksums) + _decompOpts.SkipChecksums = 1; + // Remove LZ4 frame header from the baseStream byte[] headerBuf = new byte[4]; int readHeaderSize = BaseStream.Read(headerBuf, 0, 4); @@ -356,7 +377,7 @@ public unsafe int Read(Span span) fixed (byte* header = FrameMagicNumber) fixed (byte* dest = span) { - ret = LZ4Init.Lib.FrameDecompress(_dctx, dest, ref destSizeVal, header, ref headerSizeVal, null); + ret = LZ4Init.Lib.FrameDecompress(_dctx, dest, ref destSizeVal, header, ref headerSizeVal, _decompOpts); } LZ4FrameException.CheckReturnValue(ret); @@ -397,7 +418,7 @@ public unsafe int Read(Span span) fixed (byte* src = _workBuf.AsSpan(_decompSrcIdx)) fixed (byte* dest = span) { - ret = LZ4Init.Lib.FrameDecompress(_dctx, dest, ref destSizeVal, src, ref srcSizeVal, null); + ret = LZ4Init.Lib.FrameDecompress(_dctx, dest, ref destSizeVal, src, ref srcSizeVal, _decompOpts); } LZ4FrameException.CheckReturnValue(ret); @@ -453,7 +474,7 @@ public unsafe void Write(ReadOnlySpan span) fixed (byte* dest = _workBuf) fixed (byte* src = span) { - outSizeVal = LZ4Init.Lib.FrameCompressUpdate(_cctx, dest, (UIntPtr)_destBufSize, src, (UIntPtr)srcWorkSize, null); + outSizeVal = LZ4Init.Lib.FrameCompressUpdate(_cctx, dest, (UIntPtr)_destBufSize, src, (UIntPtr)srcWorkSize, _compOpts); } LZ4FrameException.CheckReturnValue(outSizeVal); @@ -477,7 +498,7 @@ private unsafe void FinishWrite() UIntPtr outSizeVal; fixed (byte* dest = _workBuf) { - outSizeVal = LZ4Init.Lib.FrameCompressEnd(_cctx, dest, (UIntPtr)_destBufSize, null); + outSizeVal = LZ4Init.Lib.FrameCompressEnd(_cctx, dest, (UIntPtr)_destBufSize, _compOpts); } LZ4FrameException.CheckReturnValue(outSizeVal); diff --git a/Joveler.Compression.LZ4/LZ4Loader.cs b/Joveler.Compression.LZ4/LZ4Loader.cs index 93c4139..4828573 100644 --- a/Joveler.Compression.LZ4/LZ4Loader.cs +++ b/Joveler.Compression.LZ4/LZ4Loader.cs @@ -30,9 +30,6 @@ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR using Joveler.DynLoader; using System; using System.Runtime.InteropServices; -// ReSharper disable UnusedMember.Global -// ReSharper disable ArrangeTypeMemberModifiers -// ReSharper disable InconsistentNaming namespace Joveler.Compression.LZ4 { @@ -151,14 +148,20 @@ protected override void ResetFunctions() #region FrameCompress /// - /// The first thing to do is to create a compressionContext object, which will be used in all compression operations. - /// This is achieved using LZ4F_createCompressionContext(), which takes as argument a version. - /// The version provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. - /// The function will provide a pointer to a fully allocated LZ4F_cctx object. + /// The first thing to do is to create a compressionContext object, + /// which will keep track of operation state during streaming compression. + /// This is achieved using LZ4F_createCompressionContext(), which takes as argument a version, + /// and a pointer to LZ4F_cctx*, to write the resulting pointer into. /// + /// + /// MUST be != NULL. + /// + /// + /// provided MUST be LZ4F_VERSION. It is intended to track potential version mismatch, notably when using DLL. + /// The function provides a pointer to a fully allocated LZ4F_cctx object. + /// /// - /// If @return != zero, there was an error during context creation. - /// Object can release its memory using LZ4F_freeCompressionContext(); + /// If @return != zero, context creation failed. /// [UnmanagedFunctionPointer(CallingConvention.Cdecl)] internal delegate UIntPtr LZ4F_createCompressionContext( @@ -166,6 +169,15 @@ internal delegate UIntPtr LZ4F_createCompressionContext( uint version); internal LZ4F_createCompressionContext CreateFrameCompressContext; + /// + /// A created compression context can be employed multiple times for consecutive streaming operations. + /// Once all streaming compression jobs are completed, + /// the state object can be released using LZ4F_freeCompressionContext(). + /// + /// + /// Note1 : LZ4F_freeCompressionContext() is always successful. Its return value can be ignored. + /// Note2 : LZ4F_freeCompressionContext() works fine with NULL input pointers (do nothing). + /// [UnmanagedFunctionPointer(CallingConvention.Cdecl)] internal delegate UIntPtr LZ4F_freeCompressionContext(IntPtr cctx); internal LZ4F_freeCompressionContext FreeFrameCompressContext; @@ -210,13 +222,21 @@ internal delegate UIntPtr LZ4F_compressBound( internal LZ4F_compressBound FrameCompressBound; /// - /// When data must be generated and sent immediately, without waiting for a block to be completely filled, - /// it's possible to call LZ4_flush(). It will immediately compress any data buffered within cctx. - /// `dstCapacity` must be large enough to ensure the operation will be successful. - /// `cOptPtr` is optional : it's possible to provide NULL, all options will be set to default. + /// LZ4F_compressUpdate() can be called repetitively to compress as much data as necessary. /// + /// + /// Important rule: dstCapacity MUST be large enough to ensure operation success even in worst case situations. + /// This value is provided by LZ4F_compressBound(). + /// If this condition is not respected, LZ4F_compress() will fail (result is an errorCode). + /// After an error, the state is left in a UB state, and must be re-initialized or freed. + /// If previously an uncompressed block was written, buffered data is flushed + /// before appending compressed data is continued. + /// + /// + /// is optional : NULL can be provided, in which case all options are set to default. + /// /// - /// number of bytes written into dstBuffer (it can be zero, which means there was no data stored within cctx) + /// number of bytes written into `dstBuffer` (it can be zero, meaning input data was just buffered). /// or an error code if it fails (which can be tested using LZ4F_isError()) /// [UnmanagedFunctionPointer(CallingConvention.Cdecl)] @@ -275,15 +295,16 @@ internal delegate UIntPtr LZ4F_compressEnd( #region FrameDecompress /// - /// Create an LZ4F_dctx object, to track all decompression operations. - /// The version provided MUST be LZ4F_VERSION. - /// The function provides a pointer to an allocated and initialized LZ4F_dctx object. - /// The result is an errorCode, which can be tested using LZ4F_isError(). - /// dctx memory can be released using LZ4F_freeDecompressionContext(); + /// Create an LZ4F_dctx object, to track all decompression operations. /// + /// + /// MUST be valid. + /// + /// + /// MUST be LZ4F_VERSION. + /// /// - /// The result of LZ4F_freeDecompressionContext() is indicative of the current state of decompressionContext when being released. - /// That is, it should be == 0 if decompression has been completed fully and correctly. + /// The @return is an errorCode, which can be tested using LZ4F_isError(). /// [UnmanagedFunctionPointer(CallingConvention.Cdecl)] internal delegate UIntPtr LZ4F_createDecompressionContext( @@ -291,6 +312,14 @@ internal delegate UIntPtr LZ4F_createDecompressionContext( uint version); internal LZ4F_createDecompressionContext CreateFrameDecompressContext; + /// + /// dctx memory can be released using LZ4F_freeDecompressionContext(); + /// + /// + /// + /// Result of LZ4F_freeDecompressionContext() indicates current state of decompressionContext when being released. + /// That is, it should be == 0 if decompression has been completed fully and correctly. + /// [UnmanagedFunctionPointer(CallingConvention.Cdecl)] internal delegate UIntPtr LZ4F_freeDecompressionContext(IntPtr dctx); internal LZ4F_freeDecompressionContext FreeFrameDecompressContext; @@ -314,7 +343,7 @@ internal delegate UIntPtr LZ4F_createDecompressionContext( /// This function extracts frame parameters (max blockSize, dictID, etc.). /// /// - /// Its usage is optional: user can call LZ4F_decompress() directly. + /// Its usage is optional: user can also invoke LZ4F_decompress() directly. /// /// Extracted information will fill an existing LZ4F_frameInfo_t structure. /// This can be useful for allocation and dictionary identification purposes. diff --git a/Joveler.Compression.LZ4/LZ4Structs.cs b/Joveler.Compression.LZ4/LZ4Structs.cs index 2657e50..8bd95e4 100644 --- a/Joveler.Compression.LZ4/LZ4Structs.cs +++ b/Joveler.Compression.LZ4/LZ4Structs.cs @@ -111,7 +111,7 @@ internal class FramePreferences /// [MarshalAs(UnmanagedType.ByValArray, SizeConst = 3)] [System.Diagnostics.CodeAnalysis.SuppressMessage("스타일", "IDE0044:읽기 전용 한정자 추가", Justification = "<보류 중>")] - private uint[] Reserved; + private uint[] _reserved = new uint[3]; } #endregion @@ -128,7 +128,7 @@ internal class FrameCompressOptions /// [MarshalAs(UnmanagedType.ByValArray, SizeConst = 3)] [System.Diagnostics.CodeAnalysis.SuppressMessage("스타일", "IDE0044:읽기 전용 한정자 추가", Justification = "<보류 중>")] - private uint[] Reserved; + private uint[] _reserved = new uint[3] { 0, 0, 0 }; } #endregion @@ -137,15 +137,21 @@ internal class FrameCompressOptions internal class FrameDecompressOptions { /// - /// pledges that last 64KB decompressed data will remain available unmodified. This optimization skips storage operations in tmp buffers. + /// pledges that last 64KB decompressed data will remain available unmodified between invocations. + /// This optimization skips storage operations in tmp buffers. /// public uint StableDst; /// + /// disable checksum calculation and verification, even when one is present in frame, to save CPU time. + /// Setting this option to 1 once disables all checksums for the rest of the frame. + /// + public uint SkipChecksums; + /// /// must be set to zero for forward compatibility /// - [MarshalAs(UnmanagedType.ByValArray, SizeConst = 3)] + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 2)] [System.Diagnostics.CodeAnalysis.SuppressMessage("스타일", "IDE0044:읽기 전용 한정자 추가", Justification = "<보류 중>")] - private uint[] Reserved; + private uint[] _reserved = new uint[2] { 0, 0 }; } #endregion diff --git a/Joveler.Compression.LZ4/NUGET_README.md b/Joveler.Compression.LZ4/NUGET_README.md new file mode 100644 index 0000000..1e8f9a8 --- /dev/null +++ b/Joveler.Compression.LZ4/NUGET_README.md @@ -0,0 +1,23 @@ +## Usage + +Please refer to this [document](https://github.com/ied206/Joveler.Compression/blob/v4.1.0/Joveler.Compression.LZ4/USAGE.md). + +**WARNING**: The library supports many customizable options, but performance is a bit slow due to pinvoke overhead. See [README.md](https://github.com/ied206/Joveler.Compression/blob/v4.1.0/Joveler.Compression.LZ4/README.md) for details. + +## Tested liblz4 versions + +- 1.9.4 (Included) + +## Support + +### Targeted .NET platforms + +- .NET Standard 2.1 (.NET Core 3.0+) +- .NET Standard 2.0 (.NET Framework 4.6.1+, .NET Core 2.0+) +- .NET Framework 4.5.1 + +### Supported OS platforms + +- Windows (x86, x64, arm64) +- Linux (x64, arm, arm64) +- macOS (x64) diff --git a/Joveler.Compression.LZ4/runtimes/linux-arm/native/liblz4.so b/Joveler.Compression.LZ4/runtimes/linux-arm/native/liblz4.so index 485ce93..26001bc 100644 Binary files a/Joveler.Compression.LZ4/runtimes/linux-arm/native/liblz4.so and b/Joveler.Compression.LZ4/runtimes/linux-arm/native/liblz4.so differ diff --git a/Joveler.Compression.LZ4/runtimes/linux-arm64/native/liblz4.so b/Joveler.Compression.LZ4/runtimes/linux-arm64/native/liblz4.so index 6a1b5c7..a1f42c2 100644 Binary files a/Joveler.Compression.LZ4/runtimes/linux-arm64/native/liblz4.so and b/Joveler.Compression.LZ4/runtimes/linux-arm64/native/liblz4.so differ diff --git a/Joveler.Compression.LZ4/runtimes/linux-x64/native/liblz4.so b/Joveler.Compression.LZ4/runtimes/linux-x64/native/liblz4.so index e1ab4db..a991d8f 100644 Binary files a/Joveler.Compression.LZ4/runtimes/linux-x64/native/liblz4.so and b/Joveler.Compression.LZ4/runtimes/linux-x64/native/liblz4.so differ diff --git a/Joveler.Compression.LZ4/runtimes/osx-arm64/native/liblz4.dylib b/Joveler.Compression.LZ4/runtimes/osx-arm64/native/liblz4.dylib index ef42799..a7f48b6 100755 Binary files a/Joveler.Compression.LZ4/runtimes/osx-arm64/native/liblz4.dylib and b/Joveler.Compression.LZ4/runtimes/osx-arm64/native/liblz4.dylib differ diff --git a/Joveler.Compression.LZ4/runtimes/osx-x64/native/liblz4.dylib b/Joveler.Compression.LZ4/runtimes/osx-x64/native/liblz4.dylib index 6563d80..dea80da 100644 Binary files a/Joveler.Compression.LZ4/runtimes/osx-x64/native/liblz4.dylib and b/Joveler.Compression.LZ4/runtimes/osx-x64/native/liblz4.dylib differ diff --git a/Joveler.Compression.LZ4/runtimes/win-arm64/native/liblz4.dll b/Joveler.Compression.LZ4/runtimes/win-arm64/native/liblz4.dll index a83f448..2e3f00b 100644 Binary files a/Joveler.Compression.LZ4/runtimes/win-arm64/native/liblz4.dll and b/Joveler.Compression.LZ4/runtimes/win-arm64/native/liblz4.dll differ diff --git a/Joveler.Compression.LZ4/runtimes/win-x64/native/liblz4.dll b/Joveler.Compression.LZ4/runtimes/win-x64/native/liblz4.dll index ead923c..0ad8a88 100644 Binary files a/Joveler.Compression.LZ4/runtimes/win-x64/native/liblz4.dll and b/Joveler.Compression.LZ4/runtimes/win-x64/native/liblz4.dll differ diff --git a/Joveler.Compression.LZ4/runtimes/win-x86/native/liblz4.dll b/Joveler.Compression.LZ4/runtimes/win-x86/native/liblz4.dll index 50e6a70..61fcae7 100644 Binary files a/Joveler.Compression.LZ4/runtimes/win-x86/native/liblz4.dll and b/Joveler.Compression.LZ4/runtimes/win-x86/native/liblz4.dll differ diff --git a/Joveler.Compression.XZ.Tests/Joveler.Compression.XZ.Tests.csproj b/Joveler.Compression.XZ.Tests/Joveler.Compression.XZ.Tests.csproj index 930d7a8..a0d6984 100644 --- a/Joveler.Compression.XZ.Tests/Joveler.Compression.XZ.Tests.csproj +++ b/Joveler.Compression.XZ.Tests/Joveler.Compression.XZ.Tests.csproj @@ -1,11 +1,9 @@  - - - net48;net6.0 - - net6.0 + + + net481;net6.0 net6.0 @@ -15,8 +13,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Joveler.Compression.XZ.Tests/Samples/A.lz b/Joveler.Compression.XZ.Tests/Samples/A.lz new file mode 100644 index 0000000..4d6db9f Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/A.lz differ diff --git a/Joveler.Compression.XZ.Tests/Samples/A.lzma b/Joveler.Compression.XZ.Tests/Samples/A.lzma new file mode 100644 index 0000000..9c70c2f Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/A.lzma differ diff --git a/Joveler.Compression.XZ.Tests/Samples/A_mt16.xz b/Joveler.Compression.XZ.Tests/Samples/A_mt16.xz new file mode 100644 index 0000000..8661163 Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/A_mt16.xz differ diff --git a/Joveler.Compression.XZ.Tests/Samples/B1.lz b/Joveler.Compression.XZ.Tests/Samples/B1.lz new file mode 100644 index 0000000..feed077 Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/B1.lz differ diff --git a/Joveler.Compression.XZ.Tests/Samples/B1.lzma b/Joveler.Compression.XZ.Tests/Samples/B1.lzma new file mode 100644 index 0000000..d179be1 Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/B1.lzma differ diff --git a/Joveler.Compression.XZ.Tests/Samples/B1_mt16.xz b/Joveler.Compression.XZ.Tests/Samples/B1_mt16.xz new file mode 100644 index 0000000..5a05cc4 Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/B1_mt16.xz differ diff --git a/Joveler.Compression.XZ.Tests/Samples/B9.lz b/Joveler.Compression.XZ.Tests/Samples/B9.lz new file mode 100644 index 0000000..6c24c89 Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/B9.lz differ diff --git a/Joveler.Compression.XZ.Tests/Samples/B9.lzma b/Joveler.Compression.XZ.Tests/Samples/B9.lzma new file mode 100644 index 0000000..85f5c92 Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/B9.lzma differ diff --git a/Joveler.Compression.XZ.Tests/Samples/B9_mt16.xz b/Joveler.Compression.XZ.Tests/Samples/B9_mt16.xz new file mode 100644 index 0000000..031db4e Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/B9_mt16.xz differ diff --git a/Joveler.Compression.XZ.Tests/Samples/C.lz b/Joveler.Compression.XZ.Tests/Samples/C.lz new file mode 100644 index 0000000..76384cb Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/C.lz differ diff --git a/Joveler.Compression.XZ.Tests/Samples/C.lzma b/Joveler.Compression.XZ.Tests/Samples/C.lzma new file mode 100644 index 0000000..91cc7a9 Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/C.lzma differ diff --git a/Joveler.Compression.XZ.Tests/Samples/C_mt16.xz b/Joveler.Compression.XZ.Tests/Samples/C_mt16.xz new file mode 100644 index 0000000..d6350d8 Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/C_mt16.xz differ diff --git a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.elf b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.elf index c20da80..8869746 100755 Binary files a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.elf and b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.elf differ diff --git a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.exe b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.exe new file mode 100644 index 0000000..dca5874 Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.exe differ diff --git a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.mach b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.mach index d4408f8..799403b 100755 Binary files a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.mach and b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.arm64.mach differ diff --git a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.armhf.elf b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.armhf.elf index 93f7984..8b7766e 100755 Binary files a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.armhf.elf and b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.armhf.elf differ diff --git a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.elf b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.elf index 3241126..d1da7d8 100755 Binary files a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.elf and b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.elf differ diff --git a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.exe b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.exe index fbdb1f1..8e91177 100644 Binary files a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.exe and b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.exe differ diff --git a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.mach b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.mach index a2b94c5..adeaf74 100755 Binary files a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.mach and b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x64.mach differ diff --git a/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x86.exe b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x86.exe new file mode 100644 index 0000000..c8ace0d Binary files /dev/null and b/Joveler.Compression.XZ.Tests/Samples/RefBin/xz.x86.exe differ diff --git a/Joveler.Compression.XZ.Tests/TestSetup.cs b/Joveler.Compression.XZ.Tests/TestSetup.cs index e7e8391..9289c89 100644 --- a/Joveler.Compression.XZ.Tests/TestSetup.cs +++ b/Joveler.Compression.XZ.Tests/TestSetup.cs @@ -1,8 +1,6 @@ /* - Derived from liblzma header files (Public Domain) - C# Wrapper written by Hajin Jang - Copyright (C) 2018-2020 Hajin Jang + Copyright (C) 2018-2023 Hajin Jang MIT License @@ -155,41 +153,35 @@ public static int RunXZ(string tempArchiveFile) { const string binDir = "RefBin"; - string binary = null; - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + string arch = string.Empty; + switch (RuntimeInformation.ProcessArchitecture) { - binary = Path.Combine(TestSetup.SampleDir, binDir, "xz.x64.exe"); + case Architecture.X86: + arch = "x86"; + break; + case Architecture.X64: + arch = "x64"; + break; + case Architecture.Arm: + arch = "armhf"; + break; + case Architecture.Arm64: + arch = "arm64"; + break; } + + string ext = string.Empty; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + ext = "exe"; else if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - switch (RuntimeInformation.ProcessArchitecture) - { - case Architecture.X64: - binary = Path.Combine(TestSetup.SampleDir, binDir, "xz.x64.elf"); - break; - case Architecture.Arm: - binary = Path.Combine(TestSetup.SampleDir, binDir, "xz.armhf.elf"); - break; - case Architecture.Arm64: - binary = Path.Combine(TestSetup.SampleDir, binDir, "xz.arm64.elf"); - break; - } - } + ext = "elf"; else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - switch (RuntimeInformation.ProcessArchitecture) - { - case Architecture.X64: - binary = Path.Combine(TestSetup.SampleDir, binDir, "xz.x64.mach"); - break; - case Architecture.Arm64: - binary = Path.Combine(TestSetup.SampleDir, binDir, "xz.arm64.mach"); - break; - } - } + ext = "mach"; + + string binary = Path.Combine(TestSetup.SampleDir, binDir, $"xz.{arch}.{ext}"); - if (binary == null) - throw new PlatformNotSupportedException(); + if (File.Exists(binary) == false) + throw new PlatformNotSupportedException($"xz executable does not exist! [{binary}]"); Process proc = new Process { diff --git a/Joveler.Compression.XZ.Tests/XZHardwareTests.cs b/Joveler.Compression.XZ.Tests/XZHardwareTests.cs new file mode 100644 index 0000000..fb88be5 --- /dev/null +++ b/Joveler.Compression.XZ.Tests/XZHardwareTests.cs @@ -0,0 +1,51 @@ +/* + C# Wrapper written by Hajin Jang + Copyright (C) 2018-2023 Hajin Jang + + MIT License + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; + +namespace Joveler.Compression.XZ.Tests +{ + [TestClass] + [TestCategory("Joveler.Compression.XZ")] + public class XZHardwareTests + { + [TestMethod] + public void PhysMem() + { + ulong physMem = XZHardware.PhysMem(); + Console.WriteLine($"Hardware Physical Memory = {physMem}"); + } + + [TestMethod] + public void CpuThreads() + { + uint xzCoreCount = XZHardware.CpuThreads(); + uint bclCoreCount = (uint)Environment.ProcessorCount; + Assert.AreEqual(bclCoreCount, xzCoreCount); + Console.WriteLine($"Hardware CPU Threads = {xzCoreCount}"); + } + } +} diff --git a/Joveler.Compression.XZ.Tests/XZInitTests.cs b/Joveler.Compression.XZ.Tests/XZInitTests.cs index def09ca..b14f463 100644 --- a/Joveler.Compression.XZ.Tests/XZInitTests.cs +++ b/Joveler.Compression.XZ.Tests/XZInitTests.cs @@ -1,8 +1,6 @@ /* - Derived from liblzma header files (Public Domain) - C# Wrapper written by Hajin Jang - Copyright (C) 2018-2020 Hajin Jang + Copyright (C) 2018-2023 Hajin Jang MIT License @@ -43,88 +41,5 @@ public void Version() string verStr = XZInit.VersionString(); Console.WriteLine($"liblzma Version (String) = {verStr}"); } - - [TestMethod] - public void PhysMem() - { - ulong physMem = XZInit.PhysMem(); - Console.WriteLine($"Hardware Physical Memory = {physMem}"); - } - - [TestMethod] - public void CpuThreads() - { - uint xzCoreCount = XZInit.CpuThreads(); - uint bclCoreCount = (uint)Environment.ProcessorCount; - Assert.AreEqual(bclCoreCount, xzCoreCount); - Console.WriteLine($"Hardware CPU Threads = {xzCoreCount}"); - } - - #region EncoderMemUsage - private void EncoderMemUsageTemplate(LzmaCompLevel level, bool extreme) - { - void PrintMemUsage(ulong usage, int threads = 0) - { - char extremeChar = extreme ? 'e' : ' '; - uint purePreset = (uint)level; - string msg; - if (threads == 0) - msg = $"Encoder Mem Usage (p{purePreset}{extremeChar}) = {usage / (1024 * 1024) + 1}MB ({usage}B)"; - else - msg = $"Encoder Mem Usage (p{purePreset}{extremeChar}, {threads}T) = {usage / (1024 * 1024) + 1}MB ({usage}B)"; - Console.WriteLine(msg); - } - - ulong single = XZInit.EncoderMemUsage(level, extreme); - ulong multi1 = XZInit.EncoderMultiMemUsage(level, extreme, 1); - ulong multi2 = XZInit.EncoderMultiMemUsage(level, extreme, 2); - PrintMemUsage(single); - PrintMemUsage(multi1, 1); - PrintMemUsage(multi2, 2); - - Assert.AreNotEqual(ulong.MaxValue, single); - Assert.AreNotEqual(ulong.MaxValue, multi1); - Assert.AreNotEqual(ulong.MaxValue, multi2); - Assert.IsTrue(single < multi1); - } - - [TestMethod] - public void EncoderMemUsage() - { - EncoderMemUsageTemplate(LzmaCompLevel.Level0, false); - EncoderMemUsageTemplate(LzmaCompLevel.Level0, true); - EncoderMemUsageTemplate(LzmaCompLevel.Default, false); - EncoderMemUsageTemplate(LzmaCompLevel.Default, true); - EncoderMemUsageTemplate(LzmaCompLevel.Level9, false); - EncoderMemUsageTemplate(LzmaCompLevel.Level9, true); - } - #endregion - - #region DecoderMemUsage - private void DecoderMemUsageTemplate(LzmaCompLevel level, bool extreme) - { - void PrintMemUsage(ulong memUsage) - { - char extremeChar = extreme ? 'e' : ' '; - uint purePreset = (uint)level; - Console.WriteLine($"Decoder Mem Usage (p{purePreset}{extremeChar}) = {memUsage / (1024 * 1024) + 1}MB ({memUsage}B)"); - } - - ulong usage = XZInit.DecoderMemUsage(level, extreme); - PrintMemUsage(usage); - Assert.AreNotEqual(ulong.MaxValue, usage); - } - - [TestMethod] - public void DecoderMemUsage() - { - DecoderMemUsageTemplate(LzmaCompLevel.Level0, false); - DecoderMemUsageTemplate(LzmaCompLevel.Level0, true); - DecoderMemUsageTemplate(LzmaCompLevel.Default, false); - DecoderMemUsageTemplate(LzmaCompLevel.Default, true); - DecoderMemUsageTemplate(LzmaCompLevel.Level9, false); - DecoderMemUsageTemplate(LzmaCompLevel.Level9, true); - } - #endregion } } diff --git a/Joveler.Compression.XZ.Tests/XZMemoryTests.cs b/Joveler.Compression.XZ.Tests/XZMemoryTests.cs new file mode 100644 index 0000000..01293ef --- /dev/null +++ b/Joveler.Compression.XZ.Tests/XZMemoryTests.cs @@ -0,0 +1,102 @@ +/* + C# Wrapper written by Hajin Jang + Copyright (C) 2018-2023 Hajin Jang + + MIT License + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; + +namespace Joveler.Compression.XZ.Tests +{ + [TestClass] + [TestCategory("Joveler.Compression.XZ")] + public class XZMemoryTests + { + #region EncoderMemUsage + private void EncoderMemUsageTemplate(LzmaCompLevel level, bool extreme) + { + void PrintMemUsage(ulong usage, int threads = 0) + { + char extremeChar = extreme ? 'e' : ' '; + uint purePreset = (uint)level; + string msg; + if (threads == 0) + msg = $"Encoder Mem Usage (p{purePreset}{extremeChar}) = {usage / (1024 * 1024) + 1}MB ({usage}B)"; + else + msg = $"Encoder Mem Usage (p{purePreset}{extremeChar}, {threads}T) = {usage / (1024 * 1024) + 1}MB ({usage}B)"; + Console.WriteLine(msg); + } + + ulong single = XZMemory.EncoderMemUsage(level, extreme); + ulong multi1 = XZMemory.ThreadedEncoderMemUsage(level, extreme, 1); + ulong multi2 = XZMemory.ThreadedEncoderMemUsage(level, extreme, 2); + PrintMemUsage(single); + PrintMemUsage(multi1, 1); + PrintMemUsage(multi2, 2); + + Assert.AreNotEqual(ulong.MaxValue, single); + Assert.AreNotEqual(ulong.MaxValue, multi1); + Assert.AreNotEqual(ulong.MaxValue, multi2); + Assert.IsTrue(single < multi1); + } + + [TestMethod] + public void EncoderMemUsage() + { + EncoderMemUsageTemplate(LzmaCompLevel.Level0, false); + EncoderMemUsageTemplate(LzmaCompLevel.Level0, true); + EncoderMemUsageTemplate(LzmaCompLevel.Default, false); + EncoderMemUsageTemplate(LzmaCompLevel.Default, true); + EncoderMemUsageTemplate(LzmaCompLevel.Level9, false); + EncoderMemUsageTemplate(LzmaCompLevel.Level9, true); + } + #endregion + + #region DecoderMemUsage + private void DecoderMemUsageTemplate(LzmaCompLevel level, bool extreme) + { + void PrintMemUsage(ulong memUsage) + { + char extremeChar = extreme ? 'e' : ' '; + uint purePreset = (uint)level; + Console.WriteLine($"Decoder Mem Usage (p{purePreset}{extremeChar}) = {memUsage / (1024 * 1024) + 1}MB ({memUsage}B)"); + } + + ulong usage = XZMemory.DecoderMemUsage(level, extreme); + PrintMemUsage(usage); + Assert.AreNotEqual(ulong.MaxValue, usage); + } + + [TestMethod] + public void DecoderMemUsage() + { + DecoderMemUsageTemplate(LzmaCompLevel.Level0, false); + DecoderMemUsageTemplate(LzmaCompLevel.Level0, true); + DecoderMemUsageTemplate(LzmaCompLevel.Default, false); + DecoderMemUsageTemplate(LzmaCompLevel.Default, true); + DecoderMemUsageTemplate(LzmaCompLevel.Level9, false); + DecoderMemUsageTemplate(LzmaCompLevel.Level9, true); + } + #endregion + } +} diff --git a/Joveler.Compression.XZ.Tests/XZStreamTests.cs b/Joveler.Compression.XZ.Tests/XZStreamTests.cs deleted file mode 100644 index 9c5ddfb..0000000 --- a/Joveler.Compression.XZ.Tests/XZStreamTests.cs +++ /dev/null @@ -1,240 +0,0 @@ -/* - Derived from liblzma header files (Public Domain) - - C# Wrapper written by Hajin Jang - Copyright (C) 2018-2020 Hajin Jang - - MIT License - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -*/ - -using Microsoft.VisualStudio.TestTools.UnitTesting; -using System; -using System.IO; -using System.Linq; -using System.Security.Cryptography; - -namespace Joveler.Compression.XZ.Tests -{ - [TestClass] - [TestCategory("Joveler.Compression.XZ")] - public class XZStreamTests - { - #region Compress - [TestMethod] - public void Compress() - { - CompressTemplate("A.pdf", false, true, 1, LzmaCompLevel.Level7, false); - CompressTemplate("B.txt", false, true, 1, LzmaCompLevel.Default, false); - CompressTemplate("C.bin", false, true, 1, LzmaCompLevel.Level1, true); - CompressTemplate("C.bin", false, false, 1, (LzmaCompLevel)255, false); - } - - [TestMethod] - public void CompressSpan() - { - CompressTemplate("A.pdf", true, true, 1, LzmaCompLevel.Level7, false); - CompressTemplate("B.txt", true, true, 1, LzmaCompLevel.Default, false); - CompressTemplate("C.bin", true, true, 1, LzmaCompLevel.Level1, true); - CompressTemplate("C.bin", true, false, 1, (LzmaCompLevel)255, false); - } - - [TestMethod] - public void CompressMulti() - { - CompressTemplate("A.pdf", false, true, 2, LzmaCompLevel.Level7, false); - CompressTemplate("B.txt", false, true, 2, LzmaCompLevel.Level3, true); - CompressTemplate("C.bin", false, true, Environment.ProcessorCount, LzmaCompLevel.Level1, false); - CompressTemplate("C.bin", false, false, Environment.ProcessorCount, (LzmaCompLevel)255, false); - } - - private static void CompressTemplate(string sampleFileName, bool useSpan, bool success, int threads, LzmaCompLevel level, bool extreme) - { - string destDir = Path.GetTempFileName(); - File.Delete(destDir); - Directory.CreateDirectory(destDir); - try - { - string tempDecompFile = Path.Combine(destDir, Path.GetFileName(sampleFileName)); - string tempXzFile = tempDecompFile + ".xz"; - - XZCompressOptions compOpts = new XZCompressOptions - { - Level = level, - ExtremeFlag = extreme, - LeaveOpen = true, - }; - XZThreadedCompressOptions threadOpts = new XZThreadedCompressOptions - { - Threads = threads, - }; - - string sampleFile = Path.Combine(TestSetup.SampleDir, sampleFileName); - using (FileStream xzCompFs = new FileStream(tempXzFile, FileMode.Create, FileAccess.Write, FileShare.None)) - using (FileStream sampleFs = new FileStream(sampleFile, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (XZStream xzs = new XZStream(xzCompFs, compOpts, threadOpts)) - { -#if !NETFRAMEWORK - if (useSpan) - { - byte[] buffer = new byte[64 * 1024]; - - int bytesRead; - do - { - bytesRead = sampleFs.Read(buffer.AsSpan()); - xzs.Write(buffer.AsSpan(0, bytesRead)); - } while (0 < bytesRead); - } - else -#endif - { - sampleFs.CopyTo(xzs); - } - - xzs.Flush(); - xzs.GetProgress(out ulong finalIn, out ulong finalOut); - - Assert.AreEqual(sampleFs.Length, xzs.TotalIn); - Assert.AreEqual(xzCompFs.Length, xzs.TotalOut); - Assert.AreEqual((ulong)sampleFs.Length, finalIn); - Assert.AreEqual((ulong)xzCompFs.Length, finalOut); - } - - Assert.IsTrue(TestHelper.RunXZ(tempXzFile) == 0); - - byte[] decompDigest; - byte[] originDigest; - using (FileStream fs = new FileStream(sampleFile, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - using (HashAlgorithm hash = SHA256.Create()) - { - originDigest = hash.ComputeHash(fs); - } - } - - using (FileStream fs = new FileStream(tempDecompFile, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - using (HashAlgorithm hash = SHA256.Create()) - { - decompDigest = hash.ComputeHash(fs); - } - } - - Assert.IsTrue(originDigest.SequenceEqual(decompDigest)); - Assert.IsTrue(success); - } -#pragma warning disable CA1031 // Do not catch general exception types - catch (Exception) - { - Assert.IsFalse(success); - } -#pragma warning restore CA1031 // Do not catch general exception types - finally - { - if (Directory.Exists(destDir)) - Directory.Delete(destDir, true); - } - } - #endregion - - #region Decompress - [TestMethod] - public void Decompress() - { - Template("A.xz", "A.pdf", false); - Template("B9.xz", "B.txt", false); - Template("B1.xz", "B.txt", false); - Template("C.xz", "C.bin", false); - } - - [TestMethod] - public void DecompressSpan() - { - Template("A.xz", "A.pdf", true); - Template("B9.xz", "B.txt", true); - Template("B1.xz", "B.txt", true); - Template("C.xz", "C.bin", true); - } - - private static void Template(string xzFileName, string originFileName, bool useSpan) - { - byte[] decompDigest; - byte[] originDigest; - - string xzFile = Path.Combine(TestSetup.SampleDir, xzFileName); - string originFile = Path.Combine(TestSetup.SampleDir, originFileName); - using (MemoryStream decompMs = new MemoryStream()) - { - XZDecompressOptions decompOpts = new XZDecompressOptions(); - - using (FileStream compFs = new FileStream(xzFile, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (XZStream xz = new XZStream(compFs, decompOpts)) - { -#if !NETFRAMEWORK - if (useSpan) - { - byte[] buffer = new byte[64 * 1024]; - - int bytesRead; - do - { - bytesRead = xz.Read(buffer.AsSpan()); - decompMs.Write(buffer.AsSpan(0, bytesRead)); - } while (0 < bytesRead); - } - else -#endif - { - xz.CopyTo(decompMs); - } - - decompMs.Flush(); - xz.GetProgress(out ulong finalIn, out ulong finalOut); - - Assert.AreEqual(compFs.Length, xz.TotalIn); - Assert.AreEqual(decompMs.Length, xz.TotalOut); - Assert.AreEqual((ulong)compFs.Length, finalIn); - Assert.AreEqual((ulong)decompMs.Length, finalOut); - } - decompMs.Position = 0; - - using (HashAlgorithm hash = SHA256.Create()) - { - decompDigest = hash.ComputeHash(decompMs); - } - } - - using (FileStream originFs = new FileStream(originFile, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - using (HashAlgorithm hash = SHA256.Create()) - { - originDigest = hash.ComputeHash(originFs); - } - } - - Assert.IsTrue(decompDigest.SequenceEqual(originDigest)); - } - #endregion - - #region GetProgress - #endregion - } -} diff --git a/Joveler.Compression.XZ.Tests/XZStreamsTests.cs b/Joveler.Compression.XZ.Tests/XZStreamsTests.cs new file mode 100644 index 0000000..384ba09 --- /dev/null +++ b/Joveler.Compression.XZ.Tests/XZStreamsTests.cs @@ -0,0 +1,461 @@ +/* + C# Wrapper written by Hajin Jang + Copyright (C) 2018-2023 Hajin Jang + + MIT License + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.IO; +using System.Linq; +using System.Security.Cryptography; + +namespace Joveler.Compression.XZ.Tests +{ + [TestClass] + [TestCategory("Joveler.Compression.XZ")] + public class XZStreamsTests + { + #region XZ Compress + [TestMethod] + public void XZCompressSingle() + { + XZCompressTemplate("A.pdf", false, true, -1, LzmaCompLevel.Level7, false); + XZCompressTemplate("B.txt", false, true, -1, LzmaCompLevel.Default, false); + XZCompressTemplate("C.bin", false, true, -1, LzmaCompLevel.Level1, true); + XZCompressTemplate("C.bin", false, false, -1, (LzmaCompLevel)255, false); + } + + [TestMethod] + public void XZCompressSingleSpan() + { + XZCompressTemplate("A.pdf", true, true, -1, LzmaCompLevel.Level7, false); + XZCompressTemplate("B.txt", true, true, -1, LzmaCompLevel.Default, false); + XZCompressTemplate("C.bin", true, true, -1, LzmaCompLevel.Level1, true); + XZCompressTemplate("C.bin", true, false, -1, (LzmaCompLevel)255, false); + } + + [TestMethod] + public void XZCompressMulti() + { + XZCompressTemplate("A.pdf", false, true, 1, LzmaCompLevel.Level7, false); + XZCompressTemplate("A.pdf", false, true, 2, LzmaCompLevel.Default, false); + XZCompressTemplate("B.txt", false, true, 2, LzmaCompLevel.Level3, true); + XZCompressTemplate("C.bin", false, true, Environment.ProcessorCount, LzmaCompLevel.Level1, false); + XZCompressTemplate("C.bin", false, false, Environment.ProcessorCount, (LzmaCompLevel)255, false); + } + + private static void XZCompressTemplate(string sampleFileName, bool useSpan, bool success, int threads, LzmaCompLevel level, bool extreme) + { + string destDir = Path.GetTempFileName(); + File.Delete(destDir); + Directory.CreateDirectory(destDir); + try + { + string tempDecompFile = Path.Combine(destDir, Path.GetFileName(sampleFileName)); + string tempXzFile = tempDecompFile + ".xz"; + + XZCompressOptions compOpts = new XZCompressOptions + { + Level = level, + ExtremeFlag = extreme, + LeaveOpen = true, + }; + + string sampleFile = Path.Combine(TestSetup.SampleDir, sampleFileName); + + using (FileStream xzCompFs = new FileStream(tempXzFile, FileMode.Create, FileAccess.Write, FileShare.None)) + using (FileStream sampleFs = new FileStream(sampleFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + XZStream xzs = null; + try + { + if (threads == -1) + { // Single-thread compression + xzs = new XZStream(xzCompFs, compOpts); + } + else if (0 < threads) + { // Multi-thread compression + XZThreadedCompressOptions threadOpts = new XZThreadedCompressOptions + { + Threads = threads, + }; + xzs = new XZStream(xzCompFs, compOpts, threadOpts); + } + else + { + Assert.Fail($"threads [{threads}] is not a valid test value."); + } + +#if !NETFRAMEWORK + if (useSpan) + { + byte[] buffer = new byte[64 * 1024]; + + int bytesRead; + do + { + bytesRead = sampleFs.Read(buffer.AsSpan()); + xzs.Write(buffer.AsSpan(0, bytesRead)); + } while (0 < bytesRead); + } + else +#endif + { + sampleFs.CopyTo(xzs); + } + + xzs.Flush(); + xzs.GetProgress(out ulong finalIn, out ulong finalOut); + + Assert.AreEqual(sampleFs.Length, xzs.TotalIn); + Assert.AreEqual(xzCompFs.Length, xzs.TotalOut); + Assert.AreEqual((ulong)sampleFs.Length, finalIn); + Assert.AreEqual((ulong)xzCompFs.Length, finalOut); + } + finally + { + xzs?.Dispose(); + xzs = null; + } + } + + Assert.IsTrue(TestHelper.RunXZ(tempXzFile) == 0); + + byte[] decompDigest; + byte[] originDigest; + using (FileStream fs = new FileStream(sampleFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + using (HashAlgorithm hash = SHA256.Create()) + { + originDigest = hash.ComputeHash(fs); + } + } + + using (FileStream fs = new FileStream(tempDecompFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + using (HashAlgorithm hash = SHA256.Create()) + { + decompDigest = hash.ComputeHash(fs); + } + } + + Assert.IsTrue(originDigest.SequenceEqual(decompDigest)); + Assert.IsTrue(success); + } + catch (Exception) + { + Assert.IsFalse(success); + } + finally + { + if (Directory.Exists(destDir)) + Directory.Delete(destDir, true); + } + } + #endregion + + #region XZ Decompress + [TestMethod] + public void XZDecompressSingle() + { + XZDecompressTemplate("A.xz", "A.pdf", -1, false); + XZDecompressTemplate("B9.xz", "B.txt", -1, false); + XZDecompressTemplate("B1.xz", "B.txt", -1, false); + XZDecompressTemplate("C.xz", "C.bin", -1, false); + } + + [TestMethod] + public void XZDecompressSingleSpan() + { + XZDecompressTemplate("A.xz", "A.pdf", -1, true); + XZDecompressTemplate("B9.xz", "B.txt", -1, true); + XZDecompressTemplate("B1.xz", "B.txt", -1, true); + XZDecompressTemplate("C.xz", "C.bin", -1, true); + } + + [TestMethod] + public void XZDecompressMulti() + { + XZDecompressTemplate("A_mt16.xz", "A.pdf", 1, true); + XZDecompressTemplate("B9_mt16.xz", "B.txt", 2, true); + XZDecompressTemplate("B1_mt16.xz", "B.txt", Environment.ProcessorCount, true); + XZDecompressTemplate("C_mt16.xz", "C.bin", Environment.ProcessorCount, true); + } + + private static void XZDecompressTemplate(string xzFileName, string originFileName, int threads, bool useSpan) + { + byte[] decompDigest; + byte[] originDigest; + + string xzFile = Path.Combine(TestSetup.SampleDir, xzFileName); + string originFile = Path.Combine(TestSetup.SampleDir, originFileName); + using (MemoryStream decompMs = new MemoryStream()) + { + XZDecompressOptions decompOpts = new XZDecompressOptions(); + + XZStream xzs = null; + try + { + using (FileStream compFs = new FileStream(xzFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + if (threads == -1) + { // Single-thread compression + xzs = new XZStream(compFs, decompOpts); + } + else if (0 < threads) + { // Multi-thread compression + XZThreadedDecompressOptions threadOpts = new XZThreadedDecompressOptions + { + Threads = threads, + }; + xzs = new XZStream(compFs, decompOpts, threadOpts); + } + else + { + Assert.Fail($"threads [{threads}] is not a valid test value."); + } + +#if !NETFRAMEWORK + if (useSpan) + { + byte[] buffer = new byte[64 * 1024]; + + int bytesRead; + do + { + bytesRead = xzs.Read(buffer.AsSpan()); + decompMs.Write(buffer.AsSpan(0, bytesRead)); + } while (0 < bytesRead); + } + else +#endif + { + xzs.CopyTo(decompMs); + } + +#if LZMA_MEM_ENABLE + ulong memUsage = xzs.GetDecompresMemUsage(); + Console.WriteLine($"{xzFileName} ({threads}t) MEM: requires {memUsage / (1024 * 1024) + 1}MB ({memUsage}B)"); +#endif + + decompMs.Flush(); + xzs.GetProgress(out ulong finalIn, out ulong finalOut); + + Assert.AreEqual(compFs.Length, xzs.TotalIn); + Assert.AreEqual(decompMs.Length, xzs.TotalOut); + Assert.AreEqual((ulong)compFs.Length, finalIn); + Assert.AreEqual((ulong)decompMs.Length, finalOut); + } + } + finally + { + xzs?.Dispose(); + xzs = null; + } + + decompMs.Position = 0; + + using (HashAlgorithm hash = SHA256.Create()) + { + decompDigest = hash.ComputeHash(decompMs); + } + } + + using (FileStream originFs = new FileStream(originFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + using (HashAlgorithm hash = SHA256.Create()) + { + originDigest = hash.ComputeHash(originFs); + } + } + + Assert.IsTrue(decompDigest.SequenceEqual(originDigest)); + } + #endregion + + #region Legacy LZMA Decompress + [TestMethod] + public void LzmaAloneDecompressSingle() + { + LzmaAloneDecompressTemplate("A.lzma", "A.pdf"); + LzmaAloneDecompressTemplate("B9.lzma", "B.txt"); + LzmaAloneDecompressTemplate("B1.lzma", "B.txt"); + LzmaAloneDecompressTemplate("C.lzma", "C.bin"); + } + + private static void LzmaAloneDecompressTemplate(string lzmaFileName, string originFileName) + { + byte[] decompDigest; + byte[] originDigest; + + string xzFile = Path.Combine(TestSetup.SampleDir, lzmaFileName); + string originFile = Path.Combine(TestSetup.SampleDir, originFileName); + using (MemoryStream decompMs = new MemoryStream()) + { + XZDecompressOptions decompOpts = new XZDecompressOptions(); + + using (FileStream compFs = new FileStream(xzFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (LzmaAloneStream lzs = new LzmaAloneStream(compFs, decompOpts)) + { + lzs.CopyTo(decompMs); + + decompMs.Flush(); + lzs.GetProgress(out ulong finalIn, out ulong finalOut); + + Assert.AreEqual(compFs.Length, lzs.TotalIn); + Assert.AreEqual(decompMs.Length, lzs.TotalOut); + Assert.AreEqual((ulong)compFs.Length, finalIn); + Assert.AreEqual((ulong)decompMs.Length, finalOut); + } + + decompMs.Position = 0; + + using (HashAlgorithm hash = SHA256.Create()) + { + decompDigest = hash.ComputeHash(decompMs); + } + } + + using (FileStream originFs = new FileStream(originFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + using (HashAlgorithm hash = SHA256.Create()) + { + originDigest = hash.ComputeHash(originFs); + } + } + + Assert.IsTrue(decompDigest.SequenceEqual(originDigest)); + } + #endregion + + #region LZip Decompress + [TestMethod] + public void LZipDecompressSingle() + { + LZipDecompressTemplate("A.lz", "A.pdf"); + LZipDecompressTemplate("B9.lz", "B.txt"); + LZipDecompressTemplate("B1.lz", "B.txt"); + LZipDecompressTemplate("C.lz", "C.bin"); + } + + private static void LZipDecompressTemplate(string lzmaFileName, string originFileName) + { + byte[] decompDigest; + byte[] originDigest; + + string xzFile = Path.Combine(TestSetup.SampleDir, lzmaFileName); + string originFile = Path.Combine(TestSetup.SampleDir, originFileName); + using (MemoryStream decompMs = new MemoryStream()) + { + XZDecompressOptions decompOpts = new XZDecompressOptions(); + + using (FileStream compFs = new FileStream(xzFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (LZipStream lzs = new LZipStream(compFs, decompOpts)) + { + lzs.CopyTo(decompMs); + + decompMs.Flush(); + lzs.GetProgress(out ulong finalIn, out ulong finalOut); + + Assert.AreEqual(compFs.Length, lzs.TotalIn); + Assert.AreEqual(decompMs.Length, lzs.TotalOut); + Assert.AreEqual((ulong)compFs.Length, finalIn); + Assert.AreEqual((ulong)decompMs.Length, finalOut); + } + + decompMs.Position = 0; + + using (HashAlgorithm hash = SHA256.Create()) + { + decompDigest = hash.ComputeHash(decompMs); + } + } + + using (FileStream originFs = new FileStream(originFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + using (HashAlgorithm hash = SHA256.Create()) + { + originDigest = hash.ComputeHash(originFs); + } + } + + Assert.IsTrue(decompDigest.SequenceEqual(originDigest)); + } + #endregion + + #region Auto Decompress + [TestMethod] + public void AutoDecompressSingle() + { + AutoDecompressTemplate("A.lzma", "A.pdf"); + AutoDecompressTemplate("B9.xz", "B.txt"); + AutoDecompressTemplate("B1_mt16.xz", "B.txt"); + AutoDecompressTemplate("C.lz", "C.bin"); + } + + private static void AutoDecompressTemplate(string lzmaFileName, string originFileName) + { + byte[] decompDigest; + byte[] originDigest; + + string xzFile = Path.Combine(TestSetup.SampleDir, lzmaFileName); + string originFile = Path.Combine(TestSetup.SampleDir, originFileName); + using (MemoryStream decompMs = new MemoryStream()) + { + XZDecompressOptions decompOpts = new XZDecompressOptions(); + + using (FileStream compFs = new FileStream(xzFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (LzmaAutoStream lzs = new LzmaAutoStream(compFs, decompOpts)) + { + lzs.CopyTo(decompMs); + + decompMs.Flush(); + lzs.GetProgress(out ulong finalIn, out ulong finalOut); + + Assert.AreEqual(compFs.Length, lzs.TotalIn); + Assert.AreEqual(decompMs.Length, lzs.TotalOut); + Assert.AreEqual((ulong)compFs.Length, finalIn); + Assert.AreEqual((ulong)decompMs.Length, finalOut); + } + + decompMs.Position = 0; + + using (HashAlgorithm hash = SHA256.Create()) + { + decompDigest = hash.ComputeHash(decompMs); + } + } + + using (FileStream originFs = new FileStream(originFile, FileMode.Open, FileAccess.Read, FileShare.Read)) + { + using (HashAlgorithm hash = SHA256.Create()) + { + originDigest = hash.ComputeHash(originFs); + } + } + + Assert.IsTrue(decompDigest.SequenceEqual(originDigest)); + } + #endregion + } +} diff --git a/Joveler.Compression.XZ/CHANGELOG.md b/Joveler.Compression.XZ/CHANGELOG.md index 966648d..760758e 100644 --- a/Joveler.Compression.XZ/CHANGELOG.md +++ b/Joveler.Compression.XZ/CHANGELOG.md @@ -2,6 +2,13 @@ ## v4.x +### v4.2.0 + +Released in 2023-02-16 + +- Supports xz-utils 5.4.1. +- Target .NET Framework 4.6 instead of [deprecated 4.5.1](https://devblogs.microsoft.com/dotnet/net-framework-4-5-2-4-6-4-6-1-will-reach-end-of-support-on-april-26-2022/). + ### v4.1.0 Released in 2021-04-05 diff --git a/Joveler.Compression.XZ/Checksum/Crc32.cs b/Joveler.Compression.XZ/Checksum/Crc32.cs index 5604eff..72f8acb 100644 --- a/Joveler.Compression.XZ/Checksum/Crc32.cs +++ b/Joveler.Compression.XZ/Checksum/Crc32.cs @@ -105,7 +105,7 @@ protected override void HashCore(byte[] array, int ibStart, int cbSize) _crc32.Append(array, ibStart, cbSize); } -#if NETSTANDARD2_1 +#if NETCOREAPP protected override void HashCore(ReadOnlySpan source) { _crc32.Append(source); diff --git a/Joveler.Compression.XZ/Checksum/Crc64.cs b/Joveler.Compression.XZ/Checksum/Crc64.cs index 8b22c3d..9f8c2f0 100644 --- a/Joveler.Compression.XZ/Checksum/Crc64.cs +++ b/Joveler.Compression.XZ/Checksum/Crc64.cs @@ -105,7 +105,7 @@ protected override void HashCore(byte[] array, int ibStart, int cbSize) _crc64.Append(array, ibStart, cbSize); } -#if NETSTANDARD2_1 +#if NETCOREAPP protected override void HashCore(ReadOnlySpan source) { _crc64.Append(source); diff --git a/Joveler.Compression.XZ/Joveler.Compression.XZ.csproj b/Joveler.Compression.XZ/Joveler.Compression.XZ.csproj index e00a1a9..3599154 100644 --- a/Joveler.Compression.XZ/Joveler.Compression.XZ.csproj +++ b/Joveler.Compression.XZ/Joveler.Compression.XZ.csproj @@ -1,34 +1,38 @@  - net462;netstandard2.0;netcoreapp3.1 + net46;netstandard2.0;netcoreapp3.1 netstandard2.0;netcoreapp3.1 latest true AnyCPU + true + Joveler.Compression.XZ Joveler.Compression.XZ - 4.1.0 + 4.2.0 Hajin Jang Joveler XZ Utils (liblzma) pinvoke library for .NET. The library provides the fast native implementation of XZStream. It also supports fast calculation of CRC32 and CRC64 checksum. Supports Windows, Linux and macOS. - Copyright (c) 2018-2021 Hajin Jang + Copyright (c) 2018-2023 Hajin Jang MIT https://github.com/ied206/Joveler.Compression images\Logo.png https://github.com/ied206/Joveler.Compression - - Official support for Windows ARM64. + - Support xz-utils 5.4.1. +- Target .NET Framework 4.6 instead of deprecated 4.5.1. xz xz-utils liblzma lzma lzma2 compress compression decompress decompression archive crc32 crc64 crc checksum native pinvoke interop + README.md - + - - + + x86\%(FileName)%(Extension) PreserveNewest @@ -37,8 +41,12 @@ Supports Windows, Linux and macOS. x64\%(FileName)%(Extension) PreserveNewest + + arm64\%(FileName)%(Extension) + PreserveNewest + - + runtimes\win-x86\native\%(FileName)%(Extension) @@ -75,10 +83,12 @@ Supports Windows, Linux and macOS. - - + + + + diff --git a/Joveler.Compression.XZ/Joveler.Compression.XZ.netfx.targets b/Joveler.Compression.XZ/Joveler.Compression.XZ.netfx.targets index 6d9447d..c49636d 100644 --- a/Joveler.Compression.XZ/Joveler.Compression.XZ.netfx.targets +++ b/Joveler.Compression.XZ/Joveler.Compression.XZ.netfx.targets @@ -32,6 +32,7 @@ HasTrailingSlash('$(MSBuildThisFileDirectory)')"> + - + CollectInteropLibFiles_x86; CollectInteropLibFiles_x64; + CollectInteropLibFiles_arm64; $(PipelineCollectFilesPhaseDependsOn); diff --git a/Joveler.Compression.XZ/NUGET_README.md b/Joveler.Compression.XZ/NUGET_README.md new file mode 100644 index 0000000..b12e2e4 --- /dev/null +++ b/Joveler.Compression.XZ/NUGET_README.md @@ -0,0 +1,52 @@ +# Joveler.Compression.XZ + +[XZ Utils](https://tukaani.org/xz/) (liblzma) pinvoke library for .NET. + +## Features + +- XZStream, the stream for [.xz file format](https://tukaani.org/xz/xz-file-format.txt). + +## Usage + +Please refer the [project homepage](https://github.com/ied206/Joveler.Compression/blob/master/Joveler.Compression.XZ/USAGE.md). + +## Support + +### Targeted .NET platforms + +- .NET Core 3.1 +- .NET Standard 2.0 +- .NET Framework 4.6 + +#### Discontinued frameworks + +| Platform | Last Supported Version | +|----------|------------------------| +| .NET Standard 1.3 | [v1.1.2](https://www.nuget.org/packages/Joveler.Compression.XZ/1.1.2) | +| .NET Framework 4.5.1 | [v4.1.0](https://www.nuget.org/packages/Joveler.Compression.XZ/4.1.0) | + +### Supported OS platforms + +| Platform | Architecture | Tested | +|----------|--------------|--------| +| Windows | x86 | Yes | +| | x64 | Yes | +| | arm64 | Yes | +| Linux | x64 | Yes | +| | armhf | Yes | +| | arm64 | Yes | +| macOS | x64 | Yes | +| | arm64 | Yes | + +#### Tested linux distributions + +| Architecture | Distribution | Note | +|---------------|--------------|------| +| x64 | Ubuntu 20.04 | Tested on AppVeyor CI | +| armhf | Debian 11 | Emulated on QEMU | +| arm64 | Debian 11 | Emulated on QEMU | + +### Supported XZ Utils version + +- 5.4.1 (Included) + diff --git a/Joveler.Compression.XZ/README.md b/Joveler.Compression.XZ/README.md index 0b28172..ac91768 100644 --- a/Joveler.Compression.XZ/README.md +++ b/Joveler.Compression.XZ/README.md @@ -16,11 +16,16 @@ Joveler.Compression.XZ can be installed via [nuget](https://www.nuget.org/packag ### Targeted .NET platforms -- .NET Standard 2.1 (.NET Core 3.0+) -- .NET Standard 2.0 (.NET Framework 4.6.1+, .NET Core 2.0+) -- .NET Framework 4.5.1 +- .NET Core 3.1 +- .NET Standard 2.0 +- .NET Framework 4.6 -If you need .NET Standard 1.3 support, use [v1.1.2](https://www.nuget.org/packages/Joveler.Compression.XZ/1.1.2) instead. +#### Discontinued frameworks + +| Platform | Last Supported Version | +|----------|------------------------| +| .NET Standard 1.3 | [v1.1.2](https://www.nuget.org/packages/Joveler.Compression.XZ/1.1.2) | +| .NET Framework 4.5.1 | [v4.1.0](https://www.nuget.org/packages/ManagedWimLib/4.1.0) | ### Supported OS platforms @@ -33,23 +38,19 @@ If you need .NET Standard 1.3 support, use [v1.1.2](https://www.nuget.org/packag | | armhf | Yes | | | arm64 | Yes | | macOS | x64 | Yes | - -macOS arm64 should be supported on theory, but I do not have access to an Apple Sillicon device to test. Please contribute if you have an ARM64 macOS machine. +| | arm64 | Yes | #### Tested linux distributions | Architecture | Distribution | Note | |---------------|--------------|------| -| x64 | Ubuntu 18.04 | Tested on AppVeyor CI | -| armhf | Debian 10 | Emulated on QEMU | -| arm64 | Debian 10 | Emulated on QEMU | +| x64 | Ubuntu 20.04 | Tested on AppVeyor CI | +| armhf | Debian 11 | Emulated on QEMU | +| arm64 | Debian 11 | Emulated on QEMU | ### Supported XZ Utils version -- 5.2.2 -- 5.2.3 -- 5.2.4 -- 5.2.5 (Included) +- 5.4.1 (Included) ## Usage diff --git a/Joveler.Compression.XZ/USAGE.md b/Joveler.Compression.XZ/USAGE.md index 562c300..039556f 100644 --- a/Joveler.Compression.XZ/USAGE.md +++ b/Joveler.Compression.XZ/USAGE.md @@ -4,39 +4,13 @@ Joveler.Compression.XZ requires explicit loading of the liblzma library. -You must call `XZInit.GlobalInit()` before using Joveler.Compression.XZ. Please put this code snippet in your application init code: - -#### For .NET Framework 4.5.1+ - -```cs -public static void InitNativeLibrary() -{ - string arch = null; - switch (RuntimeInformation.ProcessArchitecture) - { - case Architecture.X86: - arch = "x86"; - break; - case Architecture.X64: - arch = "x64"; - break; - case Architecture.Arm: - arch = "armhf"; - break; - case Architecture.Arm64: - arch = "arm64"; - break; - } - string libPath = Path.Combine(arch, "liblzma.dll"); +### Init Code Snippet - if (!File.Exists(libPath)) - throw new PlatformNotSupportedException($"Unable to find native library [{libPath}]."); +You must call `XZInit.GlobalInit()` before using Joveler.Compression.XZ. Please put this code snippet in your application init code: - Magic.GlobalInit(libPath); -} -``` +**WARNING**: The caller process and callee library must have the same architecture! -#### For .NET Standard 2.0+: +#### On .NET Core ```cs public static void InitNativeLibrary() @@ -83,18 +57,52 @@ public static void InitNativeLibrary() } ``` -**WARNING**: Caller process and callee library must have the same architecture! +#### On .NET Framework + +```cs +public static void InitNativeLibrary() +{ + string arch = null; + switch (RuntimeInformation.ProcessArchitecture) + { + case Architecture.X86: + arch = "x86"; + break; + case Architecture.X64: + arch = "x64"; + break; + case Architecture.Arm: + arch = "armhf"; + break; + case Architecture.Arm64: + arch = "arm64"; + break; + } + string libPath = Path.Combine(arch, "liblzma.dll"); + + if (!File.Exists(libPath)) + throw new PlatformNotSupportedException($"Unable to find native library [{libPath}]."); + + Magic.GlobalInit(libPath); +} +``` ### Embedded binary -Joveler.Compression.XZ comes with sets of static binaries of `liblzma 5.2.5`. They will be copied into the build directory at build time. +Joveler.Compression.XZ comes with sets of static binaries of `liblzma 5.4.1`. They will be copied into the build directory at build time. -#### For .NET Framework 4.5.1+ +#### On .NET Standard & .NET Core -| Platform | Binary | Note | -|------------------|-----------------------------|-----------------| -| Windows x86 | `$(OutDir)\x86\liblzma.dll` | Official binary | -| Windows x64 | `$(OutDir)\x64\liblzma.dll` | Official binary | +| Platform | Binary | License | Note | +|----------------------|----------------------------------------------|---------------|---------------| +| Windows x86 | `$(OutDir)\runtimes\win-x86\liblzma.dll` | Public Domain | Universal CRT | +| Windows x64 | `$(OutDir)\runtimes\win-x64\liblzma.dll` | Public Domain | Universal CRT | +| Windows arm64 | `$(OutDir)\runtimes\win-arm64\liblzma.dll` | Public Domain | Universal CRT | +| Ubuntu 20.04 x64 | `$(OutDir)\runtimes\linux-x64\liblzma.so` | Public Domain | glibc | +| Debian 11 armhf | `$(OutDir)\runtimes\linux-arm\liblzma.so` | Public Domain | glibc | +| Debian 11 arm64 | `$(OutDir)\runtimes\linux-arm64\liblzma.so` | Public Domain | glibc | +| macOS Big Sur x64 | `$(OutDir)\runtimes\osx-x64\liblzma.dylib` | Public Domain | libSystem | +| macOS Ventura arm64 | `$(OutDir)\runtimes\osx-arm64\liblzma.dylib` | Public Domain | libSystem | - Create an empty file named `Joveler.Compression.XZ.Precompiled.Exclude` in the project directory to prevent copy of the package-embedded binary. @@ -117,11 +125,21 @@ Joveler.Compression.XZ comes with sets of static binaries of `liblzma 5.2.5`. Th - Linux binaries are not portable. They may not work on your distribution. - You may call parameter-less `XZInit.GlobalInit()` to use system-installed liblzma. +#### On .NET Framework + +| Platform | Binary | License | C Runtime | +|------------------|-------------------------------|---------------|---------------| +| Windows x86 | `$(OutDir)\x86\liblzma.dll` | Public Domain | Universal CRT | +| Windows x64 | `$(OutDir)\x64\liblzma.dll` | Public Domain | Universal CRT | +| Windows arm64 | `$(OutDir)\arm64\liblzma.dll` | Public Domain | Universal CRT | + +- Create an empty file named `Joveler.Compression.XZ.Precompiled.Exclude` in the project directory to prevent copying of the package-embedded binary. + ### Custom binary -To use custom liblzma binary instead, call `XZInit.GlobalInit()` with a path to the custom binary. +To use the custom liblzma binary instead, call `XZInit.GlobalInit()` with a path to the custom binary. -### Cleanup +## Cleanup To unload the liblzma library explicitly, call `XZInit.GlobalCleanup()`. @@ -132,12 +150,14 @@ To unload the liblzma library explicitly, call `XZInit.GlobalCleanup()`. ### Constructor ```csharp -// Create a compressing XZStream instance +// Create a compressing XZStream instance. public XZStream(Stream baseStream, XZCompressOptions compOpts) -// Create a multi-threaded compressing XZStream instance +// Create a multi-threaded compressing XZStream instance. public XZStream(Stream baseStream, XZCompressOptions compOpts, XZThreadedCompressOptions threadOpts) -// Create a decompressing XZStream instance +// Create a decompressing XZStream instance. public XZStream(Stream baseStream, XZDecompressOptions decompOpts) +// Create a multi-threaded decompressing XZStream instance. +public XZStream(Stream baseStream, XZDecompressOptions decompOpts, XZThreadedDecompressOptions threadOpts) ``` #### XZCompressOptions @@ -151,9 +171,9 @@ You can tune xz compress options with this class. | BufferSize | Size of the internal buffer. The default is 1MB. | | LeaveOpen | Whether to leave the base stream object open after disposing of the xz stream object. | -It also contains more advanced options. +It may contain more advanced options. -**NOTE**: xz file created in single-threaded mode will not be able to be decompressed in parallel in the future versions of xz-utils. It is because xz-utils does not divide the compressed stream into blocks when the multi-threaded compression is not enabled. +**NOTE**: xz file created in single-threaded mode cannot be decompressed in parallel. xz-utils does not divide the compressed stream into blocks when the multi-threaded compression is not enabled. **WARNING**: Beware of high memory usage at a high compression level. @@ -174,15 +194,17 @@ It also contains more advanced options. If you want to compress in parallel, pass an instance of this class to the `XZStream` constructor. -| Property | Summary | -|----------|---------| -| Threads | Number of worker threads to use. | +| Property | Summary | +|-----------|---------| +| Threads | Number of worker threads to use. | + +It may contain more advanced options. -It also contains more advanced options. +**NOTE**: You must use threaded compression to let xz-utils decompress in parallel, even if you are using only 1 thread. -**NOTE**: When you create XZStream with this parameter, the future versions of xz-utils may be able to be decompressed created xz file in parallel. It is even true when you used only 1 thread with threaded compression. It is because xz-utils only divide the compressed stream into blocks in threaded compression. +**WARNING**: If possible, always check the available system memory. Modern CPUs have a lot of cores, and each thread will allocate its buffer. -**WARNING**: In multi-threaded compression, each thread may allocate more memory than the single-thread mode. It is true even if you run multi-threaded mode with 1 thread because xz-utils aggressively buffers input and output in parallel compression. Use `XZInit.EncoderMemUsage()` to check exact memory requirement for your config. +**WARNING**: In multi-threaded compression, each thread may allocate more memory than the single-thread mode, including compressing in 1 thread. xz-utils aggressively buffers input and output in parallel compression. Use `XZInit.EncoderMemUsage()` to check the exact memory requirement for your config. #### XZDecompressOptions @@ -195,11 +217,24 @@ You can tune xz decompress options with this class. It also contains more advanced options. -**WARNING**: Threaded decompression is not supported yet in the xz library. +#### XZThreadedDecompressOptions + +If you want to decompress in parallel, pass an instance of this class to the `XZStream` constructor. + +| Property | Summary | +|-----------|---------| +| Threads | Number of worker threads to use. | +| MemlimitThreading | Memory usage soft limit to reduce the number of threads. | + +It may contain more advanced options. + +**WARNING**: xz-utils can decompress an xz file only if it had been compressed in parallel. + +**WARNING**: Threaded decompression will take more memory! Tweak `MemlimitThreading` to pass a guideline of maximum memory usage. ### Examples -#### Compress file to .xz +#### Compress a file to .xz ```csharp using Joveler.Compression.XZ; @@ -218,7 +253,7 @@ using (XZStream zs = new XZStream(fsComp, compOpts)) } ``` -#### Compress file to .xz in parallel +#### Compress a file to .xz in parallel ```csharp using Joveler.Compression.XZ; @@ -242,7 +277,7 @@ using (XZStream zs = new XZStream(fsComp, compOpts, threadOpts)) } ``` -#### Decompress file from .xz +#### Decompress a file from .xz ```csharp using Joveler.Compression.XZ; @@ -251,7 +286,38 @@ XZDecompressOptions decompOpts = new XZDecompressOptions(); using (FileStream fsComp = new FileStream("test.xz", FileMode.Create)) using (FileStream fsDecomp = new FileStream("file_decomp.bin", FileMode.Open)) -using (XZStream zs = new XZStream(fsComp, LzmaMode.Decompress)) +using (XZStream zs = new XZStream(fsCompp, decompOpts)) +{ + zs.CopyTo(fsDecomp); +} +``` + +#### Decompress a file from .xz in parallel + +```csharp +using Joveler.Compression.XZ; + +XZDecompressOptions decompOpts = new XZDecompressOptions(); +XZThreadedDecompressOptions threadOpts = new XZThreadedDecompressOptions +{ + Threads = Environment.ProcesserCount, +}; + +// Limit maximum memory liblzma is allowed to use. +// The following values are taken from the xz CLI program code. +switch (XZInit.Lib.PlatformBitness) +{ + case DynLoader.PlatformBitness.Bit32: + threadOpts.MemlimitThreading = Math.Min(XZHardware.PhysMem() / 4, 1400U << 20); + break; + case DynLoader.PlatformBitness.Bit64: + threadOpts.MemlimitThreading = XZHardware.PhysMem() / 4; + break; +} + +using (FileStream fsComp = new FileStream("test.xz", FileMode.Create)) +using (FileStream fsDecomp = new FileStream("file_decomp.bin", FileMode.Open)) +using (XZStream zs = new XZStream(fsComp, decompOpts, threadOpts)) { zs.CopyTo(fsDecomp); } @@ -261,11 +327,11 @@ using (XZStream zs = new XZStream(fsComp, LzmaMode.Decompress)) `Crc32Checksum` is the class designed to compute CRC32 checksum. -Use `Append()` methods to compute the checksum. -Use `Checksum` property to get checksum value. -Use `Reset()` methods to reset `Checksum` property. +- Use `Append()` method to compute the checksum. +- Use `Checksum` property to get the checksum value. +- Use `Reset()` method to reset `Checksum` property. -**NOTE**: xz-utils provides about twice faster CRC32 implementation than zlib. +**NOTE**: xz-utils provides about twice faster CRC32 implementation than zlib, in my benchmark with [Joveler.Compression.ZLib](https://www.nuget.org/packages/Joveler.Compression.ZLib). ### Examples @@ -279,17 +345,17 @@ byte[] bin = Encoding.UTF8.GetBytes("ABCDEF"); // Append(ReadOnlySpan buffer) crc.Append(bin.AsSpan(2, 3)); -Console.WriteLine($"0x{crc.Checksum:X8}"); +Console.WriteLine($"0x{crc.Checksum:X4}"); // Append(byte[] buffer, int offset, int count) crc.Reset(); crc.Append(bin, 2, 3); -Console.WriteLine($"0x{crc.Checksum:X8}"); +Console.WriteLine($"0x{crc.Checksum:X4}"); ``` #### `Append(Stream stream)` -```cs +```csharp using Joveler.Compression.XZ.Checksum; using (FileStream fs = new FileStream("read.txt", FileMode.Open)) @@ -298,7 +364,7 @@ using (FileStream fs = new FileStream("read.txt", FileMode.Open)) // Append(Stream stream) crc.Append(fs); - Console.WriteLine($"0x{crc.Checksum:X8}"); + Console.WriteLine($"0x{crc.Checksum:X4}"); } ``` @@ -307,3 +373,53 @@ using (FileStream fs = new FileStream("read.txt", FileMode.Open)) `Crc32Algorithm` is the class designed to compute CRC32 checksum. It inherits and implements [HashAlgorithm](https://docs.microsoft.com/en-US/dotnet/api/system.security.cryptography.hashalgorithm). + + +## Crc64Checksum + +`Crc64Checksum` is the class designed to compute CRC64 checksum. + +Use `Append()` method to compute the checksum. +Use `Checksum` property to get the checksum value. +Use `Reset()` method to reset `Checksum` property. + +### Examples + +#### `Append(ReadOnlySpan buffer)`, `Append(byte[] buffer, int offset, int count)` + +```cs +using Joveler.Compression.XZ.Checksum; + +Crc64Checksum crc = new Crc64Checksum(); +byte[] bin = Encoding.UTF8.GetBytes("ABCDEF"); + +// Append(ReadOnlySpan buffer) +crc.Append(bin.AsSpan(2, 3)); +Console.WriteLine($"0x{crc.Checksum:X8}"); + +// Append(byte[] buffer, int offset, int count) +crc.Reset(); +crc.Append(bin, 2, 3); +Console.WriteLine($"0x{crc.Checksum:X8}"); +``` + +#### `Append(Stream stream)` + +```csharp +using Joveler.Compression.XZ.Checksum; + +using (FileStream fs = new FileStream("read.txt", FileMode.Open)) +{ + Crc64Checksum crc = new Crc64Checksum(); + + // Append(Stream stream) + crc.Append(fs); + Console.WriteLine($"0x{crc.Checksum:X8}"); +} +``` + +## Crc64Algorithm + +`Crc64Algorithm` is the class designed to compute CRC64 checksum. + +It inherits and implements [HashAlgorithm](https://docs.microsoft.com/en-US/dotnet/api/system.security.cryptography.hashalgorithm). diff --git a/Joveler.Compression.XZ/XZException.cs b/Joveler.Compression.XZ/XZException.cs index 1db9a73..253ed22 100644 --- a/Joveler.Compression.XZ/XZException.cs +++ b/Joveler.Compression.XZ/XZException.cs @@ -2,7 +2,7 @@ Derived from liblzma header files (Public Domain) C# Wrapper written by Hajin Jang - Copyright (C) 2018-2020 Hajin Jang + Copyright (C) 2018-2023 Hajin Jang MIT License @@ -27,13 +27,11 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE using System; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; using System.Runtime.Serialization; namespace Joveler.Compression.XZ { [Serializable] - [SuppressMessage("ReSharper", "InconsistentNaming")] public class XZException : Exception { public LzmaRet ReturnCode { get; set; } @@ -56,10 +54,27 @@ public XZException(LzmaRet ret) : base(GetErrorMessage(ret)) ReturnCode = ret; } - public static void CheckReturnValue(LzmaRet ret) + public static void CheckReturnValueNormal(LzmaRet ret) { - if (ret != LzmaRet.Ok) - throw new XZException(ret); + switch (ret) + { + case LzmaRet.Ok: + break; + default: + throw new XZException(ret); + } + } + + public static void CheckReturnValueDecompress(LzmaRet ret) + { + switch (ret) + { + case LzmaRet.Ok: + case LzmaRet.SeekNeeded: + break; + default: + throw new XZException(ret); + } } #region Serializable diff --git a/Joveler.Compression.XZ/XZHardware.cs b/Joveler.Compression.XZ/XZHardware.cs new file mode 100644 index 0000000..07d603d --- /dev/null +++ b/Joveler.Compression.XZ/XZHardware.cs @@ -0,0 +1,63 @@ +/* + Derived from liblzma header files (Public Domain) + + C# Wrapper written by Hajin Jang + Copyright (C) 2018-2023 Hajin Jang + + MIT License + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +using System; + +namespace Joveler.Compression.XZ +{ + public class XZHardware + { + #region Hardware - PhysMem & CPU Threads + public static ulong PhysMem() + { + XZInit.Manager.EnsureLoaded(); + + return XZInit.Lib.LzmaPhysMem(); + } + + public static uint CpuThreads() + { + XZInit.Manager.EnsureLoaded(); + + return XZInit.Lib.LzmaCpuThreads(); + } + #endregion + + #region (internal) Thread Limiter + internal static uint CheckThreadCount(int threads) + { + if (threads < 0) + throw new ArgumentOutOfRangeException(nameof(threads)); + if (threads == 0) // Use system's thread number by default + threads = Environment.ProcessorCount; + else if (Environment.ProcessorCount < threads) // If the number of CPU cores/threads exceeds system thread number, + threads = Environment.ProcessorCount; // Limit the number of threads to keep memory usage lower. + return (uint)threads; + } + #endregion + } +} diff --git a/Joveler.Compression.XZ/XZInit.cs b/Joveler.Compression.XZ/XZInit.cs index 870071c..26f7d1c 100644 --- a/Joveler.Compression.XZ/XZInit.cs +++ b/Joveler.Compression.XZ/XZInit.cs @@ -2,7 +2,7 @@ Derived from liblzma header files (Public Domain) C# Wrapper written by Hajin Jang - Copyright (C) 2018-2020 Hajin Jang + Copyright (C) 2018-2023 Hajin Jang MIT License @@ -82,19 +82,17 @@ public static string VersionString() } #endregion - #region Hardware - PhysMem & CPU Threads + #region Hardware - PhysMem & CPU Threads (Obsolete) + [Obsolete($"Replaced to {nameof(XZHardware)}.{nameof(PhysMem)}().")] public static ulong PhysMem() { - Manager.EnsureLoaded(); - - return Lib.LzmaPhysMem(); + return XZHardware.PhysMem(); } + [Obsolete($"Replaced to {nameof(XZHardware)}.{nameof(CpuThreads)}().")] public static uint CpuThreads() { - Manager.EnsureLoaded(); - - return Lib.LzmaCpuThreads(); + return XZHardware.CpuThreads(); } #endregion @@ -106,6 +104,7 @@ public static uint CpuThreads() /// Number of bytes of memory required for the given preset when encoding. /// If an error occurs, for example due to unsupported preset, UINT64_MAX is returned. /// + [Obsolete($"Replaced to {nameof(XZMemory)}.{nameof(XZMemory.EncoderMemUsage)}().")] public static ulong EncoderMemUsage(LzmaCompLevel level, bool extremeFlag) { Manager.EnsureLoaded(); @@ -121,6 +120,7 @@ public static ulong EncoderMemUsage(LzmaCompLevel level, bool extremeFlag) /// Number of bytes of memory required for the given preset when encoding. /// If an error occurs, for example due to unsupported preset, UINT64_MAX is returned. /// + [Obsolete($"Replaced to {nameof(XZMemory)}.{nameof(XZMemory.EncoderMemUsage)}().")] public static ulong EncoderMemUsage(XZCompressOptions compOpts) { Manager.EnsureLoaded(); @@ -135,13 +135,10 @@ public static ulong EncoderMemUsage(XZCompressOptions compOpts) /// Number of bytes of memory required for encoding with the given options. /// If an error occurs, for example due to unsupported preset or filter chain, UINT64_MAX is returned. /// + [Obsolete($"Replaced to {nameof(XZMemory)}.{nameof(XZMemory.ThreadedEncoderMemUsage)}().")] public static ulong EncoderMultiMemUsage(LzmaCompLevel level, bool extremeFlag, int threads) { - Manager.EnsureLoaded(); - - uint preset = XZCompressOptions.ToPreset(level, extremeFlag); - LzmaMt mtOpts = LzmaMt.Create(preset, threads); - return Lib.LzmaStreamEncoderMtMemUsage(mtOpts); + return XZMemory.ThreadedEncoderMemUsage(level, extremeFlag, threads); } /// @@ -151,12 +148,10 @@ public static ulong EncoderMultiMemUsage(LzmaCompLevel level, bool extremeFlag, /// Number of bytes of memory required for encoding with the given options. /// If an error occurs, for example due to unsupported preset or filter chain, UINT64_MAX is returned. /// + [Obsolete($"Replaced to {nameof(XZMemory)}.{nameof(XZMemory.ThreadedEncoderMemUsage)}().")] public static ulong EncoderMultiMemUsage(XZCompressOptions compOpts, XZThreadedCompressOptions threadOpts) { - Manager.EnsureLoaded(); - - LzmaMt mtOpts = compOpts.ToLzmaMt(threadOpts); - return Lib.LzmaStreamEncoderMtMemUsage(mtOpts); + return XZMemory.ThreadedEncoderMemUsage(compOpts, threadOpts); } /// @@ -166,12 +161,10 @@ public static ulong EncoderMultiMemUsage(XZCompressOptions compOpts, XZThreadedC /// Number of bytes of memory required to decompress a file that was compressed using the given preset. /// If an error occurs, for example due to unsupported preset, UINT64_MAX is returned. /// + [Obsolete($"Replaced to {nameof(XZMemory)}.{nameof(XZMemory.DecoderMemUsage)}(). Planned to be removed in next major update.")] public static ulong DecoderMemUsage(LzmaCompLevel level, bool extremeFlag) { - Manager.EnsureLoaded(); - - uint preset = XZCompressOptions.ToPreset(level, extremeFlag); - return Lib.LzmaEasyDecoderMemUsage(preset); + return XZMemory.DecoderMemUsage(level, extremeFlag); } /// @@ -181,11 +174,10 @@ public static ulong DecoderMemUsage(LzmaCompLevel level, bool extremeFlag) /// Number of bytes of memory required to decompress a file that was compressed using the given preset. /// If an error occurs, for example due to unsupported preset, UINT64_MAX is returned. /// + [Obsolete($"Replaced to {nameof(XZMemory)}.{nameof(XZMemory.DecoderMemUsage)}. Planned to be removed in next major update.")] public static ulong DecoderMemUsage(XZCompressOptions compOpts) { - Manager.EnsureLoaded(); - - return Lib.LzmaEasyDecoderMemUsage(compOpts.Preset); + return XZMemory.DecoderMemUsage(compOpts); } #endregion } diff --git a/Joveler.Compression.XZ/XZLoadManager.cs b/Joveler.Compression.XZ/XZLoadManager.cs index 6b45961..83a0d80 100644 --- a/Joveler.Compression.XZ/XZLoadManager.cs +++ b/Joveler.Compression.XZ/XZLoadManager.cs @@ -1,5 +1,5 @@ /* - Copyright (C) 2018-2020 Hajin Jang + Copyright (C) 2018-2023 Hajin Jang MIT License diff --git a/Joveler.Compression.XZ/XZLoader.cs b/Joveler.Compression.XZ/XZLoader.cs index 4bd8078..ac77139 100644 --- a/Joveler.Compression.XZ/XZLoader.cs +++ b/Joveler.Compression.XZ/XZLoader.cs @@ -2,7 +2,7 @@ Derived from liblzma header files (Public Domain) C# Wrapper written by Hajin Jang - Copyright (C) 2018-2020 Hajin Jang + Copyright (C) 2018-2023 Hajin Jang MIT License @@ -42,7 +42,7 @@ protected override string DefaultLibFileName { get { -#if !NET451 +#if !NETFRAMEWORK if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return "liblzma.so.5"; else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) @@ -70,6 +70,10 @@ protected override void LoadFunctions() LzmaStreamEncoderMtMemUsage = GetFuncPtr(nameof(lzma_stream_encoder_mt_memusage)); LzmaStreamEncoderMt = GetFuncPtr(nameof(lzma_stream_encoder_mt)); LzmaStreamDecoder = GetFuncPtr(nameof(lzma_stream_decoder)); + LzmaStreamDecoderMt = GetFuncPtr(nameof(lzma_stream_decoder_mt)); + LzmaAloneDecoder = GetFuncPtr(nameof(lzma_alone_decoder)); + LzmaLZipDecoder = GetFuncPtr(nameof(lzma_lzip_decoder)); + LzmaAutoDecoder = GetFuncPtr(nameof(lzma_auto_decoder)); #endregion #region Hardware - PhyMem & CPU Threads @@ -77,6 +81,14 @@ protected override void LoadFunctions() LzmaCpuThreads = GetFuncPtr(nameof(lzma_cputhreads)); #endregion + #region Memory - Memusage, MemlimitGet, MemlimitSet (DISABLED) +#if LZMA_MEM_ENABLE + LzmaMemusage = GetFuncPtr(nameof(lzma_memusage)); + LzmaMemlimitGet = GetFuncPtr(nameof(lzma_memlimit_get)); + LzmaMemlimitSet = GetFuncPtr(nameof(lzma_memlimit_set)); +#endif + #endregion + #region Check - Crc32, Crc64 LzmaCrc32 = GetFuncPtr(nameof(lzma_crc32)); LzmaCrc64 = GetFuncPtr(nameof(lzma_crc64)); @@ -101,6 +113,10 @@ protected override void ResetFunctions() LzmaStreamEncoder = null; LzmaStreamEncoderMt = null; LzmaStreamDecoder = null; + LzmaStreamDecoderMt = null; + LzmaAloneDecoder = null; + LzmaLZipDecoder = null; + LzmaAutoDecoder = null; #endregion #region Hardware - PhyMem & CPU Threads @@ -108,6 +124,14 @@ protected override void ResetFunctions() LzmaCpuThreads = null; #endregion + #region Memory - Memusage, MemlimitGet, MemlimitSet (DISABLED) +#if LZMA_MEM_ENABLE + LzmaMemusage = null; + LzmaMemlimitGet = null; + LzmaMemlimitSet = null; +#endif + #endregion + #region Check - Crc32, Crc64 LzmaCrc32 = null; LzmaCrc64 = null; @@ -173,6 +197,66 @@ internal delegate LzmaRet lzma_stream_encoder_mt( LzmaMt options); internal lzma_stream_encoder_mt LzmaStreamEncoderMt; +#if LZMA_MICROLZMA_ENABLE + /// + /// MicroLZMA encoder + /// + /// + /// The MicroLZMA format is a raw LZMA stream whose first byte (always 0x00) + /// has been replaced with bitwise-negation of the LZMA properties (lc/lp/pb). + /// This encoding ensures that the first byte of MicroLZMA stream is never + /// 0x00. There is no end of payload marker and thus the uncompressed size + /// must be stored separately. For the best error detection the dictionary + /// size should be stored separately as well but alternatively one may use + /// the uncompressed size as the dictionary size when decoding. + /// + /// With the MicroLZMA encoder, lzma_code() behaves slightly unusually. + /// The action argument must be LZMA_FINISH and the return value will never be + /// LZMA_OK. Thus the encoding is always done with a single lzma_code() after + /// the initialization. The benefit of the combination of initialization + /// function and lzma_code() is that memory allocations can be re-used for + /// better performance. + /// + /// lzma_code() will try to encode as much input as is possible to fit into + /// the given output buffer. If not all input can be encoded, the stream will + /// be finished without encoding all the input. The caller must check both + /// input and output buffer usage after lzma_code() (total_in and total_out + /// in lzma_stream can be convenient). Often lzma_code() can fill the output + /// buffer completely if there is a lot of input, but sometimes a few bytes + /// may remain unused because the next LZMA symbol would require more space. + /// + /// lzma_stream.avail_out must be at least 6. Otherwise LZMA_PROG_ERROR + /// will be returned. + /// + /// The LZMA dictionary should be reasonably low to speed up the encoder + /// re-initialization. A good value is bigger than the resulting + /// uncompressed size of most of the output chunks. For example, if output + /// size is 4 KiB, dictionary size of 32 KiB or 64 KiB is good. If the + /// data compresses extremely well, even 128 KiB may be useful. + /// + /// The MicroLZMA format and this encoder variant were made with the EROFS + /// file system in mind. This format may be convenient in other embedded + /// uses too where many small streams are needed. XZ Embedded includes a + /// decoder for this format. + /// + /// + /// - LZMA_STREAM_END: All good. Check the amounts of input used + /// and output produced. Store the amount of input used + /// (uncompressed size) as it needs to be known to decompress + /// the data. + /// - LZMA_OPTIONS_ERROR + /// - LZMA_MEM_ERROR + /// - LZMA_PROG_ERROR: In addition to the generic reasons for this + /// error code, this may also be returned if there isn't enough + /// output space (6 bytes) to create a valid MicroLZMA stream. + /// + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + internal delegate LzmaRet lzma_microlzma_encoder( + LzmaStream strm, + lzma_options_lzma options); + internal lzma_microlzma_encoder LzmaMicroLzmaEncoder; +#endif + /// /// Initialize .xz Stream decoder /// @@ -183,6 +267,9 @@ internal delegate LzmaRet lzma_stream_encoder_mt( /// /// /// Bitwise-or of zero or more of the decoder flags + /// LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, + /// LZMA_TELL_ANY_CHECK, LZMA_IGNORE_CHECK, + /// LZMA_CONCATENATED, LZMA_FAIL_FAST /// /// /// LZMA_OK: Initialization was successful. @@ -196,6 +283,172 @@ internal delegate LzmaRet lzma_stream_decoder( ulong memlimit, LzmaDecodingFlag flags); internal lzma_stream_decoder LzmaStreamDecoder; + + /// + /// Initialize multithreaded .xz Stream decoder + /// + /// Pointer to properly prepared lzma_stream + /// Pointer to multithreaded compression options + /// + /// The decoder can decode multiple Blocks in parallel. This requires that each + /// Block Header contains the Compressed Size and Uncompressed size fields + /// which are added by the multi-threaded encoder, see lzma_stream_encoder_mt(). + /// + /// A Stream with one Block will only utilize one thread. A Stream with multiple + /// Blocks but without size information in Block Headers will be processed in + /// single-threaded mode in the same way as done by lzma_stream_decoder(). + /// Concatenated Streams are processed one Stream at a time; no inter-Stream + /// parallelization is done. + /// + /// This function behaves like lzma_stream_decoder() when options->threads == 1 + /// and options->memlimit_threading <= 1. + /// + /// + /// LZMA_OK: Initialization was successful. + /// LZMA_MEM_ERROR: Cannot allocate memory. + /// LZMA_MEMLIMIT_ERROR: Memory usage limit was reached. + /// LZMA_OPTIONS_ERROR: Unsupported flags. + /// LZMA_PROG_ERROR + /// + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + internal delegate LzmaRet lzma_stream_decoder_mt( + LzmaStream strm, + LzmaMt options); + internal lzma_stream_decoder_mt LzmaStreamDecoderMt; + + /// + /// Decode .xz, .lzma, and .lz (lzip) files with autodetection + /// + /// + /// This decoder autodetects between the .xz, .lzma, and .lz file formats, + /// and calls lzma_stream_decoder(), lzma_alone_decoder(), or + /// lzma_lzip_decoder() once the type of the input file has been detected. + /// + /// Support for .lz was added in 5.4.0. + /// + /// If the flag LZMA_CONCATENATED is used and the input is a .lzma file: + /// For historical reasons concatenated .lzma files aren't supported. + /// If there is trailing data after one .lzma stream, lzma_code() will + /// return LZMA_DATA_ERROR. (lzma_alone_decoder() doesn't have such a check + /// as it doesn't support any decoder flags. It will return LZMA_STREAM_END + /// after one .lzma stream.) + /// + /// + /// Pointer to properly prepared lzma_stream + /// + /// + /// Memory usage limit as bytes. Use to effectively disable the limiter. + /// liblzma 5.2.3 and earlier don't allow 0 here and return ; + /// later versions treat 0 as if 1 had been specified. + /// + /// + /// Bitwise-or of zero or more of the decoder flags: + /// , , + /// , , + /// , + /// + /// + /// - : Initialization was successful. + /// - : Cannot allocate memory. + /// - : Unsupported flags + /// - + /// + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + internal delegate LzmaRet lzma_auto_decoder( + LzmaStream strm, + ulong memlimit, + LzmaDecodingFlag flags); + internal lzma_auto_decoder LzmaAutoDecoder; + + /// + /// Initialize .lzma decoder (legacy file format) + /// + /// + /// Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + /// There is no need to use LZMA_FINISH, but it's allowed because it may + /// simplify certain types of applications. + /// + /// + /// Pointer to properly prepared lzma_stream + /// + /// + /// Memory usage limit as bytes. Use to effectively disable the limiter. + /// liblzma 5.2.3 and earlier don't allow 0 here and return LZMA_PROG_ERROR; later versions treat 0 as if 1 had been specified. + /// + /// + /// - + /// - + /// - + /// + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + internal delegate LzmaRet lzma_alone_decoder( + LzmaStream strm, + ulong memlimit); + internal lzma_alone_decoder LzmaAloneDecoder; + + /// + /// Initialize .lz (lzip) decoder (a foreign file format) + /// + /// + /// This decoder supports the .lz format version 0 and the unextended .lz + /// format version 1: + /// + /// - Files in the format version 0 were produced by lzip 1.3 and older. + /// Such files aren't common but may be found from file archives + /// as a few source packages were released in this format. People + /// might have old personal files in this format too. Decompression + /// support for the format version 0 was removed in lzip 1.18. + /// + /// - lzip 1.3 added decompression support for .lz format version 1 files. + /// Compression support was added in lzip 1.4. In lzip 1.6 the .lz format + /// version 1 was extended to support the Sync Flush marker. This extension + /// is not supported by liblzma. lzma_code() will return LZMA_DATA_ERROR + /// at the location of the Sync Flush marker. In practice files with + /// the Sync Flush marker are very rare and thus liblzma can decompress + /// almost all .lz files. + /// + /// Just like with lzma_stream_decoder() for .xz files, LZMA_CONCATENATED + /// should be used when decompressing normal standalone .lz files. + /// + /// The .lz format allows putting non-.lz data at the end of a file after at + /// least one valid .lz member. That is, one can append custom data at the end + /// of a .lz file and the decoder is required to ignore it. In liblzma this + /// is relevant only when LZMA_CONCATENATED is used. In that case lzma_code() + /// will return LZMA_STREAM_END and leave lzma_stream.next_in pointing to + /// the first byte of the non-.lz data. An exception to this is if the first + /// 1-3 bytes of the non-.lz data are identical to the .lz magic bytes + /// (0x4C, 0x5A, 0x49, 0x50; "LZIP" in US-ASCII). In such a case the 1-3 bytes + /// will have been ignored by lzma_code(). If one wishes to locate the non-.lz + /// data reliably, one must ensure that the first byte isn't 0x4C. Actually + /// one should ensure that none of the first four bytes of trailing data are + /// equal to the magic bytes because lzip >= 1.20 requires it by default. + /// + /// + /// Pointer to properly prepared lzma_stream + /// + /// + /// Memory usage limit as bytes. Use UINT64_MAX to effectively disable the limiter. + /// + /// + /// Bitwise-or of flags, or zero for no flags. + /// All decoder flags listed above are supported although only LZMA_CONCATENATED + /// and (in very rare cases) LZMA_IGNORE_CHECK are actually useful. + /// LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, and LZMA_FAIL_FAST do nothing. + /// LZMA_TELL_ANY_CHECK is supported for consistency only as CRC32 is + /// always used in the .lz format. + /// + /// + /// - : Initialization was successful. + /// - : Cannot allocate memory. + /// - : Unsupported flags + /// - + /// + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + internal delegate LzmaRet lzma_lzip_decoder( + LzmaStream strm, + ulong memlimit, + LzmaDecodingFlag flags); + internal lzma_lzip_decoder LzmaLZipDecoder; #endregion #region Hardware - PhyMem & CPU Threads @@ -208,6 +461,73 @@ internal delegate LzmaRet lzma_stream_decoder( internal lzma_cputhreads LzmaCpuThreads; #endregion + #region Memory - Memusage, MemlimitGet, MemlimitSet (DISABLED) +#if LZMA_MEM_ENABLE + /// + /// Get the memory usage of decoder filter chain + /// + /// + /// This function is currently supported only when *strm has been initialized + /// with a function that takes a memlimit argument. With other functions, you + /// should use e.g. lzma_raw_encoder_memusage() or lzma_raw_decoder_memusage() + /// to estimate the memory requirements. + /// + /// This function is useful e.g. after LZMA_MEMLIMIT_ERROR to find out how big + /// the memory usage limit should have been to decode the input. Note that + /// this may give misleading information if decoding .xz Streams that have + /// multiple Blocks, because each Block can have different memory requirements. + /// + /// + /// How much memory is currently allocated for the filter decoders. + /// If no filter chain is currently allocated, some non-zero value is still returned, + /// which is less than or equal to what any filter chain would indicate as its memory requirement. + /// + /// If this function isn't supported by *strm or some other error occurs, zero is returned. + /// + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + internal delegate ulong lzma_memusage(LzmaStream strm); + internal lzma_memusage LzmaMemusage; + + /// + /// This function is supported only when *strm has been initialized with + /// a function that takes a memlimit argument. + /// + /// + /// On success, the current memory usage limit is returned + /// (always non-zero). On error, zero is returned. + /// + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + internal delegate ulong lzma_memlimit_get(LzmaStream strm); + internal lzma_memlimit_get LzmaMemlimitGet; + + /// + /// Set the memory usage limit + /// + /// This function is supported only when *strm has been initialized with + /// a function that takes a memlimit argument. + /// + /// + /// liblzma 5.2.3 and earlier has a bug where memlimit value of 0 causes + /// this function to do nothing (leaving the limit unchanged) and still + /// return LZMA_OK. Later versions treat 0 as if 1 had been specified (so + /// lzma_memlimit_get() will return 1 even if you specify 0 here). + /// + /// liblzma 5.2.6 and earlier had a bug in single-threaded .xz decoder + /// (lzma_stream_decoder()) which made it impossible to continue decoding + /// after LZMA_MEMLIMIT_ERROR even if the limit was increased using + /// lzma_memlimit_set(). Other decoders worked correctly. + /// + /// + /// - LZMA_OK: New memory usage limit successfully set. + /// - LZMA_MEMLIMIT_ERROR: The new limit is too small. The limit was not changed. + /// - LZMA_PROG_ERROR: Invalid arguments, e.g. *strm doesn't support memory usage limit. + /// + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] + internal delegate ulong lzma_memlimit_set(LzmaStream strm); + internal lzma_memlimit_set LzmaMemlimitSet; +#endif + #endregion + #region Check - Crc32, Crc64 [UnmanagedFunctionPointer(CallingConvention.Cdecl)] internal unsafe delegate uint lzma_crc32( @@ -233,6 +553,9 @@ internal unsafe delegate ulong lzma_crc64( internal delegate IntPtr lzma_version_string(); internal lzma_version_string LzmaVersionString; #endregion + + #region Memlimit - Memlimit + #endregion #endregion } } diff --git a/Joveler.Compression.XZ/XZMemory.cs b/Joveler.Compression.XZ/XZMemory.cs new file mode 100644 index 0000000..aa79cf6 --- /dev/null +++ b/Joveler.Compression.XZ/XZMemory.cs @@ -0,0 +1,123 @@ +/* + Derived from liblzma header files (Public Domain) + + C# Wrapper written by Hajin Jang + Copyright (C) 2018-2023 Hajin Jang + + MIT License + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ + +namespace Joveler.Compression.XZ +{ + public class XZMemory + { + #region MemUsage + /// + /// Calculate approximate memory usage of single-threaded encoder + /// + /// + /// Number of bytes of memory required for the given preset when encoding. + /// If an error occurs, for example due to unsupported preset, UINT64_MAX is returned. + /// + public static ulong EncoderMemUsage(LzmaCompLevel level, bool extremeFlag) + { + XZInit.Manager.EnsureLoaded(); + + uint preset = XZCompressOptions.ToPreset(level, extremeFlag); + return XZInit.Lib.LzmaEasyEncoderMemUsage(preset); + } + + /// + /// Calculate approximate memory usage of single-threaded encoder + /// + /// + /// Number of bytes of memory required for the given preset when encoding. + /// If an error occurs, for example due to unsupported preset, UINT64_MAX is returned. + /// + public static ulong EncoderMemUsage(XZCompressOptions compOpts) + { + XZInit.Manager.EnsureLoaded(); + + return XZInit.Lib.LzmaEasyEncoderMemUsage(compOpts.Preset); + } + + /// + /// Calculate approximate memory usage of multithreaded .xz encoder + /// + /// + /// Number of bytes of memory required for encoding with the given options. + /// If an error occurs, for example due to unsupported preset or filter chain, UINT64_MAX is returned. + /// + public static ulong ThreadedEncoderMemUsage(LzmaCompLevel level, bool extremeFlag, int threads) + { + XZInit.Manager.EnsureLoaded(); + + uint preset = XZCompressOptions.ToPreset(level, extremeFlag); + LzmaMt mtOpts = LzmaMt.Create(preset, threads); + return XZInit.Lib.LzmaStreamEncoderMtMemUsage(mtOpts); + } + + /// + /// Calculate approximate memory usage of multithreaded .xz encoder + /// + /// + /// Number of bytes of memory required for encoding with the given options. + /// If an error occurs, for example due to unsupported preset or filter chain, UINT64_MAX is returned. + /// + public static ulong ThreadedEncoderMemUsage(XZCompressOptions compOpts, XZThreadedCompressOptions threadOpts) + { + XZInit.Manager.EnsureLoaded(); + + LzmaMt mtOpts = compOpts.ToLzmaMt(threadOpts); + return XZInit.Lib.LzmaStreamEncoderMtMemUsage(mtOpts); + } + + /// + /// Calculate approximate decoder memory usage of a preset + /// + /// + /// Number of bytes of memory required to decompress a file that was compressed using the given preset. + /// If an error occurs, for example due to unsupported preset, UINT64_MAX is returned. + /// + public static ulong DecoderMemUsage(LzmaCompLevel level, bool extremeFlag) + { + XZInit.Manager.EnsureLoaded(); + + uint preset = XZCompressOptions.ToPreset(level, extremeFlag); + return XZInit.Lib.LzmaEasyDecoderMemUsage(preset); + } + + /// + /// Calculate approximate decoder memory usage of a preset + /// + /// + /// Number of bytes of memory required to decompress a file that was compressed using the given preset. + /// If an error occurs, for example due to unsupported preset, UINT64_MAX is returned. + /// + public static ulong DecoderMemUsage(XZCompressOptions compOpts) + { + XZInit.Manager.EnsureLoaded(); + + return XZInit.Lib.LzmaEasyDecoderMemUsage(compOpts.Preset); + } + #endregion + } +} diff --git a/Joveler.Compression.XZ/XZStream.cs b/Joveler.Compression.XZ/XZStreams.cs similarity index 53% rename from Joveler.Compression.XZ/XZStream.cs rename to Joveler.Compression.XZ/XZStreams.cs index 1c52269..bcbba32 100644 --- a/Joveler.Compression.XZ/XZStream.cs +++ b/Joveler.Compression.XZ/XZStreams.cs @@ -2,7 +2,7 @@ Derived from liblzma header files (Public Domain) C# Wrapper written by Hajin Jang - Copyright (C) 2018-2020 Hajin Jang + Copyright (C) 2018-2023 Hajin Jang MIT License @@ -30,7 +30,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE using System.IO; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -// ReSharper disable UnusedMember.Global namespace Joveler.Compression.XZ { @@ -82,13 +81,20 @@ internal LzmaMt ToLzmaMt(XZThreadedCompressOptions threadOpts) return new LzmaMt() { BlockSize = threadOpts.BlockSize, - Threads = XZThreadedCompressOptions.CheckThreadCount(threadOpts.Threads), + Threads = XZHardware.CheckThreadCount(threadOpts.Threads), + TimeOut = threadOpts.TimeOut, Preset = Preset, Check = Check, }; } } + /// + /// Options to control threaded XZ compresion. + /// + /// + /// IT IS HIGHLY RECOMMENDED TO SET memlimitThreading AND memlimitStop YOURSELF. + /// public class XZThreadedCompressOptions { /// @@ -108,23 +114,54 @@ public class XZThreadedCompressOptions /// Number of worker threads to use. /// public int Threads { get; set; } = 1; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static uint CheckThreadCount(int threads) - { - if (threads < 0) - throw new ArgumentOutOfRangeException(nameof(threads)); - if (threads == 0) // Use system's thread number by default - threads = Environment.ProcessorCount; - else if (Environment.ProcessorCount < threads) // If the number of CPU cores/threads exceeds system thread number, - threads = Environment.ProcessorCount; // Limit the number of threads to keep memory usage lower. - return (uint)threads; - } + /// + /// Timeout (millisecond) to allow lzma_code() to return early. + /// + /// + /// Multithreading can make liblzma to consume input and produce + /// output in a very bursty way: it may first read a lot of input + /// to fill internal buffers, then no input or output occurs for + /// a while. + /// + /// In single-threaded mode, lzma_code() won't return until it has + /// either consumed all the input or filled the output buffer. If + /// this is done in multithreaded mode, it may cause a call + /// lzma_code() to take even tens of seconds, which isn't acceptable + /// in all applications. + /// + /// To avoid very long blocking times in lzma_code(), a timeout + /// (in milliseconds) may be set here. If lzma_code() would block + /// longer than this number of milliseconds, it will return with + /// LZMA_OK. Reasonable values are 100 ms or more. The xz command + /// line tool uses 300 ms. + /// + /// If long blocking times are fine for you, set timeout to a special + /// value of 0, which will disable the timeout mechanism and will make + /// lzma_code() block until all the input is consumed or the output + /// buffer has been filled. + /// + /// NOTE: Even with a timeout, lzma_code() might sometimes take + /// somewhat long time to return. No timing guarantees + /// are made. + /// + // Bench: Does not affect performance in a meaningful way. + public uint TimeOut = 0; } public class XZDecompressOptions { + /// + /// Memory usage hard limit as bytes, that should never be exceeded. + /// + /// + /// Decoder: If decompressing will need more than this amount of + /// memory even in the single-threaded mode, then lzma_code() will + /// return LZMA_MEMLIMIT_ERROR. + /// public ulong MemLimit { get; set; } = ulong.MaxValue; + /// + /// + /// public LzmaDecodingFlag DecodeFlags { get; set; } = XZStream.DefaultDecodingFlags; /// /// Size of the internal buffer. @@ -134,11 +171,132 @@ public class XZDecompressOptions /// Whether to leave the base stream object open after disposing the xz stream object. /// public bool LeaveOpen { get; set; } = false; + + internal LzmaMt ToLzmaMt(XZThreadedDecompressOptions threadOpts) + { + // Default soft limit values used within xz-utils are + // - 64bit systems: TOTAL_RAM / 4. + // - 32bit systems: + ceiling of 1.4GB (1400U << 20) + // xz-utils author suggests to use MemAvailable value of the OS, though. + // Default hard limit values used in liblzma/xz are + // - UINT64_MAX + + // Soft limit + ulong memSoftLimit = threadOpts.MemlimitThreading; + + // Use of XZThreadedDecompressionOptions means the caller explicitly wants to use multi-threaded decompression. + // liblzma requires memlimit{Threading,Stop} to be set with actual numbers, not 0. + if (memSoftLimit == 0) + { // Convert 0 to a default value + switch (XZInit.Lib.PlatformBitness) + { + case DynLoader.PlatformBitness.Bit32: + memSoftLimit = Math.Min(XZHardware.PhysMem() / 4, 1400U << 20); + break; + case DynLoader.PlatformBitness.Bit64: + memSoftLimit = XZHardware.PhysMem() / 4; + break; + } + } + + // Return LzmaMt + return new LzmaMt() + { + Flags = DecodeFlags, + Threads = XZHardware.CheckThreadCount(threadOpts.Threads), + TimeOut = threadOpts.TimeOut, + MemlimitThreading = memSoftLimit, + MemlimitStop = MemLimit, + }; + } + } + + /// + /// Options to control threaded XZ decompresion. + /// + /// + /// IT IS HIGHLY RECOMMENDED TO EXPLICITLY SET memlimitThreading WITH PROPER VALUE. + /// + public class XZThreadedDecompressOptions + { + /// + /// Number of worker threads to use. + /// + public int Threads { get; set; } = 1; + /// + /// Timeout to allow lzma_code() to return early + /// + /// + /// + /// Multithreading can make liblzma to consume input and produce + /// output in a very bursty way: it may first read a lot of input + /// to fill internal buffers, then no input or output occurs for + /// a while. + /// + /// + /// In single-threaded mode, lzma_code() won't return until it has + /// either consumed all the input or filled the output buffer. If + /// this is done in multithreaded mode, it may cause a call + /// lzma_code() to take even tens of seconds, which isn't acceptable + /// in all applications. + /// + /// + /// To avoid very long blocking times in lzma_code(), a timeout + /// (in milliseconds) may be set here. If lzma_code() would block + /// longer than this number of milliseconds, it will return with + /// LZMA_OK. Reasonable values are 100 ms or more. The xz command + /// line tool uses 300 ms. + /// + /// + /// If long blocking times are fine for you, set timeout to a special + /// value of 0, which will disable the timeout mechanism and will make + /// lzma_code() block until all the input is consumed or the output + /// buffer has been filled. + /// + /// + /// NOTE: Even with a timeout, lzma_code() might sometimes take somewhat long time to return. + /// No timing guarantees are made. + /// + /// + // Bench: Does not affect performance in a meaningful way. + public uint TimeOut = 0; + /// + /// Memory usage soft limit to reduce the number of threads. + /// Joveler.Compression.XZ specific: Set to 0 to use default value (TotalMem / 4). + /// + /// + /// If the number of threads has been set so high that more than + /// memlimit_threading bytes of memory would be needed, the number + /// of threads will be reduced so that the memory usage will not exceed + /// memlimit_threading bytes. However, if memlimit_threading cannot + /// be met even in single-threaded mode, then decoding will continue + /// in single-threaded mode and memlimit_threading may be exceeded + /// even by a large amount. That is, memlimit_threading will never make + /// lzma_code() return LZMA_MEMLIMIT_ERROR. To truly cap the memory + /// usage, see memlimit_stop below. + /// + /// Setting memlimit_threading to UINT64_MAX or a similar huge value + /// means that liblzma is allowed to keep the whole compressed file + /// and the whole uncompressed file in memory in addition to the memory + /// needed by the decompressor data structures used by each thread! + /// In other words, a reasonable value limit must be set here or it + /// will cause problems sooner or later. If you have no idea what + /// a reasonable value could be, try lzma_physmem() / 4 as a starting + /// point. Setting this limit will never prevent decompression of + /// a file; this will only reduce the number of threads. + /// + /// If memlimit_threading is greater than memlimit_stop, then the value + /// of memlimit_stop will be used for both. + /// + public ulong MemlimitThreading = 0; } #endregion #region XZStream - // ReSharper disable once InconsistentNaming + /// + /// + /// The stream to handle .xz file format. + /// public class XZStream : Stream { #region enum Mode @@ -149,6 +307,20 @@ private enum Mode } #endregion + #region enum CoderFormat + /// + /// Determine which encoder/decoder to use. + /// Single-threaded decompression only. + /// + protected enum CoderFormat + { + Auto = 0, + XZ = 1, + LegacyLzma = 2, + LZip = 3, + } + #endregion + #region Fields and Properties private readonly Mode _mode; private readonly bool _leaveOpen; @@ -199,10 +371,16 @@ AMD Ryzen 5 3600 / .NET Core 3.1.13 / Windows 10.0.19042 x64 / xz-utils 5.2.5 internal const int DefaultBufferSize = 1024 * 1024; #endregion - #region Constructor + #region Constructor (Compression) /// - /// Create single-threaded compressing XZStream. + /// Create single-threaded compressing XZStream instance. /// + /// + /// A stream of XZ container to compress. + /// + /// + /// Options to control general compression. + /// public unsafe XZStream(Stream baseStream, XZCompressOptions compOpts) { XZInit.Manager.EnsureLoaded(); @@ -226,15 +404,25 @@ public unsafe XZStream(Stream baseStream, XZCompressOptions compOpts) // Initialize the encoder LzmaRet ret = XZInit.Lib.LzmaEasyEncoder(_lzmaStream, preset, compOpts.Check); - XZException.CheckReturnValue(ret); + XZException.CheckReturnValueNormal(ret); // Set possible max memory usage. MaxMemUsage = XZInit.Lib.LzmaEasyEncoderMemUsage(preset); } /// - /// Create multi-threaded compressing XZStream. Requires more memory than single-threaded mode. + /// Create multi-threaded compressing XZStream instance. + /// Requires more memory than single-threaded mode. /// + /// + /// A stream of XZ container to compress. + /// + /// + /// Options to control general compression. + /// + /// + /// Options to control threaded compression. + /// public unsafe XZStream(Stream baseStream, XZCompressOptions compOpts, XZThreadedCompressOptions threadOpts) { XZInit.Manager.EnsureLoaded(); @@ -258,16 +446,42 @@ public unsafe XZStream(Stream baseStream, XZCompressOptions compOpts, XZThreaded // Initialize the encoder LzmaRet ret = XZInit.Lib.LzmaStreamEncoderMt(_lzmaStream, mt); - XZException.CheckReturnValue(ret); + XZException.CheckReturnValueNormal(ret); // Set possible max memory usage. MaxMemUsage = XZInit.Lib.LzmaStreamEncoderMtMemUsage(mt); } + #endregion + #region Constructors (Decompression) /// - /// Create decompressing XZStream + /// Create decompressing XZStream instance. /// - public unsafe XZStream(Stream baseStream, XZDecompressOptions decompOpts) + /// + /// A stream of XZ container to decompress. + /// + /// + /// Options to control general decompression. + /// + public XZStream(Stream baseStream, XZDecompressOptions decompOpts) + : this(baseStream, decompOpts, CoderFormat.XZ) + { + } + + /// + /// (Not Public) Create decompressing XZStream instance with . + /// + /// + /// A stream of XZ container to decompress. + /// + /// + /// Options to control general decompression. + /// + /// + /// + /// + /// + protected unsafe XZStream(Stream baseStream, XZDecompressOptions decompOpts, CoderFormat fileFormat) { XZInit.Manager.EnsureLoaded(); @@ -285,8 +499,62 @@ public unsafe XZStream(Stream baseStream, XZDecompressOptions decompOpts) _lzmaStreamPin = GCHandle.Alloc(_lzmaStream, GCHandleType.Pinned); // Initialize the decoder - LzmaRet ret = XZInit.Lib.LzmaStreamDecoder(_lzmaStream, decompOpts.MemLimit, decompOpts.DecodeFlags); - XZException.CheckReturnValue(ret); + LzmaRet ret = LzmaRet.Ok; + switch (fileFormat) + { + case CoderFormat.XZ: + ret = XZInit.Lib.LzmaStreamDecoder(_lzmaStream, decompOpts.MemLimit, decompOpts.DecodeFlags); + break; + case CoderFormat.Auto: + ret = XZInit.Lib.LzmaAutoDecoder(_lzmaStream, decompOpts.MemLimit, decompOpts.DecodeFlags); + break; + case CoderFormat.LegacyLzma: + ret = XZInit.Lib.LzmaAloneDecoder(_lzmaStream, decompOpts.MemLimit); + break; + case CoderFormat.LZip: + ret = XZInit.Lib.LzmaLZipDecoder(_lzmaStream, decompOpts.MemLimit, decompOpts.DecodeFlags); + break; + } + XZException.CheckReturnValueNormal(ret); + } + + /// + /// Create multi-threaded decompressing XZStream instance. + /// Requires more memory than single-threaded mode. + /// + /// + /// A stream of XZ container to decompress. + /// + /// + /// Options to control general decompression. + /// + /// + /// Options to control threaded decompression. + /// It is highly recommended to explicitly set value. + /// + public unsafe XZStream(Stream baseStream, XZDecompressOptions decompOpts, XZThreadedDecompressOptions threadOpts) + { + XZInit.Manager.EnsureLoaded(); + + BaseStream = baseStream ?? throw new ArgumentNullException(nameof(baseStream)); + _mode = Mode.Decompress; + _disposed = false; + + // Check and set decompress options + _leaveOpen = decompOpts.LeaveOpen; + _bufferSize = CheckBufferSize(decompOpts.BufferSize); + _workBuf = new byte[_bufferSize]; + + // Prepare LzmaStream and buffers + _lzmaStream = new LzmaStream(); + _lzmaStreamPin = GCHandle.Alloc(_lzmaStream, GCHandleType.Pinned); + + // Check LzmaMt instance + LzmaMt mt = decompOpts.ToLzmaMt(threadOpts); + + // Initialize the decoder + LzmaRet ret = XZInit.Lib.LzmaStreamDecoderMt(_lzmaStream, mt); + XZException.CheckReturnValueNormal(ret); } #endregion @@ -334,7 +602,7 @@ protected override void Dispose(bool disposing) public override int Read(byte[] buffer, int offset, int count) { // For Decompress if (_mode != Mode.Decompress) - throw new NotSupportedException("Read() not supported on compression"); + throw new NotSupportedException($"{nameof(Read)}() not supported on compression."); CheckReadWriteArgs(buffer, offset, count); if (count == 0) return 0; @@ -351,7 +619,7 @@ public unsafe int Read(Span span) #endif { // For Decompress if (_mode != Mode.Decompress) - throw new NotSupportedException("Read() not supported on compression"); + throw new NotSupportedException($"{nameof(Read)}() not supported on compression."); if (_workBufPos == ReadDone) return 0; @@ -389,15 +657,26 @@ public unsafe int Read(Span span) _workBufPos += (int)(bakAvailIn - _lzmaStream.AvailIn); readSize += (int)(bakAvailOut - _lzmaStream.AvailOut); - // Once everything has been decoded successfully, the return value of lzma_code() will be LZMA_STREAM_END. + if (ret == LzmaRet.StreamEnd) - { + { // Once everything has been decoded successfully, the return value of lzma_code() will be LZMA_STREAM_END. _workBufPos = ReadDone; break; } + else if (ret == LzmaRet.SeekNeeded) + { // Request to change the input file position -> Some coders can do random access in the input file. + // When this value is returned, the application must seek to the file position given in lzma_stream.seek_pos. + // This value is guaranteed to never exceed the file size that was specified at the coder initialization. + // After seeking the application should read new input and pass it normally via lzma_stream.next_in and .avail_in. + + // Seek BaseStream. If Seek() fails, it will throw a NotSupportedException + // v5.4.0 -> only decoder uses random seek. + BaseStream.Seek((long)_lzmaStream.SeekPos, SeekOrigin.Begin); + _lzmaStream.AvailIn = 0; + } // Normally the return value of lzma_code() will be LZMA_OK until everything has been encoded. - XZException.CheckReturnValue(ret); + XZException.CheckReturnValueDecompress(ret); } } @@ -409,7 +688,7 @@ public unsafe int Read(Span span) public override void Write(byte[] buffer, int offset, int count) { // For Compress if (_mode != Mode.Compress) - throw new NotSupportedException("Write() not supported on decompression"); + throw new NotSupportedException($"{nameof(Write)}() not supported on decompression."); CheckReadWriteArgs(buffer, offset, count); if (count == 0) return; @@ -426,7 +705,7 @@ public unsafe void Write(ReadOnlySpan span) #endif { // For Compress if (_mode != Mode.Compress) - throw new NotSupportedException("Write() not supported on decompression"); + throw new NotSupportedException($"{nameof(Write)}() not supported on decompression."); TotalIn += span.Length; @@ -458,14 +737,14 @@ public unsafe void Write(ReadOnlySpan span) } // Normally the return value of lzma_code() will be LZMA_OK until everything has been encoded. - XZException.CheckReturnValue(ret); + XZException.CheckReturnValueNormal(ret); } } } private unsafe void FinishWrite() { - Debug.Assert(_mode == Mode.Compress, "FinishWrite() must not be called in decompression"); + Debug.Assert(_mode == Mode.Compress, $"{nameof(FinishWrite)}() must not be called in decompression."); fixed (byte* writePtr = _workBuf) { @@ -485,8 +764,8 @@ private unsafe void FinishWrite() // write the data from the output buffer to the output file. if (_lzmaStream.AvailOut == 0 || ret == LzmaRet.StreamEnd) { // Write to _baseStream - // When lzma_code() has returned LZMA_STREAM_END, the output buffer is likely to be only partially - // full. Calculate how much new data there is to be written to the output file. + // When lzma_code() has returned LZMA_STREAM_END, the output buffer is likely to be only partially full. + // Calculate how much new data there is to be written to the output file. BaseStream.Write(_workBuf, 0, _workBufPos); TotalOut += _workBufPos; @@ -497,7 +776,7 @@ private unsafe void FinishWrite() } else { // Once everything has been encoded successfully, the return value of lzma_code() will be LZMA_STREAM_END. - XZException.CheckReturnValue(ret); + XZException.CheckReturnValueNormal(ret); } } } @@ -558,20 +837,20 @@ public override unsafe void Flush() /// public override long Seek(long offset, SeekOrigin origin) { - throw new NotSupportedException("Seek() not supported"); + throw new NotSupportedException($"{nameof(Seek)} not supported."); } /// public override void SetLength(long value) { - throw new NotSupportedException("SetLength not supported"); + throw new NotSupportedException($"{nameof(SetLength)} not supported."); } /// - public override long Length => throw new NotSupportedException("Length not supported"); + public override long Length => throw new NotSupportedException($"{nameof(Length)} not supported."); /// public override long Position { - get => throw new NotSupportedException("Position not supported"); - set => throw new NotSupportedException("Position not supported"); + get => throw new NotSupportedException($"{nameof(Position)} not supported."); + set => throw new NotSupportedException($"{nameof(Position)} not supported."); } public double CompressionRatio @@ -596,20 +875,24 @@ public double CompressionRatio #region GetProgress /// - /// Get progress information + /// Get progress information of XZ stream. /// /// - /// In single-threaded mode, applications can get progress information from - /// strm->total_in and strm->total_out.In multi-threaded mode this is less + /// In single-threaded mode, applications can get progress information from + /// strm->total_in and strm->total_out. + /// + /// In multi-threaded mode this is less /// useful because a significant amount of both input and output data gets - /// buffered internally by liblzma.This makes total_in and total_out give + /// buffered internally by liblzma. This makes total_in and total_out give /// misleading information and also makes the progress indicator updates /// non-smooth. - /// + /// + /// /// This function gives realistic progress information also in multi-threaded /// mode by taking into account the progress made by each thread. In /// single-threaded mode *progress_in and *progress_out are set to /// strm->total_in and strm->total_out, respectively. + /// /// public void GetProgress(out ulong progressIn, out ulong progressOut) { @@ -619,6 +902,36 @@ public void GetProgress(out ulong progressIn, out ulong progressOut) } #endregion + #region Memory Usage (Decompression Only) - DISABLED + // lzma_memusage() only works on per-thread basis. + // It would not help users to perceive how many memory cap would needed on multi-threaded decompression. +#if LZMA_MEM_ENABLE + /// + /// Get the memory usage of decompression setup. + /// Must be called after calling Read() at least once. + /// + /// + /// This function is useful e.g. after LZMA_MEMLIMIT_ERROR to find out how big + /// the memory usage limit should have been to decode the input. Note that + /// this may give misleading information if decoding .xz Streams that have + /// multiple Blocks, because each Block can have different memory requirements. + /// + /// + /// How much memory is currently allocated for the filter decoders. + /// If no filter chain is currently allocated, some non-zero value is still returned, + /// which is less than or equal to what any filter chain would indicate as its memory requirement. + /// + /// If this function isn't supported by *strm or some other error occurs, zero is returned. + /// + public ulong GetDecompresMemUsage() + { + if (_mode != Mode.Decompress) + throw new NotSupportedException($"{nameof(GetDecompresMemUsage)}() not supported on compression."); + return XZInit.Lib.LzmaMemusage(_lzmaStream); + } +#endif + #endregion + #region (internal, private) Check Arguments [MethodImpl(MethodImplOptions.AggressiveInlining)] internal static void CheckReadWriteArgs(byte[] buffer, int offset, int count) @@ -651,4 +964,83 @@ private static int CheckBufferSize(int bufferSize) #endregion } #endregion + + #region LzmaAutoStream (Decompress Only) + /// + /// + /// The stream to handle .xz, .lzma, and .lz (lzip) files with autodetection. (Decompression Only) + /// + /// + /// Does not support multi-threaded xz decompression. + /// + public class LzmaAutoStream : XZStream + { + /// + /// Create decompressing LzmaAutoStream instance. + /// Auto detects .xz, .lzma and .lz file format. + /// + /// + /// Does not support multi-threaded xz decompression. + /// + /// + /// A stream of xz/lzma/lz container to decompress. + /// + /// + /// Options to control general decompression. + /// + public LzmaAutoStream(Stream baseStream, XZDecompressOptions decompOpts) + : base(baseStream, decompOpts, CoderFormat.Auto) + { + } + } + #endregion + + #region LzmaAloneStream (Decompress Only) + /// + /// The stream to handle legacy .lzma file format. (Decompression Only) + /// + // TODO: liblzma supports .lzma compression. Since the .lzma format is the legacy one and is almost dead, + // Do we really need it on Joveler.Compression.XZ? + // To support it, lzma_options_lzma also needs to be p/invoked. + public class LzmaAloneStream : XZStream + { + /// + /// + /// Create decompressing LzmaAloneStream instance. + /// + /// + /// A stream of .lz (lzip) container to decompress. + /// + /// + /// Options to control general decompression. + /// + public LzmaAloneStream(Stream baseStream, XZDecompressOptions decompOpts) + : base(baseStream, decompOpts, CoderFormat.LegacyLzma) + { + } + } + #endregion + + #region LZipStream (Decompress Only) + /// + /// The stream to handle .lz (lzip) file format. (Decompression Only) + /// + public class LZipStream : XZStream + { + /// + /// + /// Create decompressing LZipStream instance. + /// + /// + /// A stream of .lz (lzip) container to decompress. + /// + /// + /// Options to control general decompression. + /// + public LZipStream(Stream baseStream, XZDecompressOptions decompOpts) + : base(baseStream, decompOpts, CoderFormat.LZip) + { + } + } + #endregion } diff --git a/Joveler.Compression.XZ/XZStructs.cs b/Joveler.Compression.XZ/XZStructs.cs index 4f1d095..d6b576f 100644 --- a/Joveler.Compression.XZ/XZStructs.cs +++ b/Joveler.Compression.XZ/XZStructs.cs @@ -2,7 +2,7 @@ Derived from liblzma header files (Public Domain) C# Wrapper written by Hajin Jang - Copyright (C) 2018-2020 Hajin Jang + Copyright (C) 2018-2023 Hajin Jang MIT License @@ -150,7 +150,18 @@ public ulong AvailOut private IntPtr ReservedPtr2 = IntPtr.Zero; private IntPtr ReservedPtr3 = IntPtr.Zero; private IntPtr ReservedPtr4 = IntPtr.Zero; - private ulong ReservedInt1 = 0; + /// + /// New seek input position for LZMA_SEEK_NEEDED + /// + /// + /// When lzma_code() returns LZMA_SEEK_NEEDED, the new input position + /// needed by liblzma will be available seek_pos. The value is + /// guaranteed to not exceed the file size that was specified when + /// this lzma_stream was initialized. + /// + /// In all other situations the value of this variable is undefined. + /// + public ulong SeekPos = 0; private ulong ReservedInt2 = 0; private UIntPtr ReservedInt3 = UIntPtr.Zero; private UIntPtr ReservedInt4 = UIntPtr.Zero; @@ -227,21 +238,28 @@ internal class LzmaStreamFlags #endregion #region (internal) LzmaMt + /// + /// Multithreading options + /// [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)] internal class LzmaMt { /// /// Set this to zero if no flags are wanted. /// - /// No flags are currently supported. + /// Encoder: No flags are currently supported. + /// Decoder: Bitwise-or of zero or more of the decoder flags: + /// LZMA_TELL_NO_CHECK, LZMA_TELL_UNSUPPORTED_CHECK, + /// LZMA_TELL_ANY_CHECK, LZMA_IGNORE_CHECK, + /// LZMA_CONCATENATED, LZMA_FAIL_FAST /// - private uint Flags = 0; + public LzmaDecodingFlag Flags = LzmaDecodingFlag.None; /// /// Number of worker threads to use /// public uint Threads; /// - /// Maximum uncompressed size of a Block + /// Encoder only: Maximum uncompressed size of a Block /// /// /// The encoder will start a new .xz Block every block_size bytes. @@ -297,7 +315,7 @@ internal class LzmaMt /// public uint TimeOut; /// - /// Compression preset (level and possible flags) + /// Encoder only: Compression preset /// /// /// The preset is set just like with lzma_easy_encoder(). @@ -305,15 +323,15 @@ internal class LzmaMt /// public uint Preset; /// - /// Filter chain (alternative to a preset) + /// Encoder only: Filter chain (alternative to a preset) /// /// /// If this is NULL, the preset above is used. Otherwise the preset /// is ignored and the filter chain specified here is used. /// - private IntPtr Filters; + private IntPtr _filters; /// - /// Integrity check type + /// Encoder only: Integrity check type /// /// /// See check.h for available checks. The xz command line tool @@ -322,21 +340,77 @@ internal class LzmaMt /// public LzmaCheck Check; - private uint ReservedEnum1; - private uint ReservedEnum2; - private uint ReservedEnum3; - private uint ReservedInt1; - private uint ReservedInt2; - private uint ReservedInt3; - private uint ReservedInt4; - private ulong ReservedInt5; - private ulong ReservedInt6; - private ulong ReservedInt7; - private ulong ReservedInt8; - private IntPtr ReservedPtr1; - private IntPtr ReservedPtr2; - private IntPtr ReservedPtr3; - private IntPtr ReservedPtr4; + private uint ReservedEnum1 = 0; + private uint ReservedEnum2 = 0; + private uint ReservedEnum3 = 0; + private uint ReservedInt1 = 0; + private uint ReservedInt2 = 0; + private uint ReservedInt3 = 0; + private uint ReservedInt4 = 0; + /// + /// Memory usage limit to reduce the number of threads + /// + /// Encoder: Ignored. + /// + /// Decoder: + /// + /// If the number of threads has been set so high that more than + /// memlimit_threading bytes of memory would be needed, the number + /// of threads will be reduced so that the memory usage will not exceed + /// memlimit_threading bytes. However, if memlimit_threading cannot + /// be met even in single-threaded mode, then decoding will continue + /// in single-threaded mode and memlimit_threading may be exceeded + /// even by a large amount. That is, memlimit_threading will never make + /// lzma_code() return LZMA_MEMLIMIT_ERROR. To truly cap the memory + /// usage, see memlimit_stop below. + /// + /// Setting memlimit_threading to UINT64_MAX or a similar huge value + /// means that liblzma is allowed to keep the whole compressed file + /// and the whole uncompressed file in memory in addition to the memory + /// needed by the decompressor data structures used by each thread! + /// In other words, a reasonable value limit must be set here or it + /// will cause problems sooner or later. If you have no idea what + /// a reasonable value could be, try lzma_physmem() / 4 as a starting + /// point. Setting this limit will never prevent decompression of + /// a file; this will only reduce the number of threads. + /// + /// If memlimit_threading is greater than memlimit_stop, then the value + /// of memlimit_stop will be used for both. + /// + /// + /// If single-threaded mode was requested, set the memlimit for threading to zero. + /// This forces the decoder to use single-threaded mode which matches the behavior of lzma_stream_decoder(). + /// + /// Otherwise use the limit for threaded decompression which has a sane default + /// (users are still free to make it insanely high though). + /// + /// Default values used within xz-utils are + /// - 64bit systems: TOTAL_RAM / 4. + /// - 32bit systems: 1.4GB + /// xz-utils author suggests to use MemAvailable value of the OS, though. + /// + public ulong MemlimitThreading = 0; + /// + /// Memory usage limit that should never be exceeded. + /// + /// Encoder: Ignored. + /// + /// Decoder: If decompressing will need more than this amount of + /// memory even in the single-threaded mode, then lzma_code() will + /// return LZMA_MEMLIMIT_ERROR. + /// + /// + /// Default hard limit values used in liblzma/xz are + /// - UINT64_MAX + /// + public ulong MemlimitStop = ulong.MaxValue; + private ulong _reservedInt6 = 0; + private ulong _reservedInt7 = 0; + private ulong _reservedInt8 = 0; + private IntPtr ReservedPtr1 = new IntPtr(0); + private IntPtr ReservedPtr2 = new IntPtr(0); + private IntPtr ReservedPtr3 = new IntPtr(0); + private IntPtr ReservedPtr4 = new IntPtr(0); public static LzmaMt Create(uint preset, int threads) { @@ -353,7 +427,7 @@ public static LzmaMt Create(uint preset, int threads) // See the documentation of lzma_mt in lzma/container.h for information how to choose a reasonable timeout. TimeOut = 0, // To use a preset, filters must be set to NULL. - Filters = IntPtr.Zero, + _filters = IntPtr.Zero, Preset = preset, // Use XZ default Check = LzmaCheck.Crc64, @@ -617,6 +691,12 @@ public enum LzmaRet : uint /// specified memory usage limit. To continue decoding, /// the memory usage limit has to be increased with /// lzma_memlimit_set(). + /// + /// liblzma 5.2.6 and earlier had a bug in single-threaded .xz + /// decoder (lzma_stream_decoder()) which made it impossible + /// to continue decoding after LZMA_MEMLIMIT_ERROR even if + /// the limit was increased using lzma_memlimit_set(). + /// Other decoders worked correctly. /// MemLimitError = 6, /// @@ -707,6 +787,23 @@ public enum LzmaRet : uint /// how to report bugs. /// ProgError = 11, + /// + /// Request to change the input file position + /// + /// + /// Some coders can do random access in the input file. The + /// initialization functions of these coders take the file size + /// as an argument. No other coders can return LZMA_SEEK_NEEDED. + /// + /// When this value is returned, the application must seek to + /// the file position given in lzma_stream.seek_pos. This value + /// is guaranteed to never exceed the file size that was + /// specified at the coder initialization. + /// + /// After seeking the application should read new input and + /// pass it normally via lzma_stream.next_in and .avail_in. + /// + SeekNeeded = 12, } #endregion @@ -747,6 +844,7 @@ public enum LzmaCheck [Flags] public enum LzmaDecodingFlag : uint { + None = 0, /// /// This flag makes lzma_code() return LZMA_NO_CHECK if the input stream /// being decoded has no integrity check. Note that when used with @@ -793,8 +891,8 @@ public enum LzmaDecodingFlag : uint /// /// This flag enables decoding of concatenated files with file formats that /// allow concatenating compressed files as is. From the formats currently - /// supported by liblzma, only the .xz format allows concatenated files. - /// Concatenated files are not allowed with the legacy .lzma format. + /// supported by liblzma, only the .xz and .lz formats allow concatenated + /// files. Concatenated files are not allowed with the legacy .lzma format. /// /// /// This flag also affects the usage of the `action' argument for lzma_code(). @@ -806,6 +904,33 @@ public enum LzmaDecodingFlag : uint /// as `action' for lzma_code(), but the usage of LZMA_FINISH isn't required. /// Concatenated = 0x08, + /// + /// This flag makes the threaded decoder report errors (like LZMA_DATA_ERROR) + /// as soon as they are detected. This saves time when the application has no + /// interest in a partially decompressed truncated or corrupt file. Note that + /// due to timing randomness, if the same truncated or corrupt input is + /// decompressed multiple times with this flag, a different amount of output + /// may be produced by different runs, and even the error code might vary. + /// + /// When using LZMA_FAIL_FAST, it is recommended to use LZMA_FINISH to tell + /// the decoder when no more input will be coming because it can help fast + /// detection and reporting of truncated files. Note that in this situation + /// truncated files might be diagnosed with LZMA_DATA_ERROR instead of + /// LZMA_OK or LZMA_BUF_ERROR! + /// + /// Without this flag the threaded decoder will provide as much output as + /// possible at first and then report the pending error. This default behavior + /// matches the single-threaded decoder and provides repeatable behavior + /// with truncated or corrupt input. There are a few special cases where the + /// behavior can still differ like memory allocation failures (LZMA_MEM_ERROR). + /// + /// Single-threaded decoders currently ignore this flag. + /// + /// Support for this flag was added in liblzma 5.3.3alpha. Note that in older + /// versions this flag isn't supported (LZMA_OPTIONS_ERROR) even by functions + /// that ignore this flag in newer liblzma versions. + /// + FailFast = 0x20, } #endregion diff --git a/Joveler.Compression.XZ/runtimes/linux-arm/native/liblzma.so b/Joveler.Compression.XZ/runtimes/linux-arm/native/liblzma.so index 4589e63..dd1c280 100644 Binary files a/Joveler.Compression.XZ/runtimes/linux-arm/native/liblzma.so and b/Joveler.Compression.XZ/runtimes/linux-arm/native/liblzma.so differ diff --git a/Joveler.Compression.XZ/runtimes/linux-arm64/native/liblzma.so b/Joveler.Compression.XZ/runtimes/linux-arm64/native/liblzma.so index 5b7032a..e501824 100644 Binary files a/Joveler.Compression.XZ/runtimes/linux-arm64/native/liblzma.so and b/Joveler.Compression.XZ/runtimes/linux-arm64/native/liblzma.so differ diff --git a/Joveler.Compression.XZ/runtimes/linux-x64/native/liblzma.so b/Joveler.Compression.XZ/runtimes/linux-x64/native/liblzma.so index 4cc51ba..1f1e165 100644 Binary files a/Joveler.Compression.XZ/runtimes/linux-x64/native/liblzma.so and b/Joveler.Compression.XZ/runtimes/linux-x64/native/liblzma.so differ diff --git a/Joveler.Compression.XZ/runtimes/osx-arm64/native/liblzma.dylib b/Joveler.Compression.XZ/runtimes/osx-arm64/native/liblzma.dylib index 4d83cd5..ab17334 100755 Binary files a/Joveler.Compression.XZ/runtimes/osx-arm64/native/liblzma.dylib and b/Joveler.Compression.XZ/runtimes/osx-arm64/native/liblzma.dylib differ diff --git a/Joveler.Compression.XZ/runtimes/osx-x64/native/liblzma.dylib b/Joveler.Compression.XZ/runtimes/osx-x64/native/liblzma.dylib index d54afe0..8fa0432 100644 Binary files a/Joveler.Compression.XZ/runtimes/osx-x64/native/liblzma.dylib and b/Joveler.Compression.XZ/runtimes/osx-x64/native/liblzma.dylib differ diff --git a/Joveler.Compression.XZ/runtimes/win-arm64/native/liblzma.dll b/Joveler.Compression.XZ/runtimes/win-arm64/native/liblzma.dll index d67cfd3..ed6227b 100644 Binary files a/Joveler.Compression.XZ/runtimes/win-arm64/native/liblzma.dll and b/Joveler.Compression.XZ/runtimes/win-arm64/native/liblzma.dll differ diff --git a/Joveler.Compression.XZ/runtimes/win-x64/native/liblzma.dll b/Joveler.Compression.XZ/runtimes/win-x64/native/liblzma.dll index f510ac5..d51549a 100644 Binary files a/Joveler.Compression.XZ/runtimes/win-x64/native/liblzma.dll and b/Joveler.Compression.XZ/runtimes/win-x64/native/liblzma.dll differ diff --git a/Joveler.Compression.XZ/runtimes/win-x86/native/liblzma.dll b/Joveler.Compression.XZ/runtimes/win-x86/native/liblzma.dll index 406b676..4c9223d 100644 Binary files a/Joveler.Compression.XZ/runtimes/win-x86/native/liblzma.dll and b/Joveler.Compression.XZ/runtimes/win-x86/native/liblzma.dll differ diff --git a/Joveler.Compression.ZLib.Tests/Joveler.Compression.ZLib.Tests.csproj b/Joveler.Compression.ZLib.Tests/Joveler.Compression.ZLib.Tests.csproj index 5465fd1..25f99af 100644 --- a/Joveler.Compression.ZLib.Tests/Joveler.Compression.ZLib.Tests.csproj +++ b/Joveler.Compression.ZLib.Tests/Joveler.Compression.ZLib.Tests.csproj @@ -1,13 +1,13 @@  - - - net48;net6.0 - - net6.0 + + + net481;net6.0 net6.0 + + false AnyCPU @@ -15,8 +15,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Joveler.Compression.ZLib/Checksum/Adler32.cs b/Joveler.Compression.ZLib/Checksum/Adler32.cs index f12a92b..f8b33f1 100644 --- a/Joveler.Compression.ZLib/Checksum/Adler32.cs +++ b/Joveler.Compression.ZLib/Checksum/Adler32.cs @@ -102,7 +102,7 @@ protected override void HashCore(byte[] array, int ibStart, int cbSize) _adler32.Append(array, ibStart, cbSize); } -#if NETSTANDARD2_1 +#if NETCOREAPP protected override void HashCore(ReadOnlySpan source) { _adler32.Append(source); diff --git a/Joveler.Compression.ZLib/Checksum/Crc32.cs b/Joveler.Compression.ZLib/Checksum/Crc32.cs index 93541cc..cb1844e 100644 --- a/Joveler.Compression.ZLib/Checksum/Crc32.cs +++ b/Joveler.Compression.ZLib/Checksum/Crc32.cs @@ -102,7 +102,7 @@ protected override void HashCore(byte[] array, int ibStart, int cbSize) _crc32.Append(array, ibStart, cbSize); } -#if NETSTANDARD2_1 +#if NETCOREAPP protected override void HashCore(ReadOnlySpan source) { _crc32.Append(source); diff --git a/Joveler.Compression.ZLib/Joveler.Compression.ZLib.csproj b/Joveler.Compression.ZLib/Joveler.Compression.ZLib.csproj index e2ce4eb..4aa7f6d 100644 --- a/Joveler.Compression.ZLib/Joveler.Compression.ZLib.csproj +++ b/Joveler.Compression.ZLib/Joveler.Compression.ZLib.csproj @@ -1,10 +1,12 @@  - net462;netstandard2.0;netcoreapp3.1 + net46;netstandard2.0;netcoreapp3.1 netstandard2.0;netcoreapp3.1 true latest AnyCPU + true + Joveler.Compression.ZLib Joveler.Compression.ZLib 4.2.0 @@ -22,14 +24,15 @@ Supports Windows, Linux and macOS. - Recompile Windows zlib binaries with proper compile options. - Discontinue .NET Framework 4.5.1, 4.6, 4.6.1 due to end of official Microsoft support. zlib deflate gzip compress compression decompress decompression archive adler32 crc32 crc checksum native pinvoke interop + README.md - + - - + + x86\%(FileName)%(Extension) PreserveNewest @@ -38,8 +41,12 @@ Supports Windows, Linux and macOS. x64\%(FileName)%(Extension) PreserveNewest + + arm64\%(FileName)%(Extension) + PreserveNewest + - + runtimes\win-x86\native\%(FileName)%(Extension) @@ -76,10 +83,12 @@ Supports Windows, Linux and macOS. - - + + + + diff --git a/Joveler.Compression.ZLib/Joveler.Compression.ZLib.netfx.targets b/Joveler.Compression.ZLib/Joveler.Compression.ZLib.netfx.targets index f3968c4..127ac2a 100644 --- a/Joveler.Compression.ZLib/Joveler.Compression.ZLib.netfx.targets +++ b/Joveler.Compression.ZLib/Joveler.Compression.ZLib.netfx.targets @@ -32,6 +32,7 @@ HasTrailingSlash('$(MSBuildThisFileDirectory)')"> + - + CollectInteropLibFiles_x86; - CollectInteropLibFiles_x64; + CollectInteropLibFiles_x64 + CollectInteropLibFiles_arm64; $(PipelineCollectFilesPhaseDependsOn); diff --git a/Joveler.Compression.ZLib/NUGET_README.md b/Joveler.Compression.ZLib/NUGET_README.md new file mode 100644 index 0000000..72da4fd --- /dev/null +++ b/Joveler.Compression.ZLib/NUGET_README.md @@ -0,0 +1,29 @@ +## Usage + +Please refer to this [document](https://github.com/ied206/Joveler.Compression/blob/v4.2.0/Joveler.Compression.ZLib/USAGE.md). + +## Tested zlib versions + +- 1.2.13 (Included) + +## Support + +### Targeted .NET platforms + +- .NET Core 3.1+ +- .NET Standard 2.0 (.NET Framework 4.6.1+, .NET Core 2.0+) +- .NET Framework 4.6.2+ + +#### Discontinued target frameworks + +| Platform | Last Supported Version | +|----------|------------------------| +| .NET Framework 4.5 | [ZLibWrapper](https://www.nuget.org/packages/Joveler.ZLibWrapper) | +| .NET Standard 1.3 | [v2.1.2](https://www.nuget.org/packages/Joveler.Compression.ZLib/2.1.2) | +| .NET Framework 4.5.1, 4.6, 4.6.1 | [v4.1.0](https://www.nuget.org/packages/Joveler.Compression.ZLib/4.1.0) | + +### Supported OS platforms + +- Windows (x86, x64, arm64) +- Linux (x64, arm, arm64) +- macOS (x64) diff --git a/Joveler.Compression.ZLib/README.md b/Joveler.Compression.ZLib/README.md index 207a832..9eb6369 100644 --- a/Joveler.Compression.ZLib/README.md +++ b/Joveler.Compression.ZLib/README.md @@ -20,9 +20,9 @@ Joveler.Compression.ZLib can be installed via [nuget](https://www.nuget.org/pack ### Targeted .NET platforms -- .NET Core 3.1+ -- .NET Standard 2.0 (.NET Framework 4.6.1+, .NET Core 2.0+) -- .NET Framework 4.6.2+ +- .NET Core 3.1 +- .NET Standard 2.0 +- .NET Framework 4.6.2 #### Discontinued target frameworks diff --git a/Joveler.Compression.ZLib/ZLibLoader.cs b/Joveler.Compression.ZLib/ZLibLoader.cs index a36b26b..bdd7f3d 100644 --- a/Joveler.Compression.ZLib/ZLibLoader.cs +++ b/Joveler.Compression.ZLib/ZLibLoader.cs @@ -30,8 +30,6 @@ 3. This notice may not be removed or altered from any source distribution. using Joveler.DynLoader; using System; using System.Runtime.InteropServices; -// ReSharper disable UnusedMember.Global -// ReSharper disable InconsistentNaming namespace Joveler.Compression.ZLib { @@ -51,7 +49,7 @@ protected override string DefaultLibFileName { get { -#if !NET451 +#if !NETFRAMEWORK if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return "libz.so.1"; else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) diff --git a/Joveler.Compression.ZLib/ZLibStreams.cs b/Joveler.Compression.ZLib/ZLibStreams.cs index 3266ba7..b804fea 100644 --- a/Joveler.Compression.ZLib/ZLibStreams.cs +++ b/Joveler.Compression.ZLib/ZLibStreams.cs @@ -584,20 +584,20 @@ public override unsafe void Flush() /// public override long Seek(long offset, SeekOrigin origin) { - throw new NotSupportedException("Seek() not supported"); + throw new NotSupportedException($"{nameof(Seek)}() not supported."); } /// public override void SetLength(long value) { - throw new NotSupportedException("SetLength not supported"); + throw new NotSupportedException($"{nameof(SetLength)} not supported."); } /// - public override long Length => throw new NotSupportedException("Length not supported"); + public override long Length => throw new NotSupportedException($"{nameof(Length)} not supported."); /// public override long Position { - get => throw new NotSupportedException("Position not supported"); - set => throw new NotSupportedException("Position not supported"); + get => throw new NotSupportedException($"{nameof(Position)} not supported."); + set => throw new NotSupportedException($"{nameof(Position)} not supported."); } public double CompressionRatio diff --git a/Joveler.Compression.ZLib/runtimes/linux-arm/native/libz.so b/Joveler.Compression.ZLib/runtimes/linux-arm/native/libz.so index 6e8848f..09d944c 100755 Binary files a/Joveler.Compression.ZLib/runtimes/linux-arm/native/libz.so and b/Joveler.Compression.ZLib/runtimes/linux-arm/native/libz.so differ diff --git a/Joveler.Compression.ZLib/runtimes/linux-arm64/native/libz.so b/Joveler.Compression.ZLib/runtimes/linux-arm64/native/libz.so index 6705193..7bcc950 100755 Binary files a/Joveler.Compression.ZLib/runtimes/linux-arm64/native/libz.so and b/Joveler.Compression.ZLib/runtimes/linux-arm64/native/libz.so differ diff --git a/Joveler.Compression.ZLib/runtimes/linux-x64/native/libz.so b/Joveler.Compression.ZLib/runtimes/linux-x64/native/libz.so index 899f947..30b3bc9 100644 Binary files a/Joveler.Compression.ZLib/runtimes/linux-x64/native/libz.so and b/Joveler.Compression.ZLib/runtimes/linux-x64/native/libz.so differ diff --git a/Joveler.Compression.ZLib/runtimes/osx-arm64/native/libz.dylib b/Joveler.Compression.ZLib/runtimes/osx-arm64/native/libz.dylib index 1e108c4..eab0afc 100755 Binary files a/Joveler.Compression.ZLib/runtimes/osx-arm64/native/libz.dylib and b/Joveler.Compression.ZLib/runtimes/osx-arm64/native/libz.dylib differ diff --git a/Joveler.Compression.ZLib/runtimes/osx-x64/native/libz.dylib b/Joveler.Compression.ZLib/runtimes/osx-x64/native/libz.dylib index fa4564e..8a34a0c 100755 Binary files a/Joveler.Compression.ZLib/runtimes/osx-x64/native/libz.dylib and b/Joveler.Compression.ZLib/runtimes/osx-x64/native/libz.dylib differ diff --git a/Joveler.Compression.ZLib/runtimes/win-arm64/native/zlibwapi.dll b/Joveler.Compression.ZLib/runtimes/win-arm64/native/zlibwapi.dll index e62350e..25d9a98 100644 Binary files a/Joveler.Compression.ZLib/runtimes/win-arm64/native/zlibwapi.dll and b/Joveler.Compression.ZLib/runtimes/win-arm64/native/zlibwapi.dll differ diff --git a/Joveler.Compression.ZLib/runtimes/win-x64/native/zlibwapi.dll b/Joveler.Compression.ZLib/runtimes/win-x64/native/zlibwapi.dll index 3df48ee..45d9bdd 100644 Binary files a/Joveler.Compression.ZLib/runtimes/win-x64/native/zlibwapi.dll and b/Joveler.Compression.ZLib/runtimes/win-x64/native/zlibwapi.dll differ diff --git a/Joveler.Compression.ZLib/runtimes/win-x86/native/zlibwapi.dll b/Joveler.Compression.ZLib/runtimes/win-x86/native/zlibwapi.dll index ddb66d1..e8e546c 100644 Binary files a/Joveler.Compression.ZLib/runtimes/win-x86/native/zlibwapi.dll and b/Joveler.Compression.ZLib/runtimes/win-x86/native/zlibwapi.dll differ diff --git a/Joveler.Compression.Zstd.Tests/Joveler.Compression.Zstd.Tests.csproj b/Joveler.Compression.Zstd.Tests/Joveler.Compression.Zstd.Tests.csproj index df9bba8..3337464 100644 --- a/Joveler.Compression.Zstd.Tests/Joveler.Compression.Zstd.Tests.csproj +++ b/Joveler.Compression.Zstd.Tests/Joveler.Compression.Zstd.Tests.csproj @@ -1,11 +1,9 @@ - - - net48;net6.0 - - net6.0 + + + net481;net6.0 net6.0 @@ -15,8 +13,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/Joveler.Compression.Zstd.Tests/ZstdStreamTests.cs b/Joveler.Compression.Zstd.Tests/ZstdStreamTests.cs index f25fdd9..9e8c353 100644 --- a/Joveler.Compression.Zstd.Tests/ZstdStreamTests.cs +++ b/Joveler.Compression.Zstd.Tests/ZstdStreamTests.cs @@ -33,11 +33,8 @@ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR using Microsoft.VisualStudio.TestTools.UnitTesting; using System; -using System.Collections.Generic; using System.IO; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace Joveler.Compression.Zstd.Tests { diff --git a/Joveler.Compression.Zstd/Joveler.Compression.Zstd.csproj b/Joveler.Compression.Zstd/Joveler.Compression.Zstd.csproj index a06dd2c..bbac401 100644 --- a/Joveler.Compression.Zstd/Joveler.Compression.Zstd.csproj +++ b/Joveler.Compression.Zstd/Joveler.Compression.Zstd.csproj @@ -19,7 +19,7 @@ - + x86\%(FileName)%(Extension) @@ -29,8 +29,12 @@ x64\%(FileName)%(Extension) PreserveNewest + + arm64\%(FileName)%(Extension) + PreserveNewest + - + runtimes\win-x86\native\%(FileName)%(Extension) diff --git a/Joveler.Compression.Zstd/Joveler.Compression.Zstd.netfx.targets b/Joveler.Compression.Zstd/Joveler.Compression.Zstd.netfx.targets index 29ec1a8..908997a 100644 --- a/Joveler.Compression.Zstd/Joveler.Compression.Zstd.netfx.targets +++ b/Joveler.Compression.Zstd/Joveler.Compression.Zstd.netfx.targets @@ -32,6 +32,7 @@ HasTrailingSlash('$(MSBuildThisFileDirectory)')"> + - + CollectInteropLibFiles_x86; - CollectInteropLibFiles_x64; + CollectInteropLibFiles_x64 + CollectInteropLibFiles_arm64; $(PipelineCollectFilesPhaseDependsOn); diff --git a/Joveler.Compression.Zstd/ZstdDictionary.cs b/Joveler.Compression.Zstd/ZstdDictionary.cs index 6ed831e..c66dea3 100644 --- a/Joveler.Compression.Zstd/ZstdDictionary.cs +++ b/Joveler.Compression.Zstd/ZstdDictionary.cs @@ -1,10 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace Joveler.Compression.Zstd +namespace Joveler.Compression.Zstd { public class ZstdDictionary { diff --git a/Joveler.Compression.Zstd/ZstdException.cs b/Joveler.Compression.Zstd/ZstdException.cs index 73c5ed0..38259a6 100644 --- a/Joveler.Compression.Zstd/ZstdException.cs +++ b/Joveler.Compression.Zstd/ZstdException.cs @@ -33,12 +33,8 @@ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR using System; -using System.Collections.Generic; -using System.Linq; using System.Runtime.InteropServices; using System.Runtime.Serialization; -using System.Text; -using System.Threading.Tasks; namespace Joveler.Compression.Zstd { diff --git a/Joveler.Compression.Zstd/ZstdLoader.cs b/Joveler.Compression.Zstd/ZstdLoader.cs index 6cb50fe..1656a2c 100644 --- a/Joveler.Compression.Zstd/ZstdLoader.cs +++ b/Joveler.Compression.Zstd/ZstdLoader.cs @@ -48,7 +48,7 @@ protected override string DefaultLibFileName { get { -#if !NET451 +#if !NETFRAMEWORK if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) return "libzstd.so.1"; else if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) @@ -417,13 +417,13 @@ UIntPtr dictSize // size_t [UnmanagedFunctionPointer(CallingConvention.Cdecl)] internal delegate UIntPtr ZSTD_DCtx_reset(IntPtr dctx, ResetDirective reset); internal ZSTD_DCtx_reset DctxReset; - + /* [UnmanagedFunctionPointer(CallingConvention.Cdecl)] internal delegate UIntPtr ZSTD_initDStream(IntPtr zds); internal ZSTD_initDStream InitDStream; */ - + [UnmanagedFunctionPointer(CallingConvention.Cdecl)] internal delegate UIntPtr ZSTD_decompressStream( diff --git a/Joveler.Compression.Zstd/ZstdStream.cs b/Joveler.Compression.Zstd/ZstdStream.cs index f97a87a..ea57bdf 100644 --- a/Joveler.Compression.Zstd/ZstdStream.cs +++ b/Joveler.Compression.Zstd/ZstdStream.cs @@ -32,9 +32,7 @@ DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR */ using System; -using System.Diagnostics; using System.IO; -using System.Linq; using System.Runtime.CompilerServices; namespace Joveler.Compression.Zstd @@ -375,8 +373,30 @@ protected override void Dispose(bool disposing) ZstdInit.Lib.FreeCStream(_cstream); _cstream = IntPtr.Zero; + + /* 'Joveler.Compression.Zstd (netstandard2.0)' 프로젝트에서 병합되지 않은 변경 내용 + 이전: + } + + if (_dstream != IntPtr.Zero) + 이후: + } + + if (_dstream != IntPtr.Zero) + */ + + /* 'Joveler.Compression.Zstd (netcoreapp3.1)' 프로젝트에서 병합되지 않은 변경 내용 + 이전: + } + + if (_dstream != IntPtr.Zero) + 이후: + } + + if (_dstream != IntPtr.Zero) + */ } - + if (_dstream != IntPtr.Zero) { ZstdInit.Lib.FreeDStream(_dstream); @@ -457,8 +477,30 @@ public unsafe int Read(Span span) _decompSrcIdx = DecompressComplete; break; } + + /* 'Joveler.Compression.Zstd (netstandard2.0)' 프로젝트에서 병합되지 않은 변경 내용 + 이전: + } + + // Setup InBuffer + 이후: + } + + // Setup InBuffer + */ + + /* 'Joveler.Compression.Zstd (netcoreapp3.1)' 프로젝트에서 병합되지 않은 변경 내용 + 이전: + } + + // Setup InBuffer + 이후: + } + + // Setup InBuffer + */ } - + // Setup InBuffer InBuffer inBuf = new InBuffer() { @@ -614,7 +656,7 @@ private unsafe void FinishWrite() } while (0 < stillToFlush); } - + } /// @@ -662,7 +704,7 @@ public override void Flush() } else if (_mode == Mode.Decompress) { - + } BaseStream.Flush(); diff --git a/native/lz4/README.md b/native/lz4/README.md index 9d0a022..b4ff701 100644 --- a/native/lz4/README.md +++ b/native/lz4/README.md @@ -18,10 +18,17 @@ Windows .dll files are compiled with size optimization. - Create ARM64 solution platform, using x64 as a template 1. Choose `Release - All Platforms` build target 1. Set build configurations - - Set `C/C++` - `Optimization` - `Optimization` as `Minimum Size (/O1)` - - Set `C/C++` - `Optimization` - `Small or Fast` as `Favor Small Code (/Os)` - - Set `C/C++` - `Code Generation` - `Use Run-Time Library` as `Multi Thread(/MT)` - - Set `Linker` - `Debugging` - `Generate Debug Info` as `None` +1. Set build configurations + - Korean + - Set `C/C++` - `최적화` - `최적화` as `최대 최적화(크기 우선)(/O1)` + - Set `C/C++` - `최적화` - `크기 또는 속도` as `코드 크기 우선(/Os)` + - Set `C/C++` - `코드 생성` - `런타임 라이브러리` as `다중 스레드(/MT)` + - Set `Linker` - `디버깅` - `디버그 정보 생성` as `아니요` + - English + - Set `C/C++` - `Optimization` - `Optimization` as `Minimum Size (/O1)` + - Set `C/C++` - `Optimization` - `Small or Fast` as `Favor Small Code (/Os)` + - Set `C/C++` - `Code Generation` - `Use Run-Time Library` as `Multi Thread(/MT)` + - Set `Linker` - `Debugging` - `Generate Debug Info` as `None` 1. Build the project and obtain `liblz4.dll` ## Linux - x64, armhf, arm64 diff --git a/native/xz-utils/README.md b/native/xz-utils/README.md index c1ee75a..33ebf4d 100644 --- a/native/xz-utils/README.md +++ b/native/xz-utils/README.md @@ -8,13 +8,21 @@ xz-utils source can be obtained from [homepage](https://tukaani.org/xz/). ## Windows - x86, x64, arm64 +### 5.4.0 + +xz 5.4.0 MSVC vcxproj files has an issue regarding `lzma_stream_decoder_mt`. + +Use MSYS2 as a workaround. + +### 5.2.x + | Arch | Obtain Method | |-------|---------------| | x86 | From official release, `xz-(ver)-windows\bin_i686-sse2\liblzma.dll` with strip | | x64 | From official release, `xz-(ver)-windows\bin_x86-64\liblzma.dll` with strip | | arm64 | Manual compile with MSVC 2019 | -### Manual Compile ARM64 DLL +#### Manual Compile ARM64 DLL 1. Open `windows\vs2019\xz_win.sln` with MSVC 2019 1. Select `liblzma_dll` project @@ -22,12 +30,18 @@ xz-utils source can be obtained from [homepage](https://tukaani.org/xz/). 1. Create ARM64 target platform - Open `Configuration Manager` - Create ARM64 solution platform, using x64 as a template -1. Choose `Release - All Platforms` build target +1. Choose `ReleaseMT - All Platforms` build target 1. Set build configurations - - Set `C/C++` - `Optimization` - `Optimization` as `Minimum Size (/O1)` - - Set `C/C++` - `Optimization` - `Small or Fast` as `Favor Small Code (/Os)` - - Set `C/C++` - `Code Generation` - `Use Run-Time Library` as `Multi Thread(/MT)` - - Set `Linker` - `Debugging` - `Generate Debug Info` as `None` + - Korean + - Set `C/C++` - `최적화` - `최적화` as `최대 최적화(크기 우선)(/O1)` + - Set `C/C++` - `최적화` - `크기 또는 속도` as `코드 크기 우선(/Os)` + - Set `C/C++` - `코드 생성` - `런타임 라이브러리` as `다중 스레드(/MT)` + - Set `Linker` - `디버깅` - `디버그 정보 생성` as `아니요` + - English + - Set `C/C++` - `Optimization` - `Optimization` as `Minimum Size (/O1)` + - Set `C/C++` - `Optimization` - `Small or Fast` as `Favor Small Code (/Os)` + - Set `C/C++` - `Code Generation` - `Use Run-Time Library` as `Multi Thread(/MT)` + - Set `Linker` - `Debugging` - `Generate Debug Info` as `None` 1. Build the project and obtain `liblzma.dll` ## Linux - x64, armhf, arm64 @@ -38,13 +52,13 @@ Linux .so files are built with default optimization. ```sh make -j(N) ``` -1. Strip `lib/liblzma.(ver).so` +1. Strip `src/liblzma/.libs/liblzma.so.(ver)` ```sh - strip lib/liblzma.(ver).so + strip src/liblzma/.libs/liblzma.so.(ver) ``` 1. Make sure the binary does not have unnecessary dependency ```sh - ldd lib/liblzma.(ver).so + ldd src/liblzma/.libs/liblzma.so.(ver) ``` ## macOS - x64, arm64 diff --git a/native/xz-utils/posix/liblzma-posix.sh b/native/xz-utils/posix/liblzma-posix.sh new file mode 100755 index 0000000..4580347 --- /dev/null +++ b/native/xz-utils/posix/liblzma-posix.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# Compile liblzma on Linux/macOS + +# Usage: +# ./liblzma-posix.sh ~/build/native/xz-5.4.3 + +# Check script arguments +if [[ "$#" -ne 1 ]]; then + echo "Usage: $0 " >&2 + exit 1 +fi +if ! [[ -d "$1" ]]; then + echo "[$1] is not a directory!" >&2 + exit 1 +fi +SRCDIR=$1 + +# Query environment info +OS=$(uname -s) # Linux, Darwin, MINGW64_NT-10.0-19042, MSYS_NT-10.0-18363, ... + +# Set path and command vars +# BASE_ABS_PATH: Absolute path of this script, e.g. /home/user/bin/foo.sh +# BASE_DIR: Absolute path of the parent dir of this script, e.g. /home/user/bin +if [ "${OS}" = Linux ]; then + BASE_ABS_PATH=$(readlink -f "$0") + CORES=$(grep -c ^processor /proc/cpuinfo) + DEST_LIB="liblzma.so" + DEST_EXE="xz" + STRIP="strip" + CHECKDEP="ldd" +elif [ "${OS}" = Darwin ]; then + BASE_ABS_PATH="$(cd $(dirname "$0");pwd)/$(basename "$0")" + CORES=$(sysctl -n hw.logicalcpu) + DEST_LIB="liblzma.dylib" + DEST_EXE="xz" + STRIP="strip -x" + CHECKDEP="otool -L" +else + echo "[${OS}] is not a supported platform!" >&2 + exit 1 +fi +BASE_DIR=$(dirname "${BASE_ABS_PATH}") +DEST_DIR="${BASE_DIR}/build" + +# Create dest directory +mkdir -p "${DEST_DIR}" + +# Compile liblzma, xz +BUILD_MODES=( "lib" "exe" ) +pushd "${SRCDIR}" > /dev/null +for BUILD_MODE in "${BUILD_MODES[@]}"; do + CONFIGURE_ARGS="" + if [ "$BUILD_MODE" = "lib" ]; then + CONFIGURE_ARGS="--enable-shared --disable-xz" + elif [ "$BUILD_MODE" = "exe" ]; then + CONFIGURE_ARGS="--disable-shared" + fi + + make clean + ./configure --host=${TARGET_TRIPLE} \ + --disable-debug \ + --disable-dependency-tracking \ + --disable-nls \ + --disable-scripts \ + --disable-xzdec \ + --disable-lzmadec \ + --disable-lzmainfo \ + --disable-lzma-links \ + ${CONFIGURE_ARGS} + make "-j${CORES}" + + if [ "$BUILD_MODE" = "lib" ]; then + cp "src/liblzma/.libs/${DEST_LIB}" "${DEST_DIR}/${DEST_LIB}" + elif [ "$BUILD_MODE" = "exe" ]; then + cp "src/xz/${DEST_EXE}" "${DEST_DIR}/${DEST_EXE}" + fi +done +popd > /dev/null + +# Strip a binary +pushd "${DEST_DIR}" > /dev/null +ls -lh "${DEST_LIB}" +${STRIP} "${DEST_LIB}" +ls -lh "${DEST_LIB}" +popd > /dev/null + +# Check dependency of a binary +pushd "${DEST_DIR}" > /dev/null +${CHECKDEP} "${DEST_LIB}" +popd > /dev/null + diff --git a/native/xz-utils/windows/liblzma-msys2.sh b/native/xz-utils/windows/liblzma-msys2.sh new file mode 100644 index 0000000..cb34668 --- /dev/null +++ b/native/xz-utils/windows/liblzma-msys2.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Compile liblzma for Windows on MSYS2 + +# Usage: +# ./liblzma-msys2.sh -a i686 /d/build/native/xz-5.4.0 +# ./liblzma-msys2.sh -a x86_64 /d/build/native/xz-5.4.0 +# ./liblzma-msys2.sh -a aarch64 -t /c/llvm-mingw /d/build/native/xz-5.4.0 + +# Check script arguments +while getopts "a:t:" opt; do + case $opt in + a) # architecture + ARCH=$OPTARG + ;; + t) # toolchain, required for aarch64 + TOOLCHAIN_DIR=$OPTARG + ;; + :) + echo "Usage: $0 <-a i686|x86_64|aarch64> [-t TOOLCHAIN_DIR] " >&2 + exit 1 + ;; + esac +done +# Parse +shift $(( OPTIND - 1 )) +SRCDIR="$@" +if ! [[ -d "${SRCDIR}" ]]; then + echo "[${SRCDIR}] is not a directory!" >&2 + exit 1 +fi + +# Set path and command vars +# BASE_ABS_PATH: Absolute path of this script, e.g. /home/user/bin/foo.sh +# BASE_DIR: Absolute path of the parent dir of this script, e.g. /home/user/bin +BASE_ABS_PATH=$(readlink -f "$0") +BASE_DIR=$(dirname "${BASE_ABS_PATH}") +DEST_DIR=${BASE_DIR}/build-${ARCH} +CORES=$(grep -c ^processor /proc/cpuinfo) + +# Set library paths +DEST_LIB="liblzma.dll" +STRIP="strip" +CHECKDEP="ldd" + +# Set target triple +if [ "${ARCH}" = i686 ]; then + TARGET_TRIPLE="i686-w64-mingw32" +elif [ "${ARCH}" = x86_64 ]; then + TARGET_TRIPLE="x86_64-w64-mingw32" +elif [ "${ARCH}" = aarch64 ]; then + TARGET_TRIPLE="aarch64-w64-mingw32" + # Let custom toolchain is called first in PATH + if [[ -z "${TOOLCHAIN_DIR}" ]]; then + echo "Please provide llvm-mingw as [TOOLCHAIN_DIR] for aarch64 build." >&2 + exit 1 + fi +else + HOST_ARCH=$(uname -m) + echo "[${ARCH}] is not a supported architecture, Ex) use '-a ${HOST_ARCH}'" >&2 + exit 1 +fi + +# Create dest directory +rm -rf "${DEST_DIR}" +mkdir -p "${DEST_DIR}" + +# Let custom toolchain is called first in PATH +if ! [[ -z "${TOOLCHAIN_DIR}" ]]; then + export PATH=${TOOLCHAIN_DIR}/bin:${PATH} +fi + +# Compile liblzma, xz +BUILD_MODES=( "lib" "exe" ) +pushd "${SRCDIR}" > /dev/null +for BUILD_MODE in "${BUILD_MODES[@]}"; do + CONFIGURE_ARGS="" + if [ "$BUILD_MODE" = "lib" ]; then + CONFIGURE_ARGS="--enable-shared --disable-xz" + elif [ "$BUILD_MODE" = "exe" ]; then + CONFIGURE_ARGS="--disable-shared" + fi + + make clean + ./configure --host=${TARGET_TRIPLE} \ + --disable-debug \ + --disable-dependency-tracking \ + --disable-nls \ + --disable-scripts \ + --disable-xzdec \ + --disable-lzmadec \ + --disable-lzmainfo \ + --disable-lzma-links \ + ${CONFIGURE_ARGS} + make "-j${CORES}" + + if [ "$BUILD_MODE" = "lib" ]; then + cp "src/liblzma/.libs/liblzma-5.dll" "${DEST_DIR}/${DEST_LIB}" + elif [ "$BUILD_MODE" = "exe" ]; then + cp "src/xz/xz.exe" "${DEST_DIR}/${DEST_EXE}" + fi +done +popd > /dev/null + +# Strip binaries +pushd "${DEST_DIR}" > /dev/null +ls -lh *.dll *.exe +${STRIP} "${DEST_LIB}" "${DEST_EXE}" +ls -lh *.dll *.exe +popd > /dev/null + +# Print dependency of binraies +pushd "${DEST_DIR}" > /dev/null +${CHECKDEP} "${DEST_LIB}" +popd > /dev/null diff --git a/native/zlib/README.md b/native/zlib/README.md index f888644..170102d 100644 --- a/native/zlib/README.md +++ b/native/zlib/README.md @@ -10,36 +10,43 @@ zlib source can be obtained from its [homepage](https://zlib.net). Windows .dll files are compiled with size optimization. -1. Open `build\VS2017\lz4.sln` with MSVC 2017 or later -1. Select `liblz4-dll` project +1. Open `contrib\vstudio\vc14\zlibvc.sln` with MSVC 2017 or later +1. Select `zlibvc` project 1. Open `Property` in context menu 1. Create ARM64 target platform - Open `Configuration Manager` - Create ARM64 solution platform, using x64 as a template -1. Choose `Release - All Platforms` build target +1. Choose `ReleaseWithoutAsm - All Platforms` build target 1. Set build configurations - - Set `C/C++` - `Optimization` - `Optimization` as `Minimum Size (/O1)` - - Set `C/C++` - `Optimization` - `Small or Fast` as `Favor Small Code (/Os)` - - Set `C/C++` - `Code Generation` - `Use Run-Time Library` as `Multi Thread(/MT)` - - Set `Linker` - `Debugging` - `Generate Debug Info` as `None` -1. Build the project and obtain `liblz4.dll` + - Korean + - Set `C/C++` - `최적화` - `최적화` as `최대 최적화(크기 우선)(/O1)` + - Set `C/C++` - `최적화` - `크기 또는 속도` as `코드 크기 우선(/Os)` + - Set `C/C++` - `코드 생성` - `런타임 라이브러리` as `다중 스레드(/MT)` + - Set `Linker` - `디버깅` - `디버그 정보 생성` as `아니요` + - English + - Set `C/C++` - `Optimization` - `Optimization` as `Minimum Size (/O1)` + - Set `C/C++` - `Optimization` - `Small or Fast` as `Favor Small Code (/Os)` + - Set `C/C++` - `Code Generation` - `Use Run-Time Library` as `Multi Thread(/MT)` + - Set `Linker` - `Debugging` - `Generate Debug Info` as `None` +1. Build the project and obtain `zlibwapi.dll` ## Linux - x64, armhf, arm64 -Linux .so files are built with default optimization. +Linux .so files are built with default optimization, using autotools and make. +1. Configure Makefile 1. Build with standard Makefile - ```sh - make -j(N) - ``` -1. Strip `lib/libzstd.(ver).so` - ```sh - strip lib/libzstd.(ver).so - ``` +1. Strip `libz.so.(ver)` 1. Make sure the binary does not have unnecessary dependency - ```sh - ldd lib/libzstd.(ver).so - ``` + +```sh +./configure +make -j`nproc` +ls -l libz.so.(ver) +strip libz.so.(ver) +ls -l libz.so.(ver) +ldd libz.so.(ver) +``` Patch `zstd` cli Makefile to prevent it from linking to unnecessary dependency. - Set `HAVE_ZLIB`, `HAVE_LZ4`, `HAVE_LZMA` to 0.