Merge pull request #6 from m0nsky/wip_december_update_fixes_v2

Wip december update fixes v2
SciSharp · Jan 19, 2025 · 695a4da · 695a4da
2 parents 3521b27 + 91effe9
commit 695a4da
Show file tree

Hide file tree

Showing 12 changed files with 133 additions and 33 deletions.
diff --git a/LLama/LLamaSharp.Runtime.targets b/LLama/LLamaSharp.Runtime.targets
@@ -263,6 +263,14 @@
           <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
           <Link>runtimes/osx-arm64/native/libggml-cpu.dylib</Link>
       </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-arm64/libggml-metal.dylib">
+          <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+          <Link>runtimes/osx-arm64/native/libggml-metal.dylib</Link>
+      </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-arm64/libggml-blas.dylib">
+          <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+          <Link>runtimes/osx-arm64/native/libggml-blas.dylib</Link>
+      </None>
       <None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-arm64/libggml.dylib">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/osx-arm64/native/libggml.dylib</Link>
@@ -288,6 +296,10 @@
           <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
           <Link>runtimes/osx-x64/native/libggml-cpu.dylib</Link>
       </None>
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-x64/libggml-blas.dylib">
+          <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+          <Link>runtimes/osx-x64/native/libggml-blas.dylib</Link>
+      </None>
       <None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-x64/libggml.dylib">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/osx-x64/native/libggml.dylib</Link>
@@ -309,6 +321,10 @@
           <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
           <Link>runtimes/osx-x64/native/rosetta2/libggml-cpu.dylib</Link>
       </None>        
+      <None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-x64-rosetta2/libggml-blas.dylib">
+          <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+          <Link>runtimes/osx-x64/native/rosetta2/libggml-blas.dylib</Link>
+      </None>        
       <None Include="$(MSBuildThisFileDirectory)runtimes/deps/osx-x64-rosetta2/libggml.dylib">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
         <Link>runtimes/osx-x64/native/rosetta2/libggml.dylib</Link>

diff --git a/LLama/LLamaSharp.csproj b/LLama/LLamaSharp.csproj
@@ -56,7 +56,7 @@
   </ItemGroup>
 
   <PropertyGroup>
-    <BinaryReleaseId>0827b2c1da-v2</BinaryReleaseId>
+    <BinaryReleaseId>0827b2c1da-v5</BinaryReleaseId>
   </PropertyGroup>
 
   <PropertyGroup>

diff --git a/LLama/Native/Load/NativeLibraryUtils.cs b/LLama/Native/Load/NativeLibraryUtils.cs
@@ -71,31 +71,36 @@ internal static IntPtr TryLoadLibrary(NativeLibraryConfig config, out INativeLib
                     {
                         if (systemInfo.OSPlatform == OSPlatform.OSX)
                         {
-                            // // ggml-metal (uncomment if needed, requires testing)
-                            // if (os == "osx-arm64")
-                            //     dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-metal{ext}"));
+                            // On OSX, we should load the CPU backend from the current directory
 
                             // ggml-cpu
-                            // On OSX, we should load the CPU backend from the current directory
                             dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-cpu{ext}"));
+
+                            // ggml-metal (only supported on osx-arm64)
+                            if (os == "osx-arm64")
+                                dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-metal{ext}"));
+
+                            // ggml-blas (osx-x64, osx-x64-rosetta2 and osx-arm64 all have blas)
+                            dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-blas{ext}"));
                         }
                         else
                         {
+                            // On other platforms (Windows, Linux), we need to load the CPU backend from the specified AVX level directory
+                            // We are using the AVX level supplied by NativeLibraryConfig, which automatically detects the highest supported AVX level for us
+
+                            // ggml-cpu
+                            dependencyPaths.Add(Path.Combine(
+                                $"runtimes/{os}/native/{NativeLibraryConfig.AvxLevelToString(library.Metadata.AvxLevel)}",
+                                $"{libPrefix}ggml-cpu{ext}"
+                            ));
+
                             // ggml-cuda
                             if (library.Metadata.UseCuda)
                                 dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-cuda{ext}"));
 
                             // ggml-vulkan
                             if (library.Metadata.UseVulkan)
                                 dependencyPaths.Add(Path.Combine(currentRuntimeDirectory, $"{libPrefix}ggml-vulkan{ext}"));
-
-                            // ggml-cpu
-                            // On other platforms (Windows, Linux), we need to load the CPU backend from the specified AVX level directory
-                            // We are using the AVX level supplied by NativeLibraryConfig, which automatically detects the highest supported AVX level for us
-                            dependencyPaths.Add(Path.Combine(
-                                $"runtimes/{os}/native/{NativeLibraryConfig.AvxLevelToString(library.Metadata.AvxLevel)}",
-                                $"{libPrefix}ggml-cpu{ext}"
-                            ));
                         }
                     }
 

diff --git a/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec b/LLama/runtimes/build/LLamaSharp.Backend.Cpu.nuspec
@@ -18,53 +18,77 @@
   <files>
     <file src="LLamaSharpBackend.props" target="build/netstandard2.0/LLamaSharp.Backend.Cpu.props" />
 
+    <file src="runtimes/deps/ggml.dll" target="runtimes\win-x64\native\ggml.dll" />
     <file src="runtimes/deps/ggml-base.dll" target="runtimes\win-x64\native\ggml-base.dll" />
     <file src="runtimes/deps/ggml-cpu.dll" target="runtimes\win-x64\native\ggml-cpu.dll" />
     <file src="runtimes/deps/llama.dll" target="runtimes\win-x64\native\llama.dll" />
+    <file src="runtimes/deps/llava_shared.dll" target="runtimes\win-x64\native\llava_shared.dll" />
 
+    <file src="runtimes/deps/avx/ggml.dll" target="runtimes\win-x64\native\avx\ggml.dll" />
     <file src="runtimes/deps/avx/ggml-base.dll" target="runtimes\win-x64\native\avx\ggml-base.dll" />
     <file src="runtimes/deps/avx/ggml-cpu.dll" target="runtimes\win-x64\native\avx\ggml-cpu.dll" />
     <file src="runtimes/deps/avx/llama.dll" target="runtimes\win-x64\native\avx\llama.dll" />
+    <file src="runtimes/deps/avx/llava_shared.dll" target="runtimes\win-x64\native\avx\llava_shared.dll" />
 
+    <file src="runtimes/deps/avx2/ggml.dll" target="runtimes\win-x64\native\avx2\ggml.dll" />
     <file src="runtimes/deps/avx2/ggml-base.dll" target="runtimes\win-x64\native\avx2\ggml-base.dll" />
     <file src="runtimes/deps/avx2/ggml-cpu.dll" target="runtimes\win-x64\native\avx2\ggml-cpu.dll" />
     <file src="runtimes/deps/avx2/llama.dll" target="runtimes\win-x64\native\avx2\llama.dll" />
+    <file src="runtimes/deps/avx2/llava_shared.dll" target="runtimes\win-x64\native\avx2\llava_shared.dll" />
 
+    <file src="runtimes/deps/avx512/ggml.dll" target="runtimes\win-x64\native\avx512\ggml.dll" />
     <file src="runtimes/deps/avx512/ggml-base.dll" target="runtimes\win-x64\native\avx512\ggml-base.dll" />
     <file src="runtimes/deps/avx512/ggml-cpu.dll" target="runtimes\win-x64\native\avx512\ggml-cpu.dll" />
     <file src="runtimes/deps/avx512/llama.dll" target="runtimes\win-x64\native\avx512\llama.dll" />
+    <file src="runtimes/deps/avx512/llava_shared.dll" target="runtimes\win-x64\native\avx512\llava_shared.dll" />
 
     <file src="runtimes/deps/libggml.so" target="runtimes\linux-x64\native\libggml.so" />
+    <file src="runtimes/deps/libggml-base.so" target="runtimes\linux-x64\native\libggml-base.so" />
+    <file src="runtimes/deps/libggml-cpu.so" target="runtimes\linux-x64\native\libggml-cpu.so" />
     <file src="runtimes/deps/libllama.so" target="runtimes\linux-x64\native\libllama.so" />
+    <file src="runtimes/deps/libllava_shared.so" target="runtimes\linux-x64\native\libllava_shared.so" />
+
     <file src="runtimes/deps/avx/libggml.so" target="runtimes\linux-x64\native\avx\libggml.so" />
+    <file src="runtimes/deps/avx/libggml-base.so" target="runtimes\linux-x64\native\avx\libggml-base.so" />
+    <file src="runtimes/deps/avx/libggml-cpu.so" target="runtimes\linux-x64\native\avx\libggml-cpu.so" />
     <file src="runtimes/deps/avx/libllama.so" target="runtimes\linux-x64\native\avx\libllama.so" />
+    <file src="runtimes/deps/avx/libllava_shared.so" target="runtimes\linux-x64\native\avx\libllava_shared.so" />
+
     <file src="runtimes/deps/avx2/libggml.so" target="runtimes\linux-x64\native\avx2\libggml.so" />
+    <file src="runtimes/deps/avx2/libggml-base.so" target="runtimes\linux-x64\native\avx2\libggml-base.so" />
+    <file src="runtimes/deps/avx2/libggml-cpu.so" target="runtimes\linux-x64\native\avx2\libggml-cpu.so" />
     <file src="runtimes/deps/avx2/libllama.so" target="runtimes\linux-x64\native\avx2\libllama.so" />
+    <file src="runtimes/deps/avx2/libllava_shared.so" target="runtimes\linux-x64\native\avx2\libllava_shared.so" />
+
     <file src="runtimes/deps/avx512/libggml.so" target="runtimes\linux-x64\native\avx512\libggml.so" />
+    <file src="runtimes/deps/avx512/libggml-base.so" target="runtimes\linux-x64\native\avx512\libggml-base.so" />
+    <file src="runtimes/deps/avx512/libggml-cpu.so" target="runtimes\linux-x64\native\avx512\libggml-cpu.so" />
     <file src="runtimes/deps/avx512/libllama.so" target="runtimes\linux-x64\native\avx512\libllama.so" />
+    <file src="runtimes/deps/avx512/libllava_shared.so" target="runtimes\linux-x64\native\avx512\libllava_shared.so" />
 
     <file src="runtimes/deps/osx-x64/libggml.dylib" target="runtimes\osx-x64\native\libggml.dylib" />
+    <file src="runtimes/deps/osx-x64/libggml-base.dylib" target="runtimes\osx-x64\native\libggml-base.dylib" />
+    <file src="runtimes/deps/osx-x64/libggml-cpu.dylib" target="runtimes\osx-x64\native\libggml-cpu.dylib" />
+    <file src="runtimes/deps/osx-x64/libggml-blas.dylib" target="runtimes\osx-x64\native\libggml-blas.dylib" />
     <file src="runtimes/deps/osx-x64/libllama.dylib" target="runtimes\osx-x64\native\libllama.dylib" />
     <file src="runtimes/deps/osx-x64/libllava_shared.dylib" target="runtimes\osx-x64\native\libllava_shared.dylib" />
 
     <file src="runtimes/deps/osx-x64-rosetta2/libggml.dylib" target="runtimes\osx-x64\native\rosetta2\libggml.dylib" />
+    <file src="runtimes/deps/osx-x64-rosetta2/libggml-base.dylib" target="runtimes\osx-x64\native\rosetta2\libggml-base.dylib" />
+    <file src="runtimes/deps/osx-x64-rosetta2/libggml-cpu.dylib" target="runtimes\osx-x64\native\rosetta2\libggml-cpu.dylib" />
+    <file src="runtimes/deps/osx-x64-rosetta2/libggml-blas.dylib" target="runtimes\osx-x64\native\rosetta2\libggml-blas.dylib" />
     <file src="runtimes/deps/osx-x64-rosetta2/libllama.dylib" target="runtimes\osx-x64\native\rosetta2\libllama.dylib" />
     <file src="runtimes/deps/osx-x64-rosetta2/libllava_shared.dylib" target="runtimes\osx-x64\native\rosetta2\libllava_shared.dylib" />
 
     <file src="runtimes/deps/osx-arm64/libggml.dylib" target="runtimes\osx-arm64\native\libggml.dylib" />
-    <file src="runtimes/deps/osx-arm64/libllama.dylib" target="runtimes\osx-arm64\native\libllama.dylib" />
+    <file src="runtimes/deps/osx-arm64/libggml-base.dylib" target="runtimes\osx-arm64\native\libggml-base.dylib" />
+    <file src="runtimes/deps/osx-arm64/libggml-cpu.dylib" target="runtimes\osx-arm64\native\libggml-cpu.dylib" />
+    <file src="runtimes/deps/osx-arm64/libggml-blas.dylib" target="runtimes\osx-arm64\native\libggml-blas.dylib" />
+    <file src="runtimes/deps/osx-arm64/libggml-metal.dylib" target="runtimes\osx-arm64\native\libggml-metal.dylib" />
     <file src="runtimes/deps/osx-arm64/ggml-metal.metal" target="runtimes\osx-arm64\native\ggml-metal.metal" />
+    <file src="runtimes/deps/osx-arm64/libllama.dylib" target="runtimes\osx-arm64\native\libllama.dylib" />
     <file src="runtimes/deps/osx-arm64/libllava_shared.dylib" target="runtimes\osx-arm64\native\libllava_shared.dylib" />
-
-    <file src="runtimes/deps/llava_shared.dll" target="runtimes\win-x64\native\llava_shared.dll" />
-    <file src="runtimes/deps/avx/llava_shared.dll" target="runtimes\win-x64\native\avx\llava_shared.dll" />
-    <file src="runtimes/deps/avx2/llava_shared.dll" target="runtimes\win-x64\native\avx2\llava_shared.dll" />
-    <file src="runtimes/deps/avx512/llava_shared.dll" target="runtimes\win-x64\native\avx512\llava_shared.dll" />
-
-    <file src="runtimes/deps/libllava_shared.so" target="runtimes\linux-x64\native\libllava_shared.so" />
-    <file src="runtimes/deps/avx/libllava_shared.so" target="runtimes\linux-x64\native\avx\libllava_shared.so" />
-    <file src="runtimes/deps/avx2/libllava_shared.so" target="runtimes\linux-x64\native\avx2\libllava_shared.so" />
-    <file src="runtimes/deps/avx512/libllava_shared.so" target="runtimes\linux-x64\native\avx512\libllava_shared.so" />
+
     <file src="icon512.png" target="icon512.png" />
   </files>
 </package>
diff --git a/LLama/runtimes/build/LLamaSharp.Backend.Cuda11.Linux.nuspec b/LLama/runtimes/build/LLamaSharp.Backend.Cuda11.Linux.nuspec
@@ -7,18 +7,27 @@
         <authors>llama.cpp Authors</authors>
         <requireLicenseAcceptance>false</requireLicenseAcceptance>
         <license type="expression">MIT</license>
+        <icon>icon512.png</icon>
         <projectUrl>https://github.com/SciSharp/LLamaSharp</projectUrl>
         <description>LLamaSharp.Backend.Cuda11.Linux contains the Linux binaries for LLamaSharp with Cuda11 support.</description>
         <releaseNotes></releaseNotes>
         <copyright>Copyright 2023 The llama.cpp Authors. All rights reserved.</copyright>
         <tags>LLamaSharp LLama LLM GPT AI ChatBot SciSharp</tags>
+
+        <dependencies>
+            <dependency id="LLamaSharp.Backend.Cpu" version="$version$" />
+        </dependencies>
     </metadata>
 
     <files>
         <file src="LLamaSharpBackend.props" target="build/netstandard2.0/LLamaSharp.Backend.Cuda11.props" />
-        <file src="runtimes/deps/cu11.7.1/libllava_shared.so" target="runtimes/linux-x64/native/cuda11/libllava_shared.so" />
+
         <file src="runtimes/deps/cu11.7.1/libggml.so" target="runtimes/linux-x64/native/cuda11/libggml.so" />
+        <file src="runtimes/deps/cu11.7.1/libggml-base.so" target="runtimes/linux-x64/native/cuda11/libggml-base.so" />
+        <file src="runtimes/deps/cu11.7.1/libggml-cuda.so" target="runtimes/linux-x64/native/cuda11/libggml-cuda.so" />
+
         <file src="runtimes/deps/cu11.7.1/libllama.so" target="runtimes/linux-x64/native/cuda11/libllama.so" />
+        <file src="runtimes/deps/cu11.7.1/libllava_shared.so" target="runtimes/linux-x64/native/cuda11/libllava_shared.so" />
 
         <file src="icon512.png" target="icon512.png" />
     </files>

diff --git a/LLama/runtimes/build/LLamaSharp.Backend.Cuda11.Windows.nuspec b/LLama/runtimes/build/LLamaSharp.Backend.Cuda11.Windows.nuspec
@@ -7,18 +7,27 @@
         <authors>llama.cpp Authors</authors>
         <requireLicenseAcceptance>false</requireLicenseAcceptance>
         <license type="expression">MIT</license>
+        <icon>icon512.png</icon>
         <projectUrl>https://github.com/SciSharp/LLamaSharp</projectUrl>
         <description>LLamaSharp.Backend.Cuda11.Windows contains the Windows binaries for LLamaSharp with Cuda11 support.</description>
         <releaseNotes></releaseNotes>
         <copyright>Copyright 2023 The llama.cpp Authors. All rights reserved.</copyright>
         <tags>LLamaSharp LLama LLM GPT AI ChatBot SciSharp</tags>
+
+        <dependencies>
+            <dependency id="LLamaSharp.Backend.Cpu" version="$version$" />
+        </dependencies>
     </metadata>
 
     <files>
         <file src="LLamaSharpBackend.props" target="build/netstandard2.0/LLamaSharp.Backend.Cuda11.props" />
-        <file src="runtimes/deps/cu11.7.1/llava_shared.dll" target="runtimes\win-x64\native\cuda11\llava_shared.dll" />
+
         <file src="runtimes/deps/cu11.7.1/ggml.dll" target="runtimes\win-x64\native\cuda11\ggml.dll" />
+        <file src="runtimes/deps/cu11.7.1/ggml-base.dll" target="runtimes\win-x64\native\cuda11\ggml-base.dll" />
+        <file src="runtimes/deps/cu11.7.1/ggml-cuda.dll" target="runtimes\win-x64\native\cuda11\ggml-cuda.dll" />
+
         <file src="runtimes/deps/cu11.7.1/llama.dll" target="runtimes\win-x64\native\cuda11\llama.dll" />
+        <file src="runtimes/deps/cu11.7.1/llava_shared.dll" target="runtimes\win-x64\native\cuda11\llava_shared.dll" />
 
         <file src="icon512.png" target="icon512.png" />
     </files>

diff --git a/LLama/runtimes/build/LLamaSharp.Backend.Cuda11.nuspec b/LLama/runtimes/build/LLamaSharp.Backend.Cuda11.nuspec
@@ -22,6 +22,7 @@
     </metadata>
 
     <files>
+        <file src="LLamaSharpBackend.props" target="build/netstandard2.0/LLamaSharp.Backend.Cuda11.props" />
         <file src="icon512.png" target="icon512.png" />
     </files>
 </package>
diff --git a/LLama/runtimes/build/LLamaSharp.Backend.Cuda12.Linux.nuspec b/LLama/runtimes/build/LLamaSharp.Backend.Cuda12.Linux.nuspec
@@ -7,19 +7,27 @@
         <authors>llama.cpp Authors</authors>
         <requireLicenseAcceptance>false</requireLicenseAcceptance>
         <license type="expression">MIT</license>
+        <icon>icon512.png</icon>
         <projectUrl>https://github.com/SciSharp/LLamaSharp</projectUrl>
         <description>LLamaSharp.Backend.Cuda12.Linux contains the Linux binaries for LLamaSharp with Cuda12 support.</description>
         <releaseNotes></releaseNotes>
         <copyright>Copyright 2023 The llama.cpp Authors. All rights reserved.</copyright>
         <tags>LLamaSharp LLama LLM GPT AI ChatBot SciSharp</tags>
+
+        <dependencies>
+            <dependency id="LLamaSharp.Backend.Cpu" version="$version$" />
+        </dependencies>
     </metadata>
 
     <files>
         <file src="LLamaSharpBackend.props" target="build/netstandard2.0/LLamaSharp.Backend.Cuda12.props" />
-
-        <file src="runtimes/deps/cu12.2.0/libllava_shared.so" target="runtimes/linux-x64/native/cuda12/libllava_shared.so" />
+
         <file src="runtimes/deps/cu12.2.0/libggml.so" target="runtimes/linux-x64/native/cuda12/libggml.so" />
+        <file src="runtimes/deps/cu12.2.0/libggml-base.so" target="runtimes/linux-x64/native/cuda12/libggml-base.so" />
+        <file src="runtimes/deps/cu12.2.0/libggml-cuda.so" target="runtimes/linux-x64/native/cuda12/libggml-cuda.so" />
+
         <file src="runtimes/deps/cu12.2.0/libllama.so" target="runtimes/linux-x64/native/cuda12/libllama.so" />
+        <file src="runtimes/deps/cu12.2.0/libllava_shared.so" target="runtimes/linux-x64/native/cuda12/libllava_shared.so" />
 
         <file src="icon512.png" target="icon512.png" />
     </files>