From a9d1f6cb47f9b1fa21fb51692ddd9957d2d3dbde Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Mon, 13 Nov 2023 02:05:07 +0000 Subject: [PATCH] - Renamed `NativeLibraryConfig.Default` to `NativeLibraryConfig.Instance`. It's not default any more as soon as you call `WithX`! - using `Lazy` to initialize it automatically. - Added in `AVX512` support for all dotnet versions (but not autodetected). - Added in AVX version auto detection. --- LLama.Examples/Program.cs | 2 +- LLama/Native/NativeApi.Load.cs | 24 ++--- LLama/Native/NativeLibraryConfig.cs | 156 +++++++++++++--------------- 3 files changed, 81 insertions(+), 101 deletions(-) diff --git a/LLama.Examples/Program.cs b/LLama.Examples/Program.cs index 0f180053..ee90ac84 100644 --- a/LLama.Examples/Program.cs +++ b/LLama.Examples/Program.cs @@ -7,7 +7,7 @@ Console.WriteLine(" __ __ ____ _ Console.WriteLine("======================================================================================================"); -NativeLibraryConfig.Default.WithCuda().WithLogs(); +NativeLibraryConfig.Instance.WithCuda().WithLogs(); NativeApi.llama_empty_call(); Console.WriteLine(); diff --git a/LLama/Native/NativeApi.Load.cs b/LLama/Native/NativeApi.Load.cs index 87f75324..148f1735 100644 --- a/LLama/Native/NativeApi.Load.cs +++ b/LLama/Native/NativeApi.Load.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; -using System.Text; using System.Text.Json; namespace LLama.Native @@ -227,24 +226,15 @@ namespace LLama.Native } else if (platform != OSPlatform.OSX) // in macos there's absolutely no avx { -#if NET8_0_OR_GREATER - if (configuration.AvxLevel == NativeLibraryConfig.AvxLevel.Avx512) - { - result.Add(GetAvxLibraryPath(NativeLibraryConfig.AvxLevel.Avx512, prefix, suffix))); - result.Add(GetAvxLibraryPath(NativeLibraryConfig.AvxLevel.Avx2, prefix, suffix))); - result.Add(GetAvxLibraryPath(NativeLibraryConfig.AvxLevel.Avx, prefix, suffix))); - } - else -#endif - if (configuration.AvxLevel == NativeLibraryConfig.AvxLevel.Avx2) - { + if (configuration.AvxLevel >= NativeLibraryConfig.AvxLevel.Avx512) + result.Add(GetAvxLibraryPath(NativeLibraryConfig.AvxLevel.Avx512, prefix, suffix)); + + if (configuration.AvxLevel >= NativeLibraryConfig.AvxLevel.Avx2) result.Add(GetAvxLibraryPath(NativeLibraryConfig.AvxLevel.Avx2, prefix, suffix)); + + if (configuration.AvxLevel >= NativeLibraryConfig.AvxLevel.Avx) result.Add(GetAvxLibraryPath(NativeLibraryConfig.AvxLevel.Avx, prefix, suffix)); - } - else if (configuration.AvxLevel == NativeLibraryConfig.AvxLevel.Avx) - { - result.Add(GetAvxLibraryPath(NativeLibraryConfig.AvxLevel.Avx, prefix, suffix)); - } + result.Add(GetAvxLibraryPath(NativeLibraryConfig.AvxLevel.None, prefix, suffix)); } diff --git a/LLama/Native/NativeLibraryConfig.cs b/LLama/Native/NativeLibraryConfig.cs index 7af445e6..79d8f659 100644 --- a/LLama/Native/NativeLibraryConfig.cs +++ b/LLama/Native/NativeLibraryConfig.cs @@ -4,46 +4,34 @@ namespace LLama.Native { #if NET6_0_OR_GREATER /// - /// A class about configurations when loading native libraries. - /// Note that it could be configured only once before any call to llama model apis. + /// Allows configuration of the native llama.cpp libraries to load and use. + /// All configuration must be done before using **any** other LLamaSharp methods! /// - public class NativeLibraryConfig + public sealed class NativeLibraryConfig { - private static NativeLibraryConfig? instance; - private static readonly object lockObject = new object(); - public static NativeLibraryConfig Default - { - get - { - return GetInstance(); - } - } + private static readonly Lazy _instance = new(() => new NativeLibraryConfig()); + + /// + /// Get the config instance + /// + public static NativeLibraryConfig Instance => _instance.Value; /// /// Whether there's already a config for native library. /// public static bool LibraryHasLoaded { get; internal set; } = false; - private string _libraryPath; - private bool _useCuda; + private string _libraryPath = string.Empty; + private bool _useCuda = true; private AvxLevel _avxLevel; - private bool _allowFallback; - private bool _skipCheck; - private bool _logging; + private bool _allowFallback = true; + private bool _skipCheck = false; + private bool _logging = false; - internal static NativeLibraryConfig GetInstance() + private static void ThrowIfLoaded() { - if (instance is null) - { - lock (lockObject) - { - if (instance is null) - { - instance = new NativeLibraryConfig(); - } - } - } - return instance; + if (LibraryHasLoaded) + throw new InvalidOperationException("NativeLibraryConfig must be configured before using **any** other LLamaSharp methods!"); } /// @@ -51,13 +39,11 @@ namespace LLama.Native /// When this method is called, all the other configurations will be ignored. /// /// - /// + /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfig WithLibrary(string libraryPath) { - if (LibraryHasLoaded) - { - throw new InvalidOperationException("NativeLibraryConfig could be configured only once before any call to llama model apis."); - } + ThrowIfLoaded(); + _libraryPath = libraryPath; return this; } @@ -67,13 +53,11 @@ namespace LLama.Native /// /// /// - /// + /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfig WithCuda(bool enable = true) { - if (LibraryHasLoaded) - { - throw new InvalidOperationException("NativeLibraryConfig could be configured only once before any call to llama model apis."); - } + ThrowIfLoaded(); + _useCuda = enable; return this; } @@ -83,29 +67,25 @@ namespace LLama.Native /// /// /// - /// + /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfig WithAvx(AvxLevel level) { - if (LibraryHasLoaded) - { - throw new InvalidOperationException("NativeLibraryConfig could be configured only once before any call to llama model apis."); - } + ThrowIfLoaded(); + _avxLevel = level; return this; } /// - /// Configure whether to allow fallback when there's not match for preffered settings. + /// Configure whether to allow fallback when there's no match for preferred settings. /// /// /// - /// + /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfig WithAutoFallback(bool enable = true) { - if (LibraryHasLoaded) - { - throw new InvalidOperationException("NativeLibraryConfig could be configured only once before any call to llama model apis."); - } + ThrowIfLoaded(); + _allowFallback = enable; return this; } @@ -117,13 +97,11 @@ namespace LLama.Native /// /// /// - /// + /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfig SkipCheck(bool enable = true) { - if (LibraryHasLoaded) - { - throw new InvalidOperationException("NativeLibraryConfig could be configured only once before any call to llama model apis."); - } + ThrowIfLoaded(); + _skipCheck = enable; return this; } @@ -133,24 +111,22 @@ namespace LLama.Native /// /// /// - /// + /// Thrown if `LibraryHasLoaded` is true. public NativeLibraryConfig WithLogs(bool enable = true) { - if (LibraryHasLoaded) - { - throw new InvalidOperationException("NativeLibraryConfig could be configured only once before any call to llama model apis."); - } + ThrowIfLoaded(); + _logging = enable; return this; } internal static Description CheckAndGatherDescription() { - if (Default._allowFallback && Default._skipCheck) + if (Instance._allowFallback && Instance._skipCheck) { throw new ArgumentException("Cannot skip the check when fallback is allowed."); } - return new Description(Default._libraryPath, Default._useCuda, Default._avxLevel, Default._allowFallback, Default._skipCheck, Default._logging); + return new Description(Instance._libraryPath, Instance._useCuda, Instance._avxLevel, Instance._allowFallback, Instance._skipCheck, Instance._logging); } internal static string AvxLevelToString(AvxLevel level) @@ -163,19 +139,24 @@ namespace LLama.Native #if NET8_0_OR_GREATER AvxLevel.Avx512 => "avx512" #endif - _ => throw new ArgumentException($"Cannot recognize Avx level {level}") + _ => throw new ArgumentException($"Unknown AvxLevel '{level}'") }; } - + /// + /// Private constructor prevents new instances of this class being created + /// private NativeLibraryConfig() { - _libraryPath = string.Empty; - _useCuda = true; - _avxLevel = AvxLevel.Avx2; - _allowFallback = true; - _skipCheck = false; - _logging = false; + // Automatically detect the highest supported AVX level + if (System.Runtime.Intrinsics.X86.Avx.IsSupported) + _avxLevel = AvxLevel.Avx; + if (System.Runtime.Intrinsics.X86.Avx2.IsSupported) + _avxLevel = AvxLevel.Avx2; +#if NET8_0_OR_GREATER + if (System.Runtime.Intrinsics.X86.Avx512.IsSupported) + _avxLevel = AvxLevel.Avx512; +#endif } /// @@ -183,19 +164,28 @@ namespace LLama.Native /// public enum AvxLevel { - /// - None = 0, - /// - Avx = 1, - /// - Avx2 = 2, -#if NET8_0_OR_GREATER - /// - Avx512 = 3, -#endif + /// + /// No AVX + /// + None, + + /// + /// Advanced Vector Extensions (supported by most processors after 2011) + /// + Avx, + + /// + /// AVX2 (supported by most processors after 2013) + /// + Avx2, + + /// + /// AVX512 (supported by some processors after 2016, not widely supported) + /// + Avx512, } - internal record Description(string Path = "", bool UseCuda = true, AvxLevel AvxLevel = AvxLevel.Avx2, - bool AllowFallback = true, bool SkipCheck = false, bool Logging = false); + + internal record Description(string Path, bool UseCuda, AvxLevel AvxLevel, bool AllowFallback, bool SkipCheck, bool Logging); } #endif -} + }