From 168f697db6d05e470c3ea1556a42df84ef5550e6 Mon Sep 17 00:00:00 2001 From: SignalRT Date: Sat, 13 Apr 2024 16:34:32 +0200 Subject: [PATCH] Clean up and align documentation with the changes in the interface --- .../Examples/LlavaInteractiveModeExecute.cs | 22 +++++-------- LLama/Abstractions/ILLamaExecutor.cs | 2 +- LLama/LLamaExecutorBase.cs | 2 +- docs/Examples/LLavaInteractiveModeExecute.md | 32 ++++++++++--------- docs/Tutorials/Executors.md | 4 +-- 5 files changed, 29 insertions(+), 33 deletions(-) diff --git a/LLama.Examples/Examples/LlavaInteractiveModeExecute.cs b/LLama.Examples/Examples/LlavaInteractiveModeExecute.cs index 0c291ec7..112fe23f 100644 --- a/LLama.Examples/Examples/LlavaInteractiveModeExecute.cs +++ b/LLama.Examples/Examples/LlavaInteractiveModeExecute.cs @@ -1,8 +1,6 @@ using System.Text.RegularExpressions; -using LLama.Batched; using LLama.Common; using Spectre.Console; -using LLama.Abstractions; using LLama.Native; namespace LLama.Examples.Examples @@ -20,9 +18,8 @@ namespace LLama.Examples.Examples var prompt = $"{{{modelImage}}}\nUSER:\nProvide a full description of the image.\nASSISTANT:\n"; - var parameters = new ModelParams(modelPath) - { - }; + var parameters = new ModelParams(modelPath); + using var model = LLamaWeights.LoadFromFile(parameters); using var context = model.CreateContext(parameters); @@ -45,16 +42,16 @@ namespace LLama.Examples.Examples var imageMatches = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value); var imageCount = imageMatches.Count(); var hasImages = imageCount > 0; - byte[][] imageBytes = null; if (hasImages) { var imagePathsWithCurlyBraces = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value); - var imagePaths = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Groups[1].Value); + var imagePaths = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Groups[1].Value).ToList(); + List imageBytes; try { - imageBytes = imagePaths.Select(File.ReadAllBytes).ToArray(); + imageBytes = imagePaths.Select(File.ReadAllBytes).ToList(); } catch (IOException exception) { @@ -77,10 +74,7 @@ namespace LLama.Examples.Examples foreach (var path in imagePathsWithCurlyBraces) { // First image replace to tag "); - else - prompt = prompt.Replace(path, ""); + prompt = prompt.Replace(path, index++ == 0 ? "" : ""); } @@ -105,7 +99,7 @@ namespace LLama.Examples.Examples // foreach (var image in imagePaths) { - ex.Images.Add(File.ReadAllBytes(image)); + ex.Images.Add(await File.ReadAllBytesAsync(image)); } } @@ -121,7 +115,7 @@ namespace LLama.Examples.Examples // let the user finish with exit // - if (prompt.Equals("/exit", StringComparison.OrdinalIgnoreCase)) + if (prompt != null && prompt.Equals("/exit", StringComparison.OrdinalIgnoreCase)) break; } diff --git a/LLama/Abstractions/ILLamaExecutor.cs b/LLama/Abstractions/ILLamaExecutor.cs index d6c8d2ce..574a27d8 100644 --- a/LLama/Abstractions/ILLamaExecutor.cs +++ b/LLama/Abstractions/ILLamaExecutor.cs @@ -25,7 +25,7 @@ namespace LLama.Abstractions public LLavaWeights? ClipModel { get; } /// - /// List of images: Image filen path, uri or image byte array. See ImageData. + /// List of images: List of images in byte array format. /// public List Images { get; } diff --git a/LLama/LLamaExecutorBase.cs b/LLama/LLamaExecutorBase.cs index 65c0dcb4..c721726e 100644 --- a/LLama/LLamaExecutorBase.cs +++ b/LLama/LLamaExecutorBase.cs @@ -79,7 +79,7 @@ namespace LLama public LLavaWeights? ClipModel { get; } /// - public List Images { get; set; } + public List Images { get; } /// /// Current "mu" value for mirostat sampling diff --git a/docs/Examples/LLavaInteractiveModeExecute.md b/docs/Examples/LLavaInteractiveModeExecute.md index 9c6faa4f..826ac447 100644 --- a/docs/Examples/LLavaInteractiveModeExecute.md +++ b/docs/Examples/LLavaInteractiveModeExecute.md @@ -2,9 +2,9 @@ ```cs using System.Text.RegularExpressions; -using LLama.Batched; using LLama.Common; using Spectre.Console; +using LLama.Native; namespace LLama.Examples.Examples { @@ -21,11 +21,8 @@ namespace LLama.Examples.Examples var prompt = $"{{{modelImage}}}\nUSER:\nProvide a full description of the image.\nASSISTANT:\n"; - var parameters = new ModelParams(modelPath) - { - ContextSize = 4096, - Seed = 1337, - }; + var parameters = new ModelParams(modelPath); + using var model = LLamaWeights.LoadFromFile(parameters); using var context = model.CreateContext(parameters); @@ -48,16 +45,16 @@ namespace LLama.Examples.Examples var imageMatches = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value); var imageCount = imageMatches.Count(); var hasImages = imageCount > 0; - byte[][] imageBytes = null; if (hasImages) { var imagePathsWithCurlyBraces = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Value); - var imagePaths = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Groups[1].Value); + var imagePaths = Regex.Matches(prompt, "{([^}]*)}").Select(m => m.Groups[1].Value).ToList(); + List imageBytes; try { - imageBytes = imagePaths.Select(File.ReadAllBytes).ToArray(); + imageBytes = imagePaths.Select(File.ReadAllBytes).ToList(); } catch (IOException exception) { @@ -70,15 +67,17 @@ namespace LLama.Examples.Examples break; } + // Each prompt with images we clear cache + // When the prompt contains images we clear KV_CACHE to restart conversation + // See: + // https://github.com/ggerganov/llama.cpp/discussions/3620 + ex.Context.NativeHandle.KvCacheRemove( LLamaSeqId.Zero, -1, -1 ); int index = 0; foreach (var path in imagePathsWithCurlyBraces) { // First image replace to tag "); - else - prompt = prompt.Replace(path, ""); + prompt = prompt.Replace(path, index++ == 0 ? "" : ""); } @@ -101,7 +100,10 @@ namespace LLama.Examples.Examples // Initilize Images in executor // - ex.ImagePaths = imagePaths.ToList(); + foreach (var image in imagePaths) + { + ex.Images.Add(await File.ReadAllBytesAsync(image)); + } } Console.ForegroundColor = Color.White; @@ -116,7 +118,7 @@ namespace LLama.Examples.Examples // let the user finish with exit // - if (prompt.Equals("/exit", StringComparison.OrdinalIgnoreCase)) + if (prompt != null && prompt.Equals("/exit", StringComparison.OrdinalIgnoreCase)) break; } diff --git a/docs/Tutorials/Executors.md b/docs/Tutorials/Executors.md index d014da88..8e7ce23a 100644 --- a/docs/Tutorials/Executors.md +++ b/docs/Tutorials/Executors.md @@ -28,9 +28,9 @@ public interface ILLamaExecutor public LLavaWeights? ClipModel { get; } /// - /// List of images: Image filename and path (jpeg images). + /// List of images: List of images in byte array format. /// - public List ImagePaths { get; set; } + public List Images { get; } ///