scisharp
/
LLamaSharp
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Using a pin instead of a `fixed` block

This commit is contained in:
Martin Evans 2023-08-07 01:20:34 +01:00
parent 2c933c57a1
commit f1111a9f8b
1 changed files with 10 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
LLama/Utils.cs
View File

@ -39,21 +39,19 @@ namespace LLama
lparams.rope_freq_scale = @params.RopeFrequencyScale;
lparams.mul_mat_q = @params.MulMatQ;
using var pin = @params.TensorSplits.AsMemory().Pin();
unsafe
{
fixed (float* splits = @params.TensorSplits)
{
lparams.tensor_split = (nint)splits;
var model = SafeLlamaModelHandle.LoadFromFile(@params.ModelPath, lparams);
var ctx = SafeLLamaContextHandle.Create(model, lparams);
if (!string.IsNullOrEmpty(@params.LoraAdapter))
model.ApplyLoraFromFile(@params.LoraAdapter, @params.LoraBase, @params.Threads);
return ctx;
}
lparams.tensor_split = (nint)pin.Pointer;
}
var model = SafeLlamaModelHandle.LoadFromFile(@params.ModelPath, lparams);
var ctx = SafeLLamaContextHandle.Create(model, lparams);
if (!string.IsNullOrEmpty(@params.LoraAdapter))
model.ApplyLoraFromFile(@params.LoraAdapter, @params.LoraBase, @params.Threads);
return ctx;
}
public static IEnumerable<llama_token> Tokenize(SafeLLamaContextHandle ctx, string text, bool add_bos, Encoding encoding)