Add grammar example
This commit is contained in:
parent
920efaca44
commit
71f02e08c2
|
@ -0,0 +1,27 @@
|
||||||
|
# https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/grammars/json.gbnf
|
||||||
|
|
||||||
|
root ::= object
|
||||||
|
value ::= object | array | string | number | ("true" | "false" | "null") ws
|
||||||
|
|
||||||
|
object ::=
|
||||||
|
"{" ws (
|
||||||
|
string ":" ws value
|
||||||
|
("," ws string ":" ws value)*
|
||||||
|
)? "}" ws
|
||||||
|
|
||||||
|
array ::=
|
||||||
|
"[" ws (
|
||||||
|
value
|
||||||
|
("," ws value)*
|
||||||
|
)? "]" ws
|
||||||
|
|
||||||
|
string ::=
|
||||||
|
"\"" (
|
||||||
|
[^"\\] |
|
||||||
|
"\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
|
||||||
|
)* "\"" ws
|
||||||
|
|
||||||
|
number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
|
||||||
|
|
||||||
|
# Optional space: by convention, applied in this grammar after literal chars when allowed
|
||||||
|
ws ::= ([ \t\n] ws)?
|
|
@ -49,6 +49,9 @@
|
||||||
<None Update="Assets\dan.txt">
|
<None Update="Assets\dan.txt">
|
||||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
</None>
|
</None>
|
||||||
|
<None Update="Assets\json.gbnf">
|
||||||
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
|
</None>
|
||||||
<None Update="Assets\reason-act.txt">
|
<None Update="Assets\reason-act.txt">
|
||||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||||
</None>
|
</None>
|
||||||
|
|
|
@ -0,0 +1,55 @@
|
||||||
|
using LLama.Common;
|
||||||
|
using LLama.Grammar;
|
||||||
|
using LLama.Native;
|
||||||
|
|
||||||
|
namespace LLama.Examples.NewVersion
|
||||||
|
{
|
||||||
|
public class GrammarJsonResponse
|
||||||
|
{
|
||||||
|
public static void Run()
|
||||||
|
{
|
||||||
|
var grammarBytes = File.ReadAllText("Assets/json.gbnf").Trim();
|
||||||
|
var parsedGrammar = new GrammarParser();
|
||||||
|
|
||||||
|
Console.Write("Please input your model path: ");
|
||||||
|
var modelPath = Console.ReadLine();
|
||||||
|
|
||||||
|
var parameters = new ModelParams(modelPath)
|
||||||
|
{
|
||||||
|
ContextSize = 1024,
|
||||||
|
Seed = 1337,
|
||||||
|
GpuLayerCount = 5
|
||||||
|
};
|
||||||
|
using var model = LLamaWeights.LoadFromFile(parameters);
|
||||||
|
var ex = new StatelessExecutor(model, parameters);
|
||||||
|
ParseState state = parsedGrammar.Parse(grammarBytes);
|
||||||
|
using var grammar = SafeLLamaGrammarHandle.Create(state.Rules, 0);
|
||||||
|
|
||||||
|
Console.ForegroundColor = ConsoleColor.Yellow;
|
||||||
|
Console.WriteLine("The executor has been enabled. In this example, the LLM will follow your instructions and always respond in a JSON format. For example, you can input \"Tell me the attributes of a good dish\"");
|
||||||
|
Console.ForegroundColor = ConsoleColor.White;
|
||||||
|
|
||||||
|
var inferenceParams = new InferenceParams()
|
||||||
|
{
|
||||||
|
Temperature = 0.6f,
|
||||||
|
AntiPrompts = new List<string> { "Question:", "#", "Question: ", ".\n" },
|
||||||
|
MaxTokens = 50,
|
||||||
|
Grammar = grammar
|
||||||
|
};
|
||||||
|
|
||||||
|
while (true)
|
||||||
|
{
|
||||||
|
Console.Write("\nQuestion: ");
|
||||||
|
Console.ForegroundColor = ConsoleColor.Green;
|
||||||
|
var prompt = Console.ReadLine();
|
||||||
|
Console.ForegroundColor = ConsoleColor.White;
|
||||||
|
Console.Write("Answer: ");
|
||||||
|
prompt = $"Question: {prompt?.Trim()} Answer: ";
|
||||||
|
foreach (var text in ex.Infer(prompt, inferenceParams))
|
||||||
|
{
|
||||||
|
Console.Write(text);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -17,6 +17,7 @@
|
||||||
Console.WriteLine("7: Get embeddings from LLama model.");
|
Console.WriteLine("7: Get embeddings from LLama model.");
|
||||||
Console.WriteLine("8: Quantize the model.");
|
Console.WriteLine("8: Quantize the model.");
|
||||||
Console.WriteLine("9: Automatic conversation.");
|
Console.WriteLine("9: Automatic conversation.");
|
||||||
|
Console.WriteLine("10: Constrain response to json format using grammar.");
|
||||||
|
|
||||||
while (true)
|
while (true)
|
||||||
{
|
{
|
||||||
|
@ -63,6 +64,10 @@
|
||||||
{
|
{
|
||||||
await TalkToYourself.Run();
|
await TalkToYourself.Run();
|
||||||
}
|
}
|
||||||
|
else if (choice == 10)
|
||||||
|
{
|
||||||
|
GrammarJsonResponse.Run();
|
||||||
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
Console.WriteLine("Cannot parse your choice. Please select again.");
|
Console.WriteLine("Cannot parse your choice. Please select again.");
|
||||||
|
|
Loading…
Reference in New Issue