From 71f02e08c234766838d24b048c64ad8fa65030cc Mon Sep 17 00:00:00 2001
From: Mihai <mihai.chirculescu@yahoo.com>
Date: Wed, 30 Aug 2023 17:24:39 +0300
Subject: [PATCH] Add grammar example

---
 LLama.Examples/Assets/json.gbnf               | 27 +++++++++
 LLama.Examples/LLama.Examples.csproj          |  3 +
 .../NewVersion/GrammarJsonResponse.cs         | 55 +++++++++++++++++++
 LLama.Examples/NewVersion/TestRunner.cs       |  5 ++
 4 files changed, 90 insertions(+)
 create mode 100644 LLama.Examples/Assets/json.gbnf
 create mode 100644 LLama.Examples/NewVersion/GrammarJsonResponse.cs

diff --git a/LLama.Examples/Assets/json.gbnf b/LLama.Examples/Assets/json.gbnf
new file mode 100644
index 00000000..a01c4efd
--- /dev/null
+++ b/LLama.Examples/Assets/json.gbnf
@@ -0,0 +1,27 @@
+# https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/grammars/json.gbnf
+
+root   ::= object
+value  ::= object | array | string | number | ("true" | "false" | "null") ws
+
+object ::=
+  "{" ws (
+            string ":" ws value
+    ("," ws string ":" ws value)*
+  )? "}" ws
+
+array  ::=
+  "[" ws (
+            value
+    ("," ws value)*
+  )? "]" ws
+
+string ::=
+  "\"" (
+    [^"\\] |
+    "\\" (["\\/bfnrt] | "u" [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F] [0-9a-fA-F]) # escapes
+  )* "\"" ws
+
+number ::= ("-"? ([0-9] | [1-9] [0-9]*)) ("." [0-9]+)? ([eE] [-+]? [0-9]+)? ws
+
+# Optional space: by convention, applied in this grammar after literal chars when allowed
+ws ::= ([ \t\n] ws)?
\ No newline at end of file
diff --git a/LLama.Examples/LLama.Examples.csproj b/LLama.Examples/LLama.Examples.csproj
index ef7ac437..6a1685ed 100644
--- a/LLama.Examples/LLama.Examples.csproj
+++ b/LLama.Examples/LLama.Examples.csproj
@@ -49,6 +49,9 @@
     <None Update="Assets\dan.txt">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
+    <None Update="Assets\json.gbnf">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
     <None Update="Assets\reason-act.txt">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
diff --git a/LLama.Examples/NewVersion/GrammarJsonResponse.cs b/LLama.Examples/NewVersion/GrammarJsonResponse.cs
new file mode 100644
index 00000000..926aa82b
--- /dev/null
+++ b/LLama.Examples/NewVersion/GrammarJsonResponse.cs
@@ -0,0 +1,55 @@
+using LLama.Common;
+using LLama.Grammar;
+using LLama.Native;
+
+namespace LLama.Examples.NewVersion
+{
+    public class GrammarJsonResponse
+    {
+        public static void Run()
+        {
+            var grammarBytes = File.ReadAllText("Assets/json.gbnf").Trim();
+            var parsedGrammar = new GrammarParser();
+
+            Console.Write("Please input your model path: ");
+            var modelPath = Console.ReadLine();
+
+            var parameters = new ModelParams(modelPath)
+            {
+                ContextSize = 1024,
+                Seed = 1337,
+                GpuLayerCount = 5
+            };
+            using var model = LLamaWeights.LoadFromFile(parameters);
+            var ex = new StatelessExecutor(model, parameters);
+            ParseState state = parsedGrammar.Parse(grammarBytes);
+            using var grammar = SafeLLamaGrammarHandle.Create(state.Rules, 0);
+
+            Console.ForegroundColor = ConsoleColor.Yellow;
+            Console.WriteLine("The executor has been enabled. In this example, the LLM will follow your instructions and always respond in a JSON format. For example, you can input \"Tell me the attributes of a good dish\"");
+            Console.ForegroundColor = ConsoleColor.White;
+
+            var inferenceParams = new InferenceParams() 
+            { 
+                Temperature = 0.6f, 
+                AntiPrompts = new List<string> { "Question:", "#", "Question: ", ".\n" }, 
+                MaxTokens = 50,
+                Grammar = grammar
+            };
+
+            while (true)
+            {
+                Console.Write("\nQuestion: ");
+                Console.ForegroundColor = ConsoleColor.Green;
+                var prompt = Console.ReadLine();
+                Console.ForegroundColor = ConsoleColor.White;
+                Console.Write("Answer: ");
+                prompt = $"Question: {prompt?.Trim()} Answer: ";
+                foreach (var text in ex.Infer(prompt, inferenceParams))
+                {
+                    Console.Write(text);
+                }
+            }
+        }
+    }
+}
diff --git a/LLama.Examples/NewVersion/TestRunner.cs b/LLama.Examples/NewVersion/TestRunner.cs
index 6cc3f3da..f5a10ef4 100644
--- a/LLama.Examples/NewVersion/TestRunner.cs
+++ b/LLama.Examples/NewVersion/TestRunner.cs
@@ -17,6 +17,7 @@
             Console.WriteLine("7: Get embeddings from LLama model.");
             Console.WriteLine("8: Quantize the model.");
             Console.WriteLine("9: Automatic conversation.");
+            Console.WriteLine("10: Constrain response to json format using grammar.");
 
             while (true)
             {
@@ -63,6 +64,10 @@
                 {
                     await TalkToYourself.Run();
                 }
+                else if (choice == 10)
+                {
+                    GrammarJsonResponse.Run();
+                }
                 else
                 {
                     Console.WriteLine("Cannot parse your choice. Please select again.");