From 9f694c584c5e08b99895e665e32f41d3a7b225ef Mon Sep 17 00:00:00 2001 From: Martin Evans Date: Fri, 13 Oct 2023 02:08:12 +0100 Subject: [PATCH] Further improved grammar parser test coverage (up to 92%) --- LLama.Unittest/GrammarParserTest.cs | 169 ++++++++++++++++------------ 1 file changed, 100 insertions(+), 69 deletions(-) diff --git a/LLama.Unittest/GrammarParserTest.cs b/LLama.Unittest/GrammarParserTest.cs index 40046732..8e040ce2 100644 --- a/LLama.Unittest/GrammarParserTest.cs +++ b/LLama.Unittest/GrammarParserTest.cs @@ -12,16 +12,48 @@ namespace LLama.Unittest /// public sealed class GrammarParserTest { + private static void CheckGrammar(string grammar, string rootRule, List> expected, List expectedRules) + { + var state = Grammar.Parse(grammar, rootRule); + Assert.Equal(0ul, state.StartRuleIndex); + + foreach (var symbol in expected) + { + var rule = state.Rules[(int)symbol.Value]; + Assert.Equal(symbol.Key, rule.Name); + } + + uint index = 0; + foreach (var rule in state.Rules) + { + // compare rule to expected rule + for (uint i = 0; i < rule.Elements.Count; i++) + { + var element = rule.Elements[(int)i]; + var expectedElement = expectedRules[(int)index]; + + // Pretty print error message before asserting + if (expectedElement.Type != element.Type || expectedElement.Value != element.Value) + { + Console.Error.WriteLine($"index: {index}"); + Console.Error.WriteLine($"expected_element: {expectedElement.Type}, {expectedElement.Value}"); + Console.Error.WriteLine($"actual_element: {element.Type}, {element.Value}"); + Console.Error.WriteLine("expected_element != actual_element"); + } + Assert.Equal(expectedElement.Type, element.Type); + Assert.Equal(expectedElement.Value, element.Value); + index++; + } + } + Assert.NotEmpty(state.Rules); + } + [Fact] public void ParseComplexGrammar() { - GBNFGrammarParser parsedGrammar = new GBNFGrammarParser(); - string grammarBytes = @"root ::= (expr ""="" term ""\n"")+ - expr ::= term ([-+*/] term)* - term ::= [0-9]+"; - - var state = parsedGrammar.Parse(grammarBytes, "root"); - Assert.Equal(0ul, state.StartRuleIndex); + var grammarBytes = @"root ::= (expr ""="" term ""\n"")+ + expr ::= term ([-\x2b\x2A/] term)* + term ::= [\x30-\x39]+"; var expected = new List> { @@ -35,12 +67,6 @@ namespace LLama.Unittest new KeyValuePair("term_7", 7), }; - foreach (var symbol in expected) - { - var rule = state.Rules[(int)symbol.Value]; - Assert.Equal(symbol.Key, rule.Name); - } - var expectedRules = new List { new LLamaGrammarElement(LLamaGrammarElementType.RULE_REF, 4), @@ -79,35 +105,12 @@ namespace LLama.Unittest new LLamaGrammarElement(LLamaGrammarElementType.END, 0), }; - uint index = 0; - foreach (var rule in state.Rules) - { - // compare rule to expected rule - for (uint i = 0; i < rule.Elements.Count; i++) - { - var element = rule.Elements[(int)i]; - var expectedElement = expectedRules[(int)index]; - - // Pretty print error message before asserting - if (expectedElement.Type != element.Type || expectedElement.Value != element.Value) - { - Console.Error.WriteLine($"index: {index}"); - Console.Error.WriteLine($"expected_element: {expectedElement.Type}, {expectedElement.Value}"); - Console.Error.WriteLine($"actual_element: {element.Type}, {element.Value}"); - Console.Error.WriteLine("expected_element != actual_element"); - } - Assert.Equal(expectedElement.Type, element.Type); - Assert.Equal(expectedElement.Value, element.Value); - index++; - } - } - Assert.NotEmpty(state.Rules); + CheckGrammar(grammarBytes, "root", expected, expectedRules); } [Fact] public void ParseExtraComplexGrammar() { - GBNFGrammarParser parsedGrammar = new GBNFGrammarParser(); string grammarBytes = @" root ::= (expr ""="" ws term ""\n"")+ expr ::= term ([-+*/] term)* @@ -117,9 +120,6 @@ namespace LLama.Unittest ws ::= [ \t\n]* "; - var state = parsedGrammar.Parse(grammarBytes, "root"); - Assert.Equal(0ul, state.StartRuleIndex); - var expected = new List> { new KeyValuePair("expr", 2), @@ -137,12 +137,6 @@ namespace LLama.Unittest new KeyValuePair("ws_12", 12), }; - foreach (var symbol in expected) - { - var rule = state.Rules[(int)symbol.Value]; - Assert.Equal(symbol.Key, rule.Name); - } - var expectedRules = new List { new LLamaGrammarElement(LLamaGrammarElementType.RULE_REF, 5), @@ -214,31 +208,68 @@ namespace LLama.Unittest new LLamaGrammarElement(LLamaGrammarElementType.END, 0) }; - uint index = 0; - foreach (var rule in state.Rules) - { - // compare rule to expected rule - for (uint i = 0; i < rule.Elements.Count; i++) - { - var element = rule.Elements[(int)i]; - var expectedElement = expectedRules[(int)index]; - - // Pretty print error message before asserting - if (expectedElement.Type != element.Type || expectedElement.Value != element.Value) - { - Console.Error.WriteLine($"index: {index}"); - Console.Error.WriteLine($"expected_element: {expectedElement.Type}, {expectedElement.Value}"); - Console.Error.WriteLine($"actual_element: {element.Type}, {element.Value}"); - Console.Error.WriteLine("expected_element != actual_element"); - } - Assert.Equal(expectedElement.Type, element.Type); - Assert.Equal(expectedElement.Value, element.Value); - index++; - } - } - Assert.NotEmpty(state.Rules); + CheckGrammar(grammarBytes, "root", expected, expectedRules); } + + [Fact] + public void InvalidGrammarNoClosingBracket() + { + var parsedGrammar = new GBNFGrammarParser(); + var grammarBytes = @" + root ::= (expr ""="" ws term ""\n""+ ## <--- Mismatched brackets on this line + expr ::= term ([-+*/] term)* + term ::= ident | num | ""("" ws expr "")"" ws + ident ::= [a-z] [a-z0-9_]* ws + num ::= [0-9]+ ws + ws ::= [ \t\n]* + "; + + Assert.Throws(() => + { + parsedGrammar.Parse(grammarBytes, "root"); + }); + } + + [Fact] + public void InvalidGrammarNoName() + { + var parsedGrammar = new GBNFGrammarParser(); + var grammarBytes = @" + root ::= (expr ""="" ws term ""\n"")+ + ::= term ([-+*/] term)* ## <--- Missing a name for this rule! + term ::= ident | num | ""("" ws expr "")"" ws + ident ::= [a-z] [a-z0-9_]* ws + num ::= [0-9]+ ws + ws ::= [ \t\n]* + "; + + Assert.Throws(() => + { + parsedGrammar.Parse(grammarBytes, "root"); + }); + } + + [Fact] + public void InvalidGrammarBadHex() + { + var parsedGrammar = new GBNFGrammarParser(); + var grammarBytes = @" + root ::= (expr ""="" ws term ""\n"")+ + expr ::= term ([-+*/] term)* + term ::= ident | num | ""("" ws expr "")"" ws + ident ::= [a-z] [a-z0-9_]* ws + num ::= [0-\xQQ]+ ws ## <--- `\xQQ` is not valid hex! + ws ::= [ \t\n]* + "; + + Assert.Throws(() => + { + parsedGrammar.Parse(grammarBytes, "root"); + }); + } + + [Fact] public void InvalidRuleNoElements() {