Case insensitive lexer not working

[Arcnor]

Hi,

First of all, thanks for the very useful library!

I've tried to create a simple lexer using the GenericLexer and having [Lexer(KeyWordIgnoreCase = true)] but it doesn't seem to work. I've quickly checked the code, and it seems the special string comparison function that gets set when this variable is set seems to only be used on AddLexeme when genericToken is of type Identifier (but we're expecting that to be used on KeyWords instead?)

In any case, let me know if you need more info, but the issue seems simple enough.

Thanks again!

[b3b00]

@Arcnor in csly lexer keywords are identifiers as they must respect the identifier format.
Have you a failing test case ?

[b3b00]

I've done a quick test just to check if it's working. here is the code

[Lexer(KeyWordIgnoreCase=true)]
public enum IgnoreCase {
    [Keyword("key")]
    Key,

    [Keyword("word")]
    Word,

    [AlphaId]
    Id

}

public class Program
{

    public static void Main(string[] args)
    {
        var result = new BuildResult<ILexer<IgnoreCase>>();
        var lexerBuild = LexerBuilder.BuildLexer<IgnoreCase>(result);
        if (lexerBuild.IsOk)
        {
            var lexer = lexerBuild.Result;
            var lex = lexer.Tokenize("key KEY word WoRD id");
            if (lex.IsOk)
            {
               foreach(var t in lex.Tokens) {
                    Console.WriteLine(t.ToString());
                }
            }
            else {
                Console.WriteLine(lex.Error);
            }
        }

        else
        {
            foreach (var error in lexerBuild.Errors)
            {
Console.WriteLine(error.Message);
            }
        }
    }
}

And it produces :

Key [key] @line 0, column 0 on channel 0
Key [KEY] @line 0, column 4 on channel 0
Word [word] @line 0, column 8 on channel 0
Word [WoRD] @line 0, column 13 on channel 0
Id [id] @line 0, column 18 on channel 0

[b3b00]

I've managed to make it work together with int and identifiers using both lexer extensions and lexer post processing

#:package sly@3.7.5

using sly.lexer;
using sly.lexer.fsm;
using sly.parser.generator;

public enum PostfixHexa
{
    EOS,

    [Extension] Hexa,

    [Int] Int,

    [AlphaId] Id
}

public class Parser
{
	[Production("main : item+ ")]
	public string Main(List<string> items)
	{
		return string.Join("\n", items);
	}

	[Production("item : [Id | Hexa | Int]" )]
	public string Item(Token<PostfixHexa> token)
	{
		return token.ToString();
	}
	
}


public class App
{

	public static void Main(string[] args)
	{
		Action<PostfixHexa, LexemeAttribute, GenericLexer<PostfixHexa>> extensionBuilder = (token, attribute, lexer) =>
										{
											if (token == PostfixHexa.Hexa)
											{

												bool CheckHexa(ReadOnlyMemory<char> value)
												{
													char[] hexaLetters = new char[] { 'a', 'b', 'c', 'd', 'e', 'f' };
													var ok = false;
													return value.ToArray().All(x => Char.IsDigit(x) || hexaLetters.Contains(x));
												}

												NodeCallback<GenericToken> callback = (FSMMatch<GenericToken> match) =>
														{
															// this store the token id the the FSMMatch object to be later returned by GenericLexer.Tokenize 
															match.Properties[GenericLexer<PostfixHexa>.DerivedToken] = PostfixHexa.Hexa;
															return match;
														};
												var builder = lexer.FSMBuilder;
												builder.GoTo(GenericLexer<PostfixHexa>.start)
																		.MultiRangeTransition([('0', '9'), ('a', 'f'), ('A', 'Z')])
																		.Mark("in_hexa")
																		.MultiRangeTransitionTo("in_hexa", [('0', '9'), ('a', 'f'), ('A', 'Z')])
																		.Transition('h')
																		.Mark("hexa_done")
																		.End(GenericToken.Extension, false)
																		.CallBack(callback);
												builder.GoTo(GenericLexer<PostfixHexa>.in_int)
																		.MultiRangeTransition([('0', '9'), ('a', 'f'), ('A', 'Z')])
																		.Mark("in_hexa_after_int")
																		.MultiRangeTransitionTo("in_hexa_after_int", [('0', '9'), ('a', 'f'), ('A', 'Z')])
																		.Transition('h')
																		.End(GenericToken.Extension, false)
																		.CallBack(callback);
											}
										};


		// this post processor retag identifier tokens ending with 'h' and only containing hex digits as Hexa tokens
		LexerPostProcess<PostfixHexa> postProcess = (tokens) =>
								{
									return tokens.Select(x =>
											{
												if (x.TokenID == PostfixHexa.Id && x.Value.EndsWith("h"))
												{
													x.TokenID = PostfixHexa.Hexa;
												}
												return x;
											}).ToList();
								};


		var instance = new Parser();
		ParserBuilder<PostfixHexa,string> parserBuilder = new ParserBuilder<PostfixHexa,string>();
		var parserBuild = parserBuilder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "main",extensionBuilder: extensionBuilder, lexerPostProcess: postProcess);
		if (parserBuild.IsError)
		{
			Console.WriteLine("Parser build error");
			foreach (var error in parserBuild.Errors)
			{
				Console.WriteLine(error);
			}
			return;
		}

		var parser = parserBuild.Result;


		

		var parsed = parser.Parse("01fh 42 identifier abc abch");
		if (parsed.IsError)
		{
			Console.WriteLine("parse error");
			foreach (var error in parsed.Errors)
			{
				Console.WriteLine(error);
			}
			return;
		}
Console.WriteLine("========================== SUCCESS ==========================");
		Console.WriteLine($"parsed: \n{parsed.Result}");
Console.WriteLine("=============================================================");
	}
}

Consider closing the issue if you're happy with this

[b3b00]

@Arcnor , without any news from you i'm closing this issue. Next time consider commenting