Dictionary doesn't seem to work properly with %whitespace

[mingodad]

While trying one grammar I found what seems to be a bug with the combination of a dictionary for keywords and %whitespace see bellow examples and outputs, when not using %whitespace it works as expected.

Maybe I'm missing something here ?

START <-
	(ID / CREATE / DELETE)*

CREATE	<-	"create" 
DELETE	<-	"delete"

KeyWord <- (
	"create"
	| "delete"
	) !IdCharTail

IdCharTail <- [A-Za-z0-9_]

ID <- !KeyWord <[A-Za-z_] IdCharTail*>

%whitespace <- [ \t\r\n]*

Input:

I will create a deleter to delete words

Output:

+ START
  - ID (I)
  - ID (will)
  - ID (create)
  - ID (a)
  - ID (deleter)
  - ID (to)
  - ID (delete)
  - ID (words)

Expected output:

+ START
  - ID (I)
  - ID (will)
  - CREATE (create)
  - ID (a)
  - ID (deleter)
  - ID (to)
  - DELETE (delete)
  - ID (words)

If we don't use %whitespcae:

START <-
	(ID / CREATE / DELETE / _)*

CREATE	<-	"create" 
DELETE	<-	"delete"

KeyWord <- (
	"create"
	| "delete"
	) !IdCharTail

IdCharTail <- [A-Za-z0-9_]

ID <- !KeyWord <[A-Za-z_] IdCharTail*>
~_ <- [ \t\r\n]+

Output:

+ START
  - ID (I)
  - ID (will)
  - CREATE (create)
  - ID (a)
  - ID (deleter)
  - ID (to)
  - DELETE (delete)
  - ID (words)

[mingodad]

Also the CPP_PEGLIB doesnt seem to manage line comments properly when they are not followed by a \n

START <-
    "one"

#comment without new line at the end

Input:

one

Output:

4:36 syntax error

[mingodad]

Also would be nice if we could use a token name with dictionaries to prevent typo bugs:

START <-
	(ID / CREATE / DELETE / _)*

CREATE	<-	"create" 
DELETE	<-	"delete"

KeyWord <- (
	CREATE
	| DELETE
	) !IdCharTail

IdCharTail <- [A-Za-z0-9_]

ID <- !KeyWord <[A-Za-z_] IdCharTail*>
~_ <- [ \t\r\n]+

Output:

9:2 syntax error

[mingodad]

Also it's a good idea to maintain https://github.com/yhirose/cpp-peglib/blob/master/grammar/cpp-peglib.peg updated as the grammar changes (see bellow extracted from today).

#From: https://github.com/yhirose/cpp-peglib/blob/527422aa38ce78c4214b21843e97584bc2db0150/peglib.h#L3365
# Setup PEG syntax parser
Grammar <- Spacing Definition+ EndOfFile

Definition <-
	Ignore IdentCont Parameters LEFTARROW Expression Instruction?
	/ Ignore Identifier LEFTARROW Expression Instruction?

Expression <- Sequence (SLASH Sequence)*

Sequence <- (CUT / Prefix)*

Prefix <- (AND / NOT)? SuffixWithLabel

SuffixWithLabel <- Suffix (LABEL Identifier)?

Suffix <- Primary Loop?

Loop <- QUESTION / STAR / PLUS / Repetition

Primary <-
	Ignore IdentCont Arguments !LEFTARROW
	/ Ignore Identifier !(Parameters? LEFTARROW)
	/ OPEN Expression CLOSE
	/ BeginTok Expression EndTok
	/ CapScope
	/ BeginCap Expression EndCap
	/ BackRef
	/ DictionaryI
	/ LiteralI
	/ Dictionary
	/ Literal
	/ NegatedClassI
	/ NegatedClass
	/ ClassI
	/ Class
	/ DOT

Identifier <- IdentCont Spacing

IdentCont <- <IdentStart IdentRest*>

IdentStart <- !"↑" !"⇑" ([a-zA-Z_%] / [\u0080-\uFFFF])

IdentRest <- IdentStart / [0-9]

Dictionary <- LiteralD (PIPE LiteralD)+

DictionaryI <- LiteralID (PIPE LiteralID)+

lit_ope <-
	['] <(!['] Char)*> ['] Spacing
	/ ["] <(!["] Char)*> ["] Spacing

Literal <- lit_ope
LiteralD <- lit_ope

lit_case_ignore_ope <-
	['] <(!['] Char)*> "'i" Spacing
	/ ["] <(!["] Char)*> '"i' Spacing

LiteralI <- lit_case_ignore_ope
LiteralID <- lit_case_ignore_ope

# NOTE: The original Brian Ford's paper uses 'zom' instead of 'oom'.
Class <- '[' !'^' <(!']' Range)+> ']' Spacing
ClassI <- '[' !'^' <(!']' Range)+> "]i" Spacing

NegatedClass <- "[^" <(!']' Range)+> ']' Spacing
NegatedClassI <- "[^" <(!']' Range)+> "]i" Spacing

# NOTE: This is different from The original Brian Ford's paper and this
# modification allows us to specify `[+-]` as a valid char class.
Range <- Char '-' !']' Char / Char

Char <-
	'\\' [fnrtv'\"[\]\\^]
	/ '\\' [0-3] [0-7] [0-7]
	/ '\\' [0-7] [0-7]?
	/ "\\x" [0-9a-fA-F] [0-9a-fA-F]?
	/ "\\u" (((('0' [0-9a-fA-F]) / "10") [0-9a-fA-F]{4,4}) / [0-9a-fA-F]{4,5})
	/ !'\\' .

Repetition <- BeginBracket RepetitionRange EndBracket

RepetitionRange <-
	Number COMMA Number
	/ Number COMMA
	/ Number
	/ COMMA Number

Number <- [0-9]+ Spacing

CapScope <- BeginCapScope Expression EndCapScope

LEFTARROW <- ("<-" / "←") Spacing
~SLASH <- '/' Spacing
~PIPE <- '|' Spacing
AND <- '&' Spacing
NOT <- '!' Spacing
QUESTION <- '?' Spacing
STAR <- '*' Spacing
PLUS <- '+' Spacing
~OPEN <- '(' Spacing
~CLOSE <- ')' Spacing
DOT <- '.' Spacing

CUT <- "↑" Spacing
~LABEL <- ('^' / "⇑") Spacing

~Spacing <- (Space / Comment)*
Comment <- '#' (!EndOfLine . )* EndOfLine
Space <- ' ' / '\t' / EndOfLine
EndOfLine <- "\r\n" / '\n' / '\r'
EndOfFile <- !.

~BeginTok <- '<' Spacing
~EndTok <- '>' Spacing

~BeginCapScope <- '$' '(' Spacing
~EndCapScope <- ')' Spacing

BeginCap <- '$' <IdentCont> '<' Spacing
~EndCap <- '>' Spacing

BackRef <- '$' <IdentCont> Spacing

IGNORE <- '~'

Ignore <- IGNORE?

Parameters <- OPEN Identifier (COMMA Identifier)* CLOSE

Arguments <- OPEN Expression (COMMA Expression)* CLOSE

~COMMA <- ',' Spacing

# Instruction grammars
Instruction <-
	BeginBracket
		(InstructionItem (InstructionItemSeparator InstructionItem)*)?
	EndBracket

InstructionItem <- PrecedenceClimbing / ErrorMessage / NoAstOpt
~InstructionItemSeparator <- ';' Spacing

~SpacesZom <- Space*
~SpacesOom <- Space+
~BeginBracket <- '{' Spacing
~EndBracket <- '}' Spacing

# PrecedenceClimbing instruction
PrecedenceClimbing <-
	"precedence" SpacesOom PrecedenceInfo
		(SpacesOom PrecedenceInfo)* SpacesZom

PrecedenceInfo <- PrecedenceAssoc (~SpacesOom PrecedenceOpe)+

PrecedenceOpe <-
	['] <(!(Space / [']) Char)*> [']
	/ ["] <(!(Space / ["]) Char)*> ["]
	/ <(!(PrecedenceAssoc / Space / '}') . )+>

PrecedenceAssoc <- [LR]

# Error message instruction
ErrorMessage <- "error_message" SpacesOom LiteralD SpacesZom

# No Ast node optimization instruction
NoAstOpt <- "no_ast_opt" SpacesZom

[yhirose]

@mingodad Thank you for your report. However, please report matters unrelated to this Issue in a separate Issue.