Skip to content

Commit 74363cb

Browse files
sebastienrossebastienros
authored
Fix whitespace skipping (sebastienros#43)
1 parent 5497d56 commit 74363cb

15 files changed

+260
-274
lines changed

‎docs/parsers.md

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,32 @@ Result:
482482
0 // failure
483483
```
484484

485+
### SkipWhiteSpace
486+
487+
Matches a parser after any blank spaces. This parser respects the `Scanner` options related to multi-line grammars.
488+
489+
490+
```c#
491+
Parser<T> SkipWhiteSpace<T>(Parser<T> parser)
492+
```
493+
494+
Usage:
495+
496+
```c#
497+
var parser = SkipWhiteSpace(Literals.Text("abc"));
498+
parser.Parse("abc");
499+
parser.Parse(" abc");
500+
```
501+
502+
Result:
503+
504+
```
505+
"abc"
506+
"abc"
507+
```
508+
509+
> Note: This parser is used by all Terms (e.g., Terms.Text) to skip blank spaces before a Literal.
510+
485511
### Deferred
486512

487513
Creates a parser that can be references before it is actually defined. This is used when there is a cyclic dependency between parsers.

‎src/Parlot/Cursor.cs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,17 @@ internal bool AdvanceOnce()
9898
/// <summary>
9999
/// Moves the cursor to the specific position
100100
/// </summary>
101+
[MethodImpl(MethodImplOptions.AggressiveInlining)]
101102
public void ResetPosition(in TextPosition position)
102103
{
103-
if (position.Offset == _offset)
104+
if (position.Offset != _offset)
104105
{
105-
return;
106+
ResetPositionNotInlined(position);
106107
}
108+
}
107109

110+
private void ResetPositionNotInlined(in TextPosition position)
111+
{
108112
_offset = position.Offset;
109113
_line = position.Line;
110114
_column = position.Column;

‎src/Parlot/Fluent/Between.cs

Lines changed: 39 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -10,86 +10,39 @@ public sealed class Between<A, T, B> : Parser<T>, ICompilable
1010
private readonly Parser<A> _before;
1111
private readonly Parser<B> _after;
1212

13-
private readonly bool _beforeIsChar;
14-
private readonly char _beforeChar;
15-
private readonly bool _beforeSkipWhiteSpace;
16-
17-
private readonly bool _afterIsChar;
18-
private readonly char _afterChar;
19-
private readonly bool _afterSkipWhiteSpace;
20-
2113
public Between(Parser<A> before, Parser<T> parser, Parser<B> after)
2214
{
2315
_before = before ?? throw new ArgumentNullException(nameof(before));
2416
_parser = parser ?? throw new ArgumentNullException(nameof(parser));
2517
_after = after ?? throw new ArgumentNullException(nameof(after));
26-
27-
if (before is CharLiteral literal1)
28-
{
29-
_beforeIsChar = true;
30-
_beforeChar = literal1.Char;
31-
_beforeSkipWhiteSpace = literal1.SkipWhiteSpace;
32-
}
33-
34-
if (after is CharLiteral literal2)
35-
{
36-
_afterIsChar = true;
37-
_afterChar = literal2.Char;
38-
_afterSkipWhiteSpace = literal2.SkipWhiteSpace;
39-
}
4018
}
4119

4220
public override bool Parse(ParseContext context, ref ParseResult<T> result)
4321
{
4422
context.EnterParser(this);
4523

46-
if (_beforeIsChar)
47-
{
48-
if (_beforeSkipWhiteSpace)
49-
{
50-
context.SkipWhiteSpace();
51-
}
52-
53-
if (!context.Scanner.ReadChar(_beforeChar))
54-
{
55-
return false;
56-
}
57-
}
58-
else
59-
{
60-
var parsedA = new ParseResult<A>();
24+
var start = context.Scanner.Cursor.Position;
6125

62-
if (!_before.Parse(context, ref parsedA))
63-
{
64-
return false;
65-
}
26+
var parsedA = new ParseResult<A>();
27+
28+
if (!_before.Parse(context, ref parsedA))
29+
{
30+
// Don't reset position since _before should do it
31+
return false;
6632
}
6733

6834
if (!_parser.Parse(context, ref result))
6935
{
36+
context.Scanner.Cursor.ResetPosition(start);
7037
return false;
7138
}
7239

73-
if (_afterIsChar)
74-
{
75-
if (_afterSkipWhiteSpace)
76-
{
77-
context.SkipWhiteSpace();
78-
}
79-
80-
if (!context.Scanner.ReadChar(_afterChar))
81-
{
82-
return false;
83-
}
84-
}
85-
else
86-
{
87-
var parsedB = new ParseResult<B>();
40+
var parsedB = new ParseResult<B>();
8841

89-
if (!_after.Parse(context, ref parsedB))
90-
{
91-
return false;
92-
}
42+
if (!_after.Parse(context, ref parsedB))
43+
{
44+
context.Scanner.Cursor.ResetPosition(start);
45+
return false;
9346
}
9447

9548
return true;
@@ -102,28 +55,37 @@ public CompilationResult Compile(CompilationContext context)
10255
var success = context.DeclareSuccessVariable(result, false);
10356
var value = context.DeclareValueVariable(result, Expression.Default(typeof(T)));
10457

58+
// start = context.Scanner.Cursor.Position;
59+
//
10560
// before instructions
106-
//
61+
//
10762
// if (before.Success)
10863
// {
109-
// parser instructions
110-
//
111-
// if (parser.Success)
112-
// {
113-
// after instructions
114-
//
115-
// if (after.Success)
116-
// {
117-
// success = true;
118-
// value = parser.Value;
119-
// }
120-
// }
64+
// parser instructions
65+
//
66+
// if (parser.Success)
67+
// {
68+
// after instructions
69+
//
70+
// if (after.Success)
71+
// {
72+
// success = true;
73+
// value = parser.Value;
74+
// }
75+
// }
76+
//
77+
// if (!success)
78+
// {
79+
// resetPosition(start);
80+
// }
12181
// }
12282

12383
var beforeCompileResult = _before.Build(context);
12484
var parserCompileResult = _parser.Build(context);
12585
var afterCompileResult = _after.Build(context);
12686

87+
var start = context.DeclarePositionVariable(result);
88+
12789
var block = Expression.Block(
12890
beforeCompileResult.Variables,
12991
Expression.Block(beforeCompileResult.Body),
@@ -147,6 +109,10 @@ public CompilationResult Compile(CompilationContext context)
147109
)
148110
)
149111
)
112+
),
113+
Expression.IfThen(
114+
Expression.Not(success),
115+
context.ResetPosition(start)
150116
)
151117
)
152118
)

‎src/Parlot/Fluent/CharLiteral.cs

Lines changed: 1 addition & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,21 @@
11
using Parlot.Compilation;
2-
using System;
32
using System.Linq.Expressions;
43

54
namespace Parlot.Fluent
65
{
76
public sealed class CharLiteral : Parser<char>, ICompilable
87
{
9-
public CharLiteral(char c, bool skipWhiteSpace = true)
8+
public CharLiteral(char c)
109
{
1110
Char = c;
12-
SkipWhiteSpace = skipWhiteSpace;
1311
}
1412

1513
public char Char { get; }
1614

17-
public bool SkipWhiteSpace { get; }
18-
1915
public override bool Parse(ParseContext context, ref ParseResult<char> result)
2016
{
2117
context.EnterParser(this);
2218

23-
if (SkipWhiteSpace)
24-
{
25-
context.SkipWhiteSpace();
26-
}
27-
2819
var start = context.Scanner.Cursor.Offset;
2920

3021
if (context.Scanner.ReadChar(Char))
@@ -43,16 +34,6 @@ public CompilationResult Compile(CompilationContext context)
4334
var success = context.DeclareSuccessVariable(result, false);
4435
var value = context.DeclareValueVariable(result, Expression.Default(typeof(char)));
4536

46-
//if (_skipWhiteSpace)
47-
//{
48-
// context.SkipWhiteSpace();
49-
//}
50-
51-
if (SkipWhiteSpace)
52-
{
53-
result.Body.Add(context.ParserSkipWhiteSpace());
54-
}
55-
5637
// if (context.Scanner.ReadChar(Char))
5738
// {
5839
// success = true;

‎src/Parlot/Fluent/DecimalLiteral.cs

Lines changed: 5 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -8,26 +8,18 @@ namespace Parlot.Fluent
88
public sealed class DecimalLiteral : Parser<decimal>, ICompilable
99
{
1010
private readonly NumberOptions _numberOptions;
11-
private readonly bool _skipWhiteSpace;
1211

13-
public DecimalLiteral(NumberOptions numberOptions = NumberOptions.Default, bool skipWhiteSpace = true)
12+
public DecimalLiteral(NumberOptions numberOptions = NumberOptions.Default)
1413
{
1514
_numberOptions = numberOptions;
16-
_skipWhiteSpace = skipWhiteSpace;
1715
}
1816

1917
public override bool Parse(ParseContext context, ref ParseResult<decimal> result)
2018
{
2119
context.EnterParser(this);
2220

2321
var reset = context.Scanner.Cursor.Position;
24-
25-
if (_skipWhiteSpace)
26-
{
27-
context.SkipWhiteSpace();
28-
}
29-
30-
var start = context.Scanner.Cursor.Offset;
22+
var start = reset.Offset;
3123

3224
if ((_numberOptions & NumberOptions.AllowSign) == NumberOptions.AllowSign)
3325
{
@@ -42,7 +34,7 @@ public override bool Parse(ParseContext context, ref ParseResult<decimal> result
4234
{
4335
var end = context.Scanner.Cursor.Offset;
4436
#if NETSTANDARD2_0
45-
var sourceToParse = context.Scanner.Buffer.Substring(start, end -start);
37+
var sourceToParse = context.Scanner.Buffer.Substring(start, end - start);
4638
#else
4739
var sourceToParse = context.Scanner.Buffer.AsSpan(start, end - start);
4840
#endif
@@ -66,31 +58,12 @@ public CompilationResult Compile(CompilationContext context)
6658
var success = context.DeclareSuccessVariable(result, false);
6759
var value = context.DeclareValueVariable(result, Expression.Default(typeof(decimal)));
6860

69-
// TODO: if !_skiptWhiteSpace and !NumberOptions.AllowSign then we don't need to store the reset position
70-
// since the ReadDecimal method will do it at the correct location.
71-
72-
//
61+
// var start = context.Scanner.Cursor.Offset;
7362
// var reset = context.Scanner.Cursor.Position;
74-
//
75-
// if (_skipWhiteSpace)
76-
// {
77-
// context.SkipWhiteSpace();
78-
// }
7963

64+
var start = context.DeclareOffsetVariable(result);
8065
var reset = context.DeclarePositionVariable(result);
8166

82-
if (_skipWhiteSpace)
83-
{
84-
result.Body.Add(context.ParserSkipWhiteSpace());
85-
}
86-
87-
// var start = context.Scanner.Cursor.Offset;
88-
89-
var start = Expression.Variable(typeof(int), $"start{context.NextNumber}");
90-
result.Variables.Add(start);
91-
92-
result.Body.Add(Expression.Assign(start, context.Offset()));
93-
9467
if ((_numberOptions & NumberOptions.AllowSign) == NumberOptions.AllowSign)
9568
{
9669
// if (!context.Scanner.ReadChar('-'))

‎src/Parlot/Fluent/Identifier.cs

Lines changed: 1 addition & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -8,24 +8,17 @@ public sealed class Identifier : Parser<TextSpan>, ICompilable
88
{
99
private readonly Func<char, bool> _extraStart;
1010
private readonly Func<char, bool> _extraPart;
11-
private readonly bool _skipWhiteSpace;
1211

13-
public Identifier(Func<char, bool> extraStart = null, Func<char, bool> extraPart = null, bool skipWhiteSpace = true)
12+
public Identifier(Func<char, bool> extraStart = null, Func<char, bool> extraPart = null)
1413
{
1514
_extraStart = extraStart;
1615
_extraPart = extraPart;
17-
_skipWhiteSpace = skipWhiteSpace;
1816
}
1917

2018
public override bool Parse(ParseContext context, ref ParseResult<TextSpan> result)
2119
{
2220
context.EnterParser(this);
2321

24-
if (_skipWhiteSpace)
25-
{
26-
context.SkipWhiteSpace();
27-
}
28-
2922
var first = context.Scanner.Cursor.Current;
3023

3124
if (Character.IsIdentifierStart(first) || _extraStart != null && _extraStart(first))
@@ -57,16 +50,6 @@ public CompilationResult Compile(CompilationContext context)
5750
var success = context.DeclareSuccessVariable(result, false);
5851
var value = context.DeclareValueVariable(result, Expression.Default(typeof(TextSpan)));
5952

60-
//if (_skipWhiteSpace)
61-
//{
62-
// context.SkipWhiteSpace();
63-
//}
64-
65-
if (_skipWhiteSpace)
66-
{
67-
result.Body.Add(context.ParserSkipWhiteSpace());
68-
}
69-
7053
// var first = context.Scanner.Cursor.Current;
7154

7255
var first = Expression.Parameter(typeof(char), $"first{context.NextNumber}");

0 commit comments

Comments
 (0)