Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Commit 0967d19

Browse files
jensjohacommit-bot@chromium.org
jensjoha
authored and
commit-bot@chromium.org
committed
[parser] Fix assert error in scanner
If an identifier starts with a non-ascii-character and has a comment an assert was triggered about the comment not being attached to the token. This CL attaches the comment to the token (which it should be) and thus avoiding the assert trigger. Change-Id: Id261970b88ca721d4b3a996abfb8ff43f0ec8341 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/161102 Reviewed-by: Johnni Winther <johnniwinther@google.com> Reviewed-by: Brian Wilkerson <brianwilkerson@google.com> Commit-Queue: Jens Johansen <jensj@google.com>
1 parent 460c2d2 commit 0967d19

File tree

6 files changed

+422
-2
lines changed

6 files changed

+422
-2
lines changed

‎pkg/_fe_analyzer_shared/lib/src/scanner/abstract_scanner.dart‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1894,8 +1894,9 @@ abstract class AbstractScanner implements Scanner {
18941894
codeUnits.add(next);
18951895
next = advance();
18961896
}
1897-
appendToken(new StringToken.fromString(TokenType.IDENTIFIER,
1898-
new String.fromCharCodes(codeUnits), charOffset));
1897+
appendToken(new StringToken.fromString(
1898+
TokenType.IDENTIFIER, new String.fromCharCodes(codeUnits), charOffset,
1899+
precedingComments: comments));
18991900
return next;
19001901
} else {
19011902
prependErrorToken(errorToken);

‎pkg/front_end/parser_testcases/error_recovery/comment_on_non_ascii_identifier.dart‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
main() {
2+
// Identifiers can't have non-ascii characters.
3+
// Having this comment shouldn't trigger any asserts though.
4+
int æFoo = 42;
5+
// Try comment on an identifier that doesn't start with a non-ascii char too.
6+
int fooÆ = 42;
7+
// Try comment on an OK identifier too.
8+
int foo = 42;
9+
print(/* comment */ æFoo);
10+
print(/* comment */ fooÆ);
11+
print(/* comment */ foo);
12+
}

‎pkg/front_end/parser_testcases/error_recovery/comment_on_non_ascii_identifier.dart.expect‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
Problems reported:
2+
3+
parser/error_recovery/comment_on_non_ascii_identifier:4:7: The non-ASCII character 'æ' (U+00E6) can't be used in identifiers, only in strings and comments.
4+
int æFoo = 42;
5+
^
6+
7+
parser/error_recovery/comment_on_non_ascii_identifier:6:10: The non-ASCII character 'Æ' (U+00C6) can't be used in identifiers, only in strings and comments.
8+
int fooÆ = 42;
9+
^
10+
11+
parser/error_recovery/comment_on_non_ascii_identifier:9:23: The non-ASCII character 'æ' (U+00E6) can't be used in identifiers, only in strings and comments.
12+
print(/* comment */ æFoo);
13+
^
14+
15+
parser/error_recovery/comment_on_non_ascii_identifier:10:26: The non-ASCII character 'Æ' (U+00C6) can't be used in identifiers, only in strings and comments.
16+
print(/* comment */ fooÆ);
17+
^
18+
19+
beginCompilationUnit(main)
20+
beginMetadataStar(main)
21+
endMetadataStar(0)
22+
beginTopLevelMember(main)
23+
beginTopLevelMethod(NonAsciiIdentifierToken(198), null)
24+
handleNoType(NonAsciiIdentifierToken(198))
25+
handleIdentifier(main, topLevelFunctionDeclaration)
26+
handleNoTypeVariables(()
27+
beginFormalParameters((, MemberKind.TopLevelMethod)
28+
endFormalParameters(0, (, ), MemberKind.TopLevelMethod)
29+
handleAsyncModifier(null, null)
30+
beginBlockFunctionBody({)
31+
beginMetadataStar(int)
32+
endMetadataStar(0)
33+
handleIdentifier(int, typeReference)
34+
handleNoTypeArguments(æFoo)
35+
handleType(int, null)
36+
beginVariablesDeclaration(æFoo, null, null)
37+
handleIdentifier(æFoo, localVariableDeclaration)
38+
beginInitializedIdentifier(æFoo)
39+
beginVariableInitializer(=)
40+
handleLiteralInt(42)
41+
endVariableInitializer(=)
42+
endInitializedIdentifier(æFoo)
43+
endVariablesDeclaration(1, ;)
44+
beginMetadataStar(int)
45+
endMetadataStar(0)
46+
handleIdentifier(int, typeReference)
47+
handleNoTypeArguments(fooÆ)
48+
handleType(int, null)
49+
beginVariablesDeclaration(fooÆ, null, null)
50+
handleIdentifier(fooÆ, localVariableDeclaration)
51+
beginInitializedIdentifier(fooÆ)
52+
beginVariableInitializer(=)
53+
handleLiteralInt(42)
54+
endVariableInitializer(=)
55+
endInitializedIdentifier(fooÆ)
56+
endVariablesDeclaration(1, ;)
57+
beginMetadataStar(int)
58+
endMetadataStar(0)
59+
handleIdentifier(int, typeReference)
60+
handleNoTypeArguments(foo)
61+
handleType(int, null)
62+
beginVariablesDeclaration(foo, null, null)
63+
handleIdentifier(foo, localVariableDeclaration)
64+
beginInitializedIdentifier(foo)
65+
beginVariableInitializer(=)
66+
handleLiteralInt(42)
67+
endVariableInitializer(=)
68+
endInitializedIdentifier(foo)
69+
endVariablesDeclaration(1, ;)
70+
handleIdentifier(print, expression)
71+
handleNoTypeArguments(()
72+
beginArguments(()
73+
handleIdentifier(æFoo, expression)
74+
handleNoTypeArguments())
75+
handleNoArguments())
76+
handleSend(æFoo, ))
77+
endArguments(1, (, ))
78+
handleSend(print, ;)
79+
handleExpressionStatement(;)
80+
handleIdentifier(print, expression)
81+
handleNoTypeArguments(()
82+
beginArguments(()
83+
handleIdentifier(fooÆ, expression)
84+
handleNoTypeArguments())
85+
handleNoArguments())
86+
handleSend(fooÆ, ))
87+
endArguments(1, (, ))
88+
handleSend(print, ;)
89+
handleExpressionStatement(;)
90+
handleIdentifier(print, expression)
91+
handleNoTypeArguments(()
92+
beginArguments(()
93+
handleIdentifier(foo, expression)
94+
handleNoTypeArguments())
95+
handleNoArguments())
96+
handleSend(foo, ))
97+
endArguments(1, (, ))
98+
handleSend(print, ;)
99+
handleExpressionStatement(;)
100+
endBlockFunctionBody(6, {, })
101+
endTopLevelMethod(main, null, })
102+
endTopLevelDeclaration()
103+
handleErrorToken(NonAsciiIdentifierToken(230))
104+
handleRecoverableError(Message[NonAsciiIdentifier, The non-ASCII character 'æ' (U+00E6) can't be used in identifiers, only in strings and comments., Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign)., {character: æ, codePoint: 230}], NonAsciiIdentifierToken(230), NonAsciiIdentifierToken(230))
105+
handleErrorToken(NonAsciiIdentifierToken(198))
106+
handleRecoverableError(Message[NonAsciiIdentifier, The non-ASCII character 'Æ' (U+00C6) can't be used in identifiers, only in strings and comments., Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign)., {character: Æ, codePoint: 198}], NonAsciiIdentifierToken(198), NonAsciiIdentifierToken(198))
107+
handleErrorToken(NonAsciiIdentifierToken(230))
108+
handleRecoverableError(Message[NonAsciiIdentifier, The non-ASCII character 'æ' (U+00E6) can't be used in identifiers, only in strings and comments., Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign)., {character: æ, codePoint: 230}], NonAsciiIdentifierToken(230), NonAsciiIdentifierToken(230))
109+
handleErrorToken(NonAsciiIdentifierToken(198))
110+
handleRecoverableError(Message[NonAsciiIdentifier, The non-ASCII character 'Æ' (U+00C6) can't be used in identifiers, only in strings and comments., Try using an US-ASCII letter, a digit, '_' (an underscore), or '$' (a dollar sign)., {character: Æ, codePoint: 198}], NonAsciiIdentifierToken(198), NonAsciiIdentifierToken(198))
111+
endCompilationUnit(1, )

0 commit comments

Comments
 (0)