Skip to content

Fix next token parsing after a function definition #573

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
saghul merged 2 commits into from
Oct 5, 2024
Merged

Fix next token parsing after a function definition #573

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ff019ee
Fix next token parsing after a function definition
saghul Oct 4, 2024
94513ae
fixup!
saghul Oct 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 53 additions & 34 deletions quickjs.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19224,6 +19224,48 @@ static __exception int ident_realloc(JSContext *ctx, char **pbuf, size_t *psize,
return 0;
}

/* convert a TOK_IDENT to a keyword when needed */
static void update_token_ident(JSParseState *s)
{
if (s->token.u.ident.atom <= JS_ATOM_LAST_KEYWORD ||
(s->token.u.ident.atom <= JS_ATOM_LAST_STRICT_KEYWORD &&
(s->cur_func->js_mode & JS_MODE_STRICT)) ||
(s->token.u.ident.atom == JS_ATOM_yield &&
((s->cur_func->func_kind & JS_FUNC_GENERATOR) ||
(s->cur_func->func_type == JS_PARSE_FUNC_ARROW &&
!s->cur_func->in_function_body && s->cur_func->parent &&
(s->cur_func->parent->func_kind & JS_FUNC_GENERATOR)))) ||
(s->token.u.ident.atom == JS_ATOM_await &&
(s->is_module ||
(s->cur_func->func_kind & JS_FUNC_ASYNC) ||
s->cur_func->func_type == JS_PARSE_FUNC_CLASS_STATIC_INIT ||
(s->cur_func->func_type == JS_PARSE_FUNC_ARROW &&
!s->cur_func->in_function_body && s->cur_func->parent &&
((s->cur_func->parent->func_kind & JS_FUNC_ASYNC) ||
s->cur_func->parent->func_type == JS_PARSE_FUNC_CLASS_STATIC_INIT))))) {
if (s->token.u.ident.has_escape) {
s->token.u.ident.is_reserved = TRUE;
s->token.val = TOK_IDENT;
} else {
/* The keywords atoms are pre allocated */
s->token.val = s->token.u.ident.atom - 1 + TOK_FIRST_KEYWORD;
}
}
}

/* if the current token is an identifier or keyword, reparse it
according to the current function type */
static void reparse_ident_token(JSParseState *s)
{
if (s->token.val == TOK_IDENT ||
(s->token.val >= TOK_FIRST_KEYWORD &&
s->token.val <= TOK_LAST_KEYWORD)) {
s->token.val = TOK_IDENT;
s->token.u.ident.is_reserved = FALSE;
update_token_ident(s);
}
}

/* 'c' is the first character. Return JS_ATOM_NULL in case of error */
static JSAtom parse_ident(JSParseState *s, const uint8_t **pp,
BOOL *pident_has_escape, int c, BOOL is_private)
Expand Down Expand Up @@ -19436,36 +19478,8 @@ static __exception int next_token(JSParseState *s)
s->token.u.ident.atom = atom;
s->token.u.ident.has_escape = ident_has_escape;
s->token.u.ident.is_reserved = FALSE;
// TODO(bnoordhuis) accept await when used in a function expression
// inside the static initializer block
if (s->cur_func->func_type == JS_PARSE_FUNC_CLASS_STATIC_INIT &&
(atom == JS_ATOM_arguments || atom == JS_ATOM_await)) {
s->token.u.ident.is_reserved = TRUE;
s->token.val = TOK_IDENT;
} else if (s->token.u.ident.atom <= JS_ATOM_LAST_KEYWORD ||
(s->token.u.ident.atom <= JS_ATOM_LAST_STRICT_KEYWORD &&
(s->cur_func->js_mode & JS_MODE_STRICT)) ||
(s->token.u.ident.atom == JS_ATOM_yield &&
((s->cur_func->func_kind & JS_FUNC_GENERATOR) ||
(s->cur_func->func_type == JS_PARSE_FUNC_ARROW &&
!s->cur_func->in_function_body && s->cur_func->parent &&
(s->cur_func->parent->func_kind & JS_FUNC_GENERATOR)))) ||
(s->token.u.ident.atom == JS_ATOM_await &&
(s->is_module ||
(((s->cur_func->func_kind & JS_FUNC_ASYNC) ||
(s->cur_func->func_type == JS_PARSE_FUNC_ARROW &&
!s->cur_func->in_function_body && s->cur_func->parent &&
(s->cur_func->parent->func_kind & JS_FUNC_ASYNC))))))) {
if (ident_has_escape) {
s->token.u.ident.is_reserved = TRUE;
s->token.val = TOK_IDENT;
} else {
/* The keywords atoms are pre allocated */
s->token.val = s->token.u.ident.atom - 1 + TOK_FIRST_KEYWORD;
}
} else {
s->token.val = TOK_IDENT;
}
s->token.val = TOK_IDENT;
update_token_ident(s);
break;
case '#':
/* private name */
Expand Down Expand Up @@ -32498,10 +32512,9 @@ static __exception int js_parse_function_decl2(JSParseState *s,
func_type == JS_PARSE_FUNC_EXPR &&
(func_kind & JS_FUNC_GENERATOR)) ||
(s->token.u.ident.atom == JS_ATOM_await &&
func_type == JS_PARSE_FUNC_EXPR &&
(func_kind & JS_FUNC_ASYNC)) ||
(s->token.u.ident.atom == JS_ATOM_await &&
func_type == JS_PARSE_FUNC_CLASS_STATIC_INIT)) {
((func_type == JS_PARSE_FUNC_EXPR &&
(func_kind & JS_FUNC_ASYNC)) ||
func_type == JS_PARSE_FUNC_CLASS_STATIC_INIT))) {
return js_parse_error_reserved_identifier(s);
}
}
Expand Down Expand Up @@ -32903,6 +32916,12 @@ static __exception int js_parse_function_decl2(JSParseState *s,
done:
s->cur_func = fd->parent;

/* Reparse identifiers after the function is terminated so that
the token is parsed in the englobing function. It could be done
by just using next_token() here for normal functions, but it is
necessary for arrow functions with an expression body. */
reparse_ident_token(s);

/* create the function object */
{
int idx;
Expand Down
6 changes: 0 additions & 6 deletions test262_errors.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -630,8 +630,6 @@ test262/test/built-ins/TypedArrayConstructors/internals/Set/key-is-canonical-inv
test262/test/built-ins/TypedArrayConstructors/internals/Set/key-is-canonical-invalid-index-prototype-chain-set.js:35: strict mode: Test262Error: value should not be coerced Expected SameValue(«110», «0») to be true
test262/test/built-ins/TypedArrayConstructors/internals/Set/key-is-canonical-invalid-index-reflect-set.js:35: Test262Error: value should not be coerced Expected SameValue(«160», «0») to be true
test262/test/built-ins/TypedArrayConstructors/internals/Set/key-is-canonical-invalid-index-reflect-set.js:35: strict mode: Test262Error: value should not be coerced Expected SameValue(«160», «0») to be true
test262/test/language/expressions/arrow-function/static-init-await-reference.js:12: unexpected error type: Test262: This statement should not be evaluated.
test262/test/language/expressions/arrow-function/static-init-await-reference.js:12: strict mode: unexpected error type: Test262: This statement should not be evaluated.
test262/test/language/expressions/assignment/destructuring/iterator-destructuring-property-reference-target-evaluation-order.js:42: Test262Error: Expected [source, iterator, target, target-key, target-key-tostring, iterator-step, iterator-done, set] and [source, iterator, target, target-key, iterator-step, iterator-done, target-key-tostring, set] to have the same contents.
test262/test/language/expressions/assignment/destructuring/iterator-destructuring-property-reference-target-evaluation-order.js:42: strict mode: Test262Error: Expected [source, iterator, target, target-key, target-key-tostring, iterator-step, iterator-done, set] and [source, iterator, target, target-key, iterator-step, iterator-done, target-key-tostring, set] to have the same contents.
test262/test/language/expressions/assignment/destructuring/keyed-destructuring-property-reference-target-evaluation-order.js:32: Test262Error: Expected [source, source-key, source-key-tostring, target, target-key, target-key-tostring, get, set] and [source, source-key, source-key-tostring, target, target-key, get, target-key-tostring, set] to have the same contents.
Expand All @@ -640,10 +638,6 @@ test262/test/language/expressions/assignment/target-member-computed-reference.js
test262/test/language/expressions/assignment/target-member-computed-reference.js:22: strict mode: Test262Error: Expected a DummyError but got a Test262Error
test262/test/language/expressions/assignment/target-super-computed-reference.js:20: Test262Error: Expected a DummyError but got a Test262Error
test262/test/language/expressions/assignment/target-super-computed-reference.js:20: strict mode: Test262Error: Expected a DummyError but got a Test262Error
test262/test/language/expressions/function/static-init-await-binding.js:16: SyntaxError: 'await' is a reserved identifier
test262/test/language/expressions/function/static-init-await-binding.js:16: strict mode: SyntaxError: 'await' is a reserved identifier
test262/test/language/expressions/generators/static-init-await-binding.js:16: SyntaxError: 'await' is a reserved identifier
test262/test/language/expressions/generators/static-init-await-binding.js:16: strict mode: SyntaxError: 'await' is a reserved identifier
test262/test/language/expressions/in/private-field-invalid-assignment-target.js:23: unexpected error type: Test262: This statement should not be evaluated.
test262/test/language/expressions/in/private-field-invalid-assignment-target.js:23: strict mode: unexpected error type: Test262: This statement should not be evaluated.
test262/test/language/expressions/member-expression/computed-reference-null-or-undefined.js:28: Test262Error: Expected a TypeError but got a Test262Error
Expand Down
23 changes: 23 additions & 0 deletions tests/test_language.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -691,6 +691,28 @@ function test_optional_chaining()
assert((a?.["b"])().c, 42);
}

function test_parse_semicolon()
{
/* 'yield' or 'await' may not be considered as a token if the
previous ';' is missing */
function *f()
{
function func() {
}
yield 1;
var h = x => x + 1
yield 2;
}
async function g()
{
function func() {
}
await 1;
var h = x => x + 1
await 2;
}
}

test_op1();
test_cvt();
test_eq();
Expand All @@ -714,3 +736,4 @@ test_reserved_names();
test_number_literals();
test_syntax();
test_optional_chaining();
test_parse_semicolon();