Skip to content

Commit 9011aac

Browse files
ykolbinIaroslav Kolbinalexander-fenster
authored
fix: handling properly fields with leading and trailing comments after field with trailing comment (#1593)
Co-authored-by: Iaroslav Kolbin <y.kolbin@team.bumble.com> Co-authored-by: Alexander Fenster <fenster@google.com>
1 parent 57fc524 commit 9011aac

File tree

6 files changed

+51
-21
lines changed

6 files changed

+51
-21
lines changed

‎src/tokenize.js

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -103,11 +103,8 @@ function tokenize(source, alternateCommentMode) {
103103
var offset = 0,
104104
length = source.length,
105105
line = 1,
106-
commentType = null,
107-
commentText = null,
108-
commentLine = 0,
109-
commentLineEmpty = false,
110-
commentIsLeading = false;
106+
lastCommentLine = 0,
107+
comments = {};
111108

112109
var stack = [];
113110

@@ -160,10 +157,11 @@ function tokenize(source, alternateCommentMode) {
160157
* @inner
161158
*/
162159
function setComment(start, end, isLeading) {
163-
commentType = source.charAt(start++);
164-
commentLine = line;
165-
commentLineEmpty = false;
166-
commentIsLeading = isLeading;
160+
var comment = {
161+
type: source.charAt(start++),
162+
lineEmpty: false,
163+
leading: isLeading,
164+
};
167165
var lookback;
168166
if (alternateCommentMode) {
169167
lookback = 2; // alternate comment parsing: "//" or "/*"
@@ -175,7 +173,7 @@ function tokenize(source, alternateCommentMode) {
175173
do {
176174
if (--commentOffset < 0 ||
177175
(c = source.charAt(commentOffset)) === "\n") {
178-
commentLineEmpty = true;
176+
comment.lineEmpty = true;
179177
break;
180178
}
181179
} while (c === " " || c === "\t");
@@ -186,9 +184,12 @@ function tokenize(source, alternateCommentMode) {
186184
lines[i] = lines[i]
187185
.replace(alternateCommentMode ? setCommentAltRe : setCommentRe, "")
188186
.trim();
189-
commentText = lines
187+
comment.text = lines
190188
.join("\n")
191189
.trim();
190+
191+
comments[line] = comment;
192+
lastCommentLine = line;
192193
}
193194

194195
function isDoubleSlashCommentLine(startOffset) {
@@ -257,6 +258,9 @@ function tokenize(source, alternateCommentMode) {
257258
++offset;
258259
if (isDoc) {
259260
setComment(start, offset - 1, isLeadingComment);
261+
// Trailing comment cannot not be multi-line,
262+
// so leading comment state should be reset to handle potential next comments
263+
isLeadingComment = true;
260264
}
261265
++line;
262266
repeat = true;
@@ -272,12 +276,17 @@ function tokenize(source, alternateCommentMode) {
272276
break;
273277
}
274278
offset++;
279+
if (!isLeadingComment) {
280+
// Trailing comment cannot not be multi-line
281+
break;
282+
}
275283
} while (isDoubleSlashCommentLine(offset));
276284
} else {
277285
offset = Math.min(length, findEndOfLine(offset) + 1);
278286
}
279287
if (isDoc) {
280288
setComment(start, offset, isLeadingComment);
289+
isLeadingComment = true;
281290
}
282291
line++;
283292
repeat = true;
@@ -299,6 +308,7 @@ function tokenize(source, alternateCommentMode) {
299308
++offset;
300309
if (isDoc) {
301310
setComment(start, offset - 2, isLeadingComment);
311+
isLeadingComment = true;
302312
}
303313
repeat = true;
304314
} else {
@@ -374,17 +384,22 @@ function tokenize(source, alternateCommentMode) {
374384
*/
375385
function cmnt(trailingLine) {
376386
var ret = null;
387+
var comment;
377388
if (trailingLine === undefined) {
378-
if (commentLine === line - 1 && (alternateCommentMode || commentType === "*" || commentLineEmpty)) {
379-
ret = commentIsLeading ? commentText : null;
389+
comment = comments[line - 1];
390+
delete comments[line - 1];
391+
if (comment && (alternateCommentMode || comment.type === "*" || comment.lineEmpty)) {
392+
ret = comment.leading ? comment.text : null;
380393
}
381394
} else {
382395
/* istanbul ignore else */
383-
if (commentLine < trailingLine) {
396+
if (lastCommentLine < trailingLine) {
384397
peek();
385398
}
386-
if (commentLine === trailingLine && !commentLineEmpty && (alternateCommentMode || commentType === "/")) {
387-
ret = commentIsLeading ? null : commentText;
399+
comment = comments[trailingLine];
400+
delete comments[trailingLine];
401+
if (comment && !comment.lineEmpty && (alternateCommentMode || comment.type === "/")) {
402+
ret = comment.leading ? null : comment.text;
388403
}
389404
}
390405
return ret;

‎tests/api_tokenize.js

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,11 @@ tape.test("tokenize", function(test) {
4242
tn.next();
4343
test.equal(tn.cmnt(), null, "trailing comment should not be recognized as leading comment for next line");
4444
test.equal(tn.cmnt(tn.line), "trailing comment B", "should parse trailing comment");
45+
tn = tokenize("/// leading comment A\na /// trailing comment A\n/// leading comment B\nb /// trailing comment B\n");
46+
tn.next();
47+
test.equal(tn.cmnt(tn.line), "trailing comment A", "trailing comment should not contain leading comment from next line");
48+
tn.next();
49+
test.equal(tn.cmnt(), 'leading comment B', "leading comment should be present after trailing comment");
4550

4651
test.ok(expectError("something; /"), "should throw for unterminated line comments");
4752
test.ok(expectError("something; /* comment"), "should throw for unterminated block comments");
@@ -78,4 +83,4 @@ function expectError(proto) {
7883
} catch (e) {
7984
return e;
8085
}
81-
}
86+
}

‎tests/data/comments-alternate-parse.proto

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ enum Test3 {
7474
THREE = 3; // ignored
7575

7676
FOUR = 4; /// Other value with a comment.
77+
78+
// Leading comment for value with both types of comments after field with trailing comment.
79+
FIVE = 5; // Trailing comment for value with both types of comments after field with trailing comment.
7780
}
7881

7982
service ServiceTest {

‎tests/data/comments.proto

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,7 @@ enum Test3 {
4848
THREE = 3; /// Value with a comment.
4949

5050
FOUR = 4; /// Other value with a comment.
51+
52+
/// Leading comment for value with both types of comments after field with trailing comment.
53+
FIVE = 5; /// Trailing comment for value with both types of comments after field with trailing comment.
5154
}

‎tests/docs_comments.js

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ var tape = require("tape");
33
var protobuf = require("..");
44

55
tape.test("proto comments", function(test) {
6-
test.plan(10);
6+
test.plan(11);
77
protobuf.load("tests/data/comments.proto", function(err, root) {
88
if (err)
99
throw test.fail(err.message);
@@ -20,13 +20,14 @@ tape.test("proto comments", function(test) {
2020
test.equal(root.lookup("Test3").comments.TWO, null, "should not parse lines for enum values");
2121
test.equal(root.lookup("Test3").comments.THREE, "Preferred value with a comment.", "should parse lines for enum values and prefer on top over trailing");
2222
test.equal(root.lookup("Test3").comments.FOUR, "Other value with a comment.", "should not confuse previous trailing comments with comments for the next field");
23+
test.equal(root.lookup("Test3").comments.FIVE, "Leading comment for value with both types of comments after field with trailing comment.", "should not confuse previous field with trailing comment when leading comment is present");
2324

2425
test.end();
2526
});
2627
});
2728

2829
tape.test("proto comments with trailing comment preferred", function(test) {
29-
test.plan(10);
30+
test.plan(11);
3031
var options = {preferTrailingComment: true};
3132
var root = new protobuf.Root();
3233
root.load("tests/data/comments.proto", options, function(err, root) {
@@ -45,6 +46,7 @@ tape.test("proto comments with trailing comment preferred", function(test) {
4546
test.equal(root.lookup("Test3").comments.TWO, null, "should not parse lines for enum values");
4647
test.equal(root.lookup("Test3").comments.THREE, "Value with a comment.", "should prefer trailing comment when preferTrailingComment option enabled");
4748
test.equal(root.lookup("Test3").comments.FOUR, "Other value with a comment.", "should not confuse previous trailing comments with comments for the next field");
49+
test.equal(root.lookup("Test3").comments.FIVE, "Trailing comment for value with both types of comments after field with trailing comment.", "should not confuse previous field with trailing comment when leading comment is present");
4850

4951
test.end();
5052
});

‎tests/docs_comments_alternate_parse.js

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ var tape = require("tape");
33
var protobuf = require("..");
44

55
tape.test("proto comments in alternate-parse mode", function(test) {
6-
test.plan(23);
6+
test.plan(24);
77
var options = {alternateCommentMode: true};
88
var root = new protobuf.Root();
99
root.load("tests/data/comments-alternate-parse.proto", options, function(err, root) {
@@ -31,6 +31,7 @@ tape.test("proto comments in alternate-parse mode", function(test) {
3131
test.equal(root.lookup("Test3").comments.TWO, "Value with a single-line comment.", "should parse double-slash comments for enum values");
3232
test.equal(root.lookup("Test3").comments.THREE, "Value with a triple-slash comment.", "should parse lines for enum values and prefer on top over trailing");
3333
test.equal(root.lookup("Test3").comments.FOUR, "Other value with a comment.", "should not confuse previous trailing comments with comments for the next field");
34+
test.equal(root.lookup("Test3").comments.FIVE, "Leading comment for value with both types of comments after field with trailing comment.", "should not confuse previous field with trailing comment when leading comment is present");
3435

3536
test.equal(root.lookup("ServiceTest.SingleLineMethod").comment, 'My method does things');
3637
test.equal(root.lookup("ServiceTest.TwoLineMethodWithComment").comment, 'TwoLineMethodWithComment documentation');
@@ -42,7 +43,7 @@ tape.test("proto comments in alternate-parse mode", function(test) {
4243
});
4344

4445
tape.test("proto comments in alternate-parse mode with trailing comment preferred", function(test) {
45-
test.plan(23);
46+
test.plan(24);
4647
var options = {alternateCommentMode: true, preferTrailingComment: true};
4748
var root = new protobuf.Root();
4849
root.load("tests/data/comments-alternate-parse.proto", options, function(err, root) {
@@ -70,6 +71,7 @@ tape.test("proto comments in alternate-parse mode with trailing comment preferre
7071
test.equal(root.lookup("Test3").comments.TWO, "Value with a single-line comment.", "should parse double-slash comments for enum values");
7172
test.equal(root.lookup("Test3").comments.THREE, "ignored", "should prefer trailing comment when preferTrailingComment option enabled");
7273
test.equal(root.lookup("Test3").comments.FOUR, "Other value with a comment.", "should not confuse previous trailing comments with comments for the next field");
74+
test.equal(root.lookup("Test3").comments.FIVE, "Trailing comment for value with both types of comments after field with trailing comment.", "should not confuse previous field with trailing comment when leading comment is present");
7375

7476
test.equal(root.lookup("ServiceTest.SingleLineMethod").comment, 'My method does things');
7577
test.equal(root.lookup("ServiceTest.TwoLineMethodWithComment").comment, 'TwoLineMethodWithComment documentation');

0 commit comments

Comments
 (0)