Add support for numbered lists.

As part of that support, allow for the lexer tokens produced by the Markdown parse to have contained text. Specifically, a list item will have text like `"* "` or `"1. "`, which we can then use to determine how far to indent continuation lines.

PiperOrigin-RevId: 894215568

Author: eamonnmcmanus

core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocLexer.java

@@ -126,7 +126,12 @@ private ImmutableList<Token> generateTokens() throws LexException {
     tokens.add(token);
 
     while (!input.isExhausted()) {
-      tokens.addAll(markdownPositions.tokensAt(input.position()));
+      for (Token markdownToken : markdownPositions.tokensAt(input.position())) {
+        boolean consumed = input.tryConsume(markdownToken.value());
+        verify(consumed, "Did not consume markdown token: %s", markdownToken);
+        var unused = input.readAndResetRecorded();
+        tokens.add(markdownToken);
+      }
       token = readToken();
       tokens.add(token);
     }

core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocWriter.java

@@ -183,14 +183,18 @@ void writeListOpen(Token token) {
   }
 
   void writeListClose(Token token) {
-    requestNewline();
+    if (classicJavadoc) {
+      requestNewline();
+    }
 
     continuingListItemStack.popIfNotEmpty();
     continuingListStack.popIfNotEmpty();
     writeToken(token);
     postWriteModifiedContinuingListStack.popIfNotEmpty();
 
-    requestBlankLine();
+    if (classicJavadoc) {
+      requestBlankLine();
+    }
   }
 
   void writeListItemOpen(Token token) {
@@ -202,7 +206,7 @@ void writeListItemOpen(Token token) {
     }
     writeToken(token);
     continuingListItemOfInnermostList = true;
-    int indent = token.value().isEmpty() ? 2 : 4; // Indent 2 for Markdown, 4 for HTML.
+    int indent = token.value().length();
     continuingListItemStack.push(indent);
   }
 

core/src/main/java/com/google/googlejavaformat/java/javadoc/MarkdownPositions.java

@@ -13,14 +13,19 @@
  */
 package com.google.googlejavaformat.java.javadoc;
 
+import static com.google.common.base.Verify.verify;
+
 import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.ListMultimap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import org.commonmark.node.BulletList;
 import org.commonmark.node.Heading;
 import org.commonmark.node.ListItem;
 import org.commonmark.node.Node;
+import org.commonmark.node.OrderedList;
 import org.commonmark.node.Paragraph;
 import org.commonmark.parser.IncludeSourceSpans;
 import org.commonmark.parser.Parser;
@@ -50,7 +55,7 @@ private MarkdownPositions(ImmutableListMultimap<Integer, Token> positionToToken)
   static MarkdownPositions parse(String input) {
     Node document = PARSER.parse(input);
     ListMultimap<Integer, Token> positionToToken = ArrayListMultimap.create();
-    new TokenVisitor(positionToToken).visit(document);
+    new TokenVisitor(input, positionToToken).visit(document);
     return new MarkdownPositions(ImmutableListMultimap.copyOf(positionToToken));
   }
 
@@ -59,9 +64,11 @@ ImmutableList<Token> tokensAt(int position) {
   }
 
   private static class TokenVisitor {
+    private final String input;
     private final ListMultimap<Integer, Token> positionToToken;
 
-    TokenVisitor(ListMultimap<Integer, Token> positionToToken) {
+    TokenVisitor(String input, ListMultimap<Integer, Token> positionToToken) {
+      this.input = input;
       this.positionToToken = positionToToken;
     }
 
@@ -74,8 +81,15 @@ void visit(Node node) {
             addSpan(positionToToken, paragraph, PARAGRAPH_OPEN_TOKEN, PARAGRAPH_CLOSE_TOKEN);
         case BulletList bulletList ->
             addSpan(positionToToken, bulletList, LIST_OPEN_TOKEN, LIST_CLOSE_TOKEN);
+        case OrderedList orderedList ->
+            addSpan(positionToToken, orderedList, LIST_OPEN_TOKEN, LIST_CLOSE_TOKEN);
         case ListItem listItem -> {
-          addSpan(positionToToken, listItem, LIST_ITEM_OPEN_TOKEN, LIST_ITEM_CLOSE_TOKEN);
+          int startPosition = listItem.getSourceSpans().getFirst().getInputIndex();
+          Matcher matcher =
+              LIST_ITEM_START_PATTERN.matcher(input).region(startPosition, input.length());
+          verify(matcher.lookingAt());
+          Token openToken = new Token(Token.Type.LIST_ITEM_OPEN_TAG, matcher.group(1));
+          addSpan(positionToToken, listItem, openToken, LIST_ITEM_CLOSE_TOKEN);
           if (listItem.getFirstChild() instanceof Paragraph paragraph) {
             // A ListItem typically contains a Paragraph, but we don't want to visit that Paragraph
             // because that would lead us to introduce a line break after the list introduction
@@ -133,12 +147,17 @@ public String toString() {
 
   private static final Parser PARSER =
       Parser.builder().includeSourceSpans(IncludeSourceSpans.BLOCKS_AND_INLINES).build();
+
   private static final Token HEADER_OPEN_TOKEN = new Token(Token.Type.HEADER_OPEN_TAG, "");
   private static final Token HEADER_CLOSE_TOKEN = new Token(Token.Type.HEADER_CLOSE_TAG, "");
   private static final Token PARAGRAPH_OPEN_TOKEN = new Token(Token.Type.PARAGRAPH_OPEN_TAG, "");
   private static final Token PARAGRAPH_CLOSE_TOKEN = new Token(Token.Type.PARAGRAPH_CLOSE_TAG, "");
   private static final Token LIST_OPEN_TOKEN = new Token(Token.Type.LIST_OPEN_TAG, "");
   private static final Token LIST_CLOSE_TOKEN = new Token(Token.Type.LIST_CLOSE_TAG, "");
-  private static final Token LIST_ITEM_OPEN_TOKEN = new Token(Token.Type.LIST_ITEM_OPEN_TAG, "");
   private static final Token LIST_ITEM_CLOSE_TOKEN = new Token(Token.Type.LIST_ITEM_CLOSE_TAG, "");
+
+  // The leading \s here works around what appears to be a CommonMark bug. We shouldn't ever see
+  // space at the purported start of a list item?
+  private static final Pattern LIST_ITEM_START_PATTERN =
+      Pattern.compile("(?:\\s*)((-|\\*|[0-9]+\\.)\\s)");
 }

core/src/test/java/com/google/googlejavaformat/java/JavadocFormattingTest.java

@@ -1677,7 +1677,7 @@ public void moduleMarkdown() {
   }
 
   @Test
-  public void markdownBulletList() {
+  public void markdownLists() {
     assume().that(MARKDOWN_JAVADOC_SUPPORTED).isTrue();
     String input =
 """
@@ -1689,6 +1689,14 @@ public void markdownBulletList() {
 /// - a nested list
 ///   * nested thing 1
 ///   * nested thing 2
+/// -    a nested numbered list with unnecessary leading whitespace
+///      1. nested thing 1
+///         on more than one line
+///      2. nested thing 2 on only one line but which is long enough that it is going to need to be wrapped
+///
+///      3. nested thing 3 after a blank line
+///
+/// A following paragraph.
 class Test {}
 """;
     String expected =
@@ -1701,6 +1709,13 @@ class Test {}
 /// - a nested list
 ///   * nested thing 1
 ///   * nested thing 2
+/// - a nested numbered list with unnecessary leading whitespace
+///   1. nested thing 1 on more than one line
+///   2. nested thing 2 on only one line but which is long enough that it is going to need to be
+///      wrapped
+///   3. nested thing 3 after a blank line
+///
+/// A following paragraph.
 class Test {}
 """;
     doFormatTest(input, expected);

core/src/test/java/com/google/googlejavaformat/java/javadoc/MarkdownPositionsTest.java

@@ -47,9 +47,9 @@ public void list() {
     ImmutableListMultimap<Integer, Token> expected =
         ImmutableListMultimap.<Integer, Token>builder()
             .put(firstBullet, new Token(Token.Type.LIST_OPEN_TAG, ""))
-            .put(firstBullet, new Token(Token.Type.LIST_ITEM_OPEN_TAG, ""))
+            .put(firstBullet, new Token(Token.Type.LIST_ITEM_OPEN_TAG, "- "))
             .put(secondBullet - 1, new Token(Token.Type.LIST_ITEM_CLOSE_TAG, ""))
-            .put(secondBullet, new Token(Token.Type.LIST_ITEM_OPEN_TAG, ""))
+            .put(secondBullet, new Token(Token.Type.LIST_ITEM_OPEN_TAG, "- "))
             .put(end, new Token(Token.Type.LIST_ITEM_CLOSE_TAG, ""))
             .put(end, new Token(Token.Type.LIST_CLOSE_TAG, ""))
             .build();