Initial support for fenced code blocks in Markdown Javadoc.

This removes one of the main areas where Markdown comments would be mangled.

PiperOrigin-RevId: 896076206

Author: eamonnmcmanus

core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocFormatter.java

@@ -40,6 +40,7 @@
 import com.google.googlejavaformat.java.javadoc.Token.ListItemOpenTag;
 import com.google.googlejavaformat.java.javadoc.Token.ListOpenTag;
 import com.google.googlejavaformat.java.javadoc.Token.Literal;
+import com.google.googlejavaformat.java.javadoc.Token.MarkdownFencedCodeBlock;
 import com.google.googlejavaformat.java.javadoc.Token.MoeBeginStripComment;
 import com.google.googlejavaformat.java.javadoc.Token.MoeEndStripComment;
 import com.google.googlejavaformat.java.javadoc.Token.OptionalLineBreak;
@@ -134,6 +135,7 @@ private static String render(List<Token> input, int blockIndent, boolean classic
         case ParagraphCloseTag unused -> {}
         case ListItemCloseTag unused -> {}
         case OptionalLineBreak unused -> {}
+        case MarkdownFencedCodeBlock t -> output.writeMarkdownFencedCodeBlock(t);
       }
     }
     throw new AssertionError();

core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocLexer.java

@@ -127,11 +127,26 @@ private ImmutableList<Token> generateTokens() throws LexException {
     tokens.add(token);
 
     while (!input.isExhausted()) {
-      for (Token markdownToken : markdownPositions.tokensAt(input.position())) {
-        boolean consumed = input.tryConsume(markdownToken.value());
-        verify(consumed, "Did not consume markdown token: %s", markdownToken);
-        var unused = input.readAndResetRecorded();
-        tokens.add(markdownToken);
+      boolean moreMarkdown;
+      do {
+        moreMarkdown = false;
+        // If there are one or more markdown tokens at the current position, consume their text and
+        // add them to the token list. If a token has non-empty text, consuming its text changes the
+        // position, so we need to start looking for markdown tokens at the new position. It is
+        // assumed that there are no other tokens (markdown or otherwise) in a non-empty text span
+        // covered by a markdown token.
+        for (Token markdownToken : markdownPositions.tokensAt(input.position())) {
+          tokens.add(markdownToken);
+          if (!markdownToken.value().isEmpty()) {
+            boolean consumed = input.tryConsume(markdownToken.value());
+            verify(consumed, "Did not consume markdown token: %s", markdownToken);
+            var unused = input.readAndResetRecorded();
+            moreMarkdown = true;
+          }
+        }
+      } while (moreMarkdown);
+      if (input.isExhausted()) {
+        break;
       }
       token = readToken();
       tokens.add(token);

core/src/main/java/com/google/googlejavaformat/java/javadoc/JavadocWriter.java

@@ -23,6 +23,7 @@
 import static com.google.googlejavaformat.java.javadoc.JavadocWriter.RequestedWhitespace.NONE;
 import static com.google.googlejavaformat.java.javadoc.JavadocWriter.RequestedWhitespace.WHITESPACE;
 
+import com.google.googlejavaformat.java.javadoc.Token.BrTag;
 import com.google.googlejavaformat.java.javadoc.Token.CodeCloseTag;
 import com.google.googlejavaformat.java.javadoc.Token.CodeOpenTag;
 import com.google.googlejavaformat.java.javadoc.Token.FooterJavadocTagStart;
@@ -33,6 +34,7 @@
 import com.google.googlejavaformat.java.javadoc.Token.ListItemOpenTag;
 import com.google.googlejavaformat.java.javadoc.Token.ListOpenTag;
 import com.google.googlejavaformat.java.javadoc.Token.Literal;
+import com.google.googlejavaformat.java.javadoc.Token.MarkdownFencedCodeBlock;
 import com.google.googlejavaformat.java.javadoc.Token.MoeBeginStripComment;
 import com.google.googlejavaformat.java.javadoc.Token.MoeEndStripComment;
 import com.google.googlejavaformat.java.javadoc.Token.PreCloseTag;
@@ -310,7 +312,7 @@ void writeHtmlComment(HtmlComment token) {
     requestNewline();
   }
 
-  void writeBr(Token token) {
+  void writeBr(BrTag token) {
     writeToken(token);
 
     requestNewline();
@@ -324,6 +326,22 @@ void writeLiteral(Literal token) {
     writeToken(token);
   }
 
+  void writeMarkdownFencedCodeBlock(MarkdownFencedCodeBlock token) {
+    flushWhitespace();
+    output.append(token.start());
+    token
+        .literal()
+        .lines()
+        .forEach(
+            line -> {
+              writeNewline();
+              output.append(line);
+            });
+    writeNewline();
+    output.append(token.end());
+    requestBlankLine();
+  }
+
   @Override
   public String toString() {
     return output.toString();
@@ -350,12 +368,13 @@ enum RequestedWhitespace {
     BLANK_LINE,
   }
 
-  private void writeToken(Token token) {
+  private void flushWhitespace() {
     if (requestedMoeBeginStripComment != null) {
       requestNewline();
     }
 
-    if (requestedWhitespace == BLANK_LINE
+    if (classicJavadoc
+        && requestedWhitespace == BLANK_LINE
         && (!postWriteModifiedContinuingListStack.isEmpty() || continuingFooterTag)) {
       /*
        * We don't write blank lines inside lists or footer tags, even in cases where we otherwise
@@ -374,6 +393,14 @@ private void writeToken(Token token) {
       writeNewline();
       requestedWhitespace = NONE;
     }
+  }
+
+  private void writeToken(Token token) {
+    if (token.value().isEmpty()) {
+      return;
+    }
+
+    flushWhitespace();
     boolean needWhitespace = (requestedWhitespace == WHITESPACE);
 
     /*

core/src/main/java/com/google/googlejavaformat/java/javadoc/MarkdownPositions.java

@@ -25,11 +25,13 @@
 import com.google.googlejavaformat.java.javadoc.Token.ListItemCloseTag;
 import com.google.googlejavaformat.java.javadoc.Token.ListItemOpenTag;
 import com.google.googlejavaformat.java.javadoc.Token.ListOpenTag;
+import com.google.googlejavaformat.java.javadoc.Token.MarkdownFencedCodeBlock;
 import com.google.googlejavaformat.java.javadoc.Token.ParagraphCloseTag;
 import com.google.googlejavaformat.java.javadoc.Token.ParagraphOpenTag;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import org.commonmark.node.BulletList;
+import org.commonmark.node.FencedCodeBlock;
 import org.commonmark.node.Heading;
 import org.commonmark.node.ListItem;
 import org.commonmark.node.Node;
@@ -102,6 +104,25 @@ void visit(Node node) {
             visitNodeList(paragraph.getNext());
           }
         }
+        case FencedCodeBlock fencedCodeBlock -> {
+          // Any indentation before the code block is part of FencedCodeBlock. This makes sense
+          // because the lines inside the code block must also be indented by that amount. That
+          // indentation gets subtracted from FencedCodeBlock.getLiteral(), which is the actual text
+          // represented by the code block.
+          int start = startPosition(fencedCodeBlock) + fencedCodeBlock.getFenceIndent();
+          MarkdownFencedCodeBlock token =
+              new MarkdownFencedCodeBlock(
+                  input.substring(start, endPosition(fencedCodeBlock)),
+                  fencedCodeBlock
+                          .getFenceCharacter()
+                          .repeat(fencedCodeBlock.getOpeningFenceLength())
+                      + fencedCodeBlock.getInfo(),
+                  fencedCodeBlock
+                      .getFenceCharacter()
+                      .repeat(fencedCodeBlock.getClosingFenceLength()),
+                  fencedCodeBlock.getLiteral());
+          positionToToken.get(start).addLast(token);
+        }
         // TODO: others
         default -> {}
       }
@@ -131,12 +152,17 @@ private void visitNodeList(Node node) {
      */
     private void addSpan(Node node, Token startToken, Token endToken) {
       // We could write the first part more simply as a `put`, but we do it this way for symmetry.
-      var first = node.getSourceSpans().getFirst();
-      int startPosition = first.getInputIndex();
-      positionToToken.get(startPosition).addLast(startToken);
+      positionToToken.get(startPosition(node)).addLast(startToken);
+      positionToToken.get(endPosition(node)).addFirst(endToken);
+    }
+
+    private int startPosition(Node node) {
+      return node.getSourceSpans().getFirst().getInputIndex();
+    }
+
+    private int endPosition(Node node) {
       var last = node.getSourceSpans().getLast();
-      int endPosition = last.getInputIndex() + last.getLength();
-      positionToToken.get(endPosition).addFirst(endToken);
+      return last.getInputIndex() + last.getLength();
     }
   }
 

core/src/main/java/com/google/googlejavaformat/java/javadoc/Token.java

@@ -103,6 +103,27 @@ record HtmlComment(String value) implements Token {}
 
   record BrTag(String value) implements Token {}
 
+  /**
+   * A fenced code block, like:
+   *
+   * <pre>
+   * ```java
+   * code block
+   * with an info string ("java")
+   * ```
+   * </pre>
+   *
+   * @param value the full text of the code block as it appeared in the input, including the start
+   *     and end fences and the literal content.
+   * @param start the start fence, including the info string if any ({@code ```java} in the
+   *     example).
+   * @param end the end fence.
+   * @param literal the text that the code block represents. This does not include the start and end
+   *     fences, nor any indentation that precedes these fences and every intervening line.
+   */
+  record MarkdownFencedCodeBlock(String value, String start, String end, String literal)
+      implements Token {}
+
   /**
    * Whitespace that is not in a {@code <pre>} or {@code <table>} section. Whitespace includes
    * leading newlines, asterisks, and tabs and spaces. In the output, it is translated to newlines

core/src/test/java/com/google/googlejavaformat/java/JavadocFormattingTest.java

@@ -1717,6 +1717,60 @@ class Test {}
 ///
 /// A following paragraph.
 class Test {}
+""";
+    doFormatTest(input, expected);
+  }
+
+  @Test
+  public void markdownFencedCodeBlocks() {
+    assume().that(MARKDOWN_JAVADOC_SUPPORTED).isTrue();
+    // If fenced code blocks are not supported correctly, the contents of each one will be joined.
+    // If the input lines survive as separate lines, that means we identified the code block.
+    String input =
+"""
+/// ```
+/// foo
+/// bar
+/// ```
+///
+/// -  ```
+///    code block
+///    in a list
+///    ```
+///
+/// ~~~java
+/// code block
+/// with tildes and an info string ("java")
+/// ~~~
+///
+///  ````
+///  code block
+///  with more than three backticks and an extra leading space
+///  ````
+class Test {}
+""";
+    String expected =
+"""
+/// ```
+/// foo
+/// bar
+/// ```
+///
+/// - ```
+///   code block
+///   in a list
+///   ```
+///
+/// ~~~java
+/// code block
+/// with tildes and an info string ("java")
+/// ~~~
+///
+/// ````
+/// code block
+/// with more than three backticks and an extra leading space
+/// ````
+class Test {}
 """;
     doFormatTest(input, expected);
   }

core/src/test/java/com/google/googlejavaformat/java/javadoc/MarkdownPositionsTest.java

@@ -23,6 +23,7 @@
 import com.google.googlejavaformat.java.javadoc.Token.ListItemCloseTag;
 import com.google.googlejavaformat.java.javadoc.Token.ListItemOpenTag;
 import com.google.googlejavaformat.java.javadoc.Token.ListOpenTag;
+import com.google.googlejavaformat.java.javadoc.Token.MarkdownFencedCodeBlock;
 import com.google.googlejavaformat.java.javadoc.Token.ParagraphCloseTag;
 import com.google.googlejavaformat.java.javadoc.Token.ParagraphOpenTag;
 import java.util.Map;
@@ -100,6 +101,62 @@ public void heading() {
     assertThat(map).isEqualTo(expected);
   }
 
+  @Test
+  public void codeBlock() {
+    String text =
+"""
+- ```
+  foo
+  bar
+  ```
+
+~~~java
+code
+with tildes
+~~~
+
+  ````
+  indented code
+  with more than three backticks
+  ````
+""";
+    var positions = MarkdownPositions.parse(text);
+    ImmutableListMultimap<Integer, Token> map = positionToToken(positions, text);
+    int bullet = text.indexOf('-');
+    int firstCodeStart = text.indexOf("```");
+    int firstCodeEnd = text.indexOf("```", firstCodeStart + 3) + 3;
+    int secondCodeStart = text.indexOf("~~~", firstCodeEnd);
+    int secondCodeEnd = text.indexOf("~~~", secondCodeStart + 3) + 3;
+    int thirdCodeStart = text.indexOf("````", secondCodeEnd);
+    int thirdCodeEnd = text.indexOf("````", thirdCodeStart + 4) + 4;
+    ImmutableListMultimap<Integer, Token> expected =
+        ImmutableListMultimap.<Integer, Token>builder()
+            .put(bullet, new ListOpenTag(""))
+            .put(bullet, new ListItemOpenTag("- "))
+            .put(
+                firstCodeStart,
+                new MarkdownFencedCodeBlock(
+                    text.substring(firstCodeStart, firstCodeEnd), "```", "```", "foo\nbar\n"))
+            .put(firstCodeEnd, new ListItemCloseTag(""))
+            .put(firstCodeEnd, new ListCloseTag(""))
+            .put(
+                secondCodeStart,
+                new MarkdownFencedCodeBlock(
+                    text.substring(secondCodeStart, secondCodeEnd),
+                    "~~~java",
+                    "~~~",
+                    "code\nwith tildes\n"))
+            .put(
+                thirdCodeStart,
+                new MarkdownFencedCodeBlock(
+                    text.substring(thirdCodeStart, thirdCodeEnd),
+                    "````",
+                    "````",
+                    "indented code\nwith more than three backticks\n"))
+            .build();
+    assertThat(map).isEqualTo(expected);
+  }
+
   private static ImmutableListMultimap<Integer, Token> positionToToken(
       MarkdownPositions positions, String input) {
     return IntStream.rangeClosed(0, input.length())