Add temporary replacement for MarkdownRenderer

This includes the change needed in
commonmark/commonmark-java#361

and removes the previous snapshot

Author: kabir

langchain4j-parent/pom.xml

@@ -93,8 +93,7 @@
         <infinispan.version>15.0.0.Final</infinispan.version>
         <kotlin.version>1.9.25</kotlin.version>
         <kotlinx.version>1.9.0</kotlinx.version>
-        <!-- SNAPSHOT including https://github.com/commonmark/commonmark-java/pull/361 -->
-        <commonmark.version>0.24.1-SNAPSHOT</commonmark.version>
+        <commonmark.version>0.24.0</commonmark.version>
     </properties>
 
     <dependencyManagement>

langchain4j/src/main/java/dev/langchain4j/data/document/splitter/MarkdownSectionSplitter.java

@@ -2,6 +2,7 @@
 
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
+import dev.langchain4j.data.document.DefaultDocument;
 import dev.langchain4j.data.document.Document;
 import dev.langchain4j.data.document.DocumentSplitter;
 import dev.langchain4j.data.document.Metadata;
@@ -23,7 +24,6 @@
 import org.commonmark.renderer.markdown.CoreMarkdownNodeRenderer;
 import org.commonmark.renderer.markdown.MarkdownNodeRendererContext;
 import org.commonmark.renderer.markdown.MarkdownNodeRendererFactory;
-import org.commonmark.renderer.markdown.MarkdownRenderer;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -108,10 +108,10 @@ public List<TextSegment> split(Document document) {
         }
 
         MarkdownSplitterContext context = new MarkdownSplitterContext(document.metadata());
-        MarkdownRenderer renderer = MarkdownRenderer.builder()
+        TempMarkdownRenderer renderer = TempMarkdownRenderer.builder()
                 .nodeRendererFactory(new MarkdownSectionSplitterNodeRendererFactory(context))
                 .extensions(EXTENSIONS)
-                .build();
+                .tempBuild();
         // We use the Appendable allowed by the renderer as the hook in.
         // I tried a few other approaches, but this is the only one I can find that works...
         renderer.render(node, context.getBuffer());
@@ -375,7 +375,7 @@ private void addSectionSegments(Header header, String sectionText) {
                 // Document constructor does not like blank text
                 sectionText = emptySectionPlaceholderText;
             }
-            Document document = new Document(sectionText, metadata);
+            Document document = new DefaultDocument(sectionText, metadata);
             document = documentAdjuster.adjust(document);
 
             List<TextSegment> segments = sectionSplitter.split(document);

langchain4j/src/main/java/dev/langchain4j/data/document/splitter/TempMarkdownRenderer.java

@@ -0,0 +1,126 @@
+package dev.langchain4j.data.document.splitter;
+
+import org.commonmark.Extension;
+import org.commonmark.internal.renderer.NodeRendererMap;
+import org.commonmark.node.Node;
+import org.commonmark.renderer.NodeRenderer;
+import org.commonmark.renderer.Renderer;
+import org.commonmark.renderer.markdown.CoreMarkdownNodeRenderer;
+import org.commonmark.renderer.markdown.MarkdownNodeRendererContext;
+import org.commonmark.renderer.markdown.MarkdownNodeRendererFactory;
+import org.commonmark.renderer.markdown.MarkdownRenderer;
+import org.commonmark.renderer.markdown.MarkdownWriter;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Temporary replacement for MarkdownRenderer, including the fix in
+ * https://github.com/commonmark/commonmark-java/pull/361.
+ *
+ * It contains that fix, and adjustments needed to be able to use this replacement.
+ */
+class TempMarkdownRenderer implements Renderer {
+
+    private final List<MarkdownNodeRendererFactory> nodeRendererFactories;
+
+    private TempMarkdownRenderer(Builder builder) {
+        this.nodeRendererFactories = new ArrayList<>(builder.nodeRendererFactories.size() + 1);
+        this.nodeRendererFactories.addAll(builder.nodeRendererFactories);
+        // Add as last. This means clients can override the rendering of core nodes if they want.
+        this.nodeRendererFactories.add(new MarkdownNodeRendererFactory() {
+            @Override
+            public NodeRenderer create(MarkdownNodeRendererContext context) {
+                return new CoreMarkdownNodeRenderer(context);
+            }
+
+            @Override
+            public Set<Character> getSpecialCharacters() {
+                return Set.of();
+            }
+        });
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    @Override
+    public void render(Node node, Appendable output) {
+        RendererContext context = new RendererContext(new MarkdownWriter(output));
+        context.render(node);
+    }
+
+    @Override
+    public String render(Node node) {
+        StringBuilder sb = new StringBuilder();
+        render(node, sb);
+        return sb.toString();
+    }
+
+    static class Builder extends MarkdownRenderer.Builder {
+
+        private final List<MarkdownNodeRendererFactory> nodeRendererFactories = new ArrayList<>();
+
+        public TempMarkdownRenderer tempBuild() {
+            return new TempMarkdownRenderer(this);
+        }
+
+        public Builder nodeRendererFactory(MarkdownNodeRendererFactory nodeRendererFactory) {
+            this.nodeRendererFactories.add(nodeRendererFactory);
+            return this;
+        }
+
+        public Builder extensions(Iterable<? extends Extension> extensions) {
+            for (Extension extension : extensions) {
+                if (extension instanceof MarkdownRenderer.MarkdownRendererExtension) {
+                    MarkdownRenderer.MarkdownRendererExtension markdownRendererExtension = (MarkdownRenderer.MarkdownRendererExtension) extension;
+                    markdownRendererExtension.extend(this);
+                }
+            }
+            return this;
+        }
+    }
+
+    private class RendererContext implements MarkdownNodeRendererContext {
+        private final MarkdownWriter writer;
+        private final NodeRendererMap nodeRendererMap = new NodeRendererMap();
+        private final Set<Character> additionalTextEscapes;
+
+        private RendererContext(MarkdownWriter writer) {
+            // Set fields that are used by interface
+            this.writer = writer;
+            Set<Character> escapes = new HashSet<>();
+            for (MarkdownNodeRendererFactory factory : nodeRendererFactories) {
+                escapes.addAll(factory.getSpecialCharacters());
+            }
+            additionalTextEscapes = Collections.unmodifiableSet(escapes);
+
+            // The first node renderer for a node type "wins". The NodeRendererMap
+            // disallows overwriting.
+            for (MarkdownNodeRendererFactory nodeRendererFactory : nodeRendererFactories) {
+                // Pass in this as context here, which uses the fields set above
+                NodeRenderer nodeRenderer = nodeRendererFactory.create(this);
+                nodeRendererMap.add(nodeRenderer);
+            }
+        }
+
+        @Override
+        public MarkdownWriter getWriter() {
+            return writer;
+        }
+
+        @Override
+        public void render(Node node) {
+            nodeRendererMap.render(node);
+        }
+
+        @Override
+        public Set<Character> getSpecialCharacters() {
+            return additionalTextEscapes;
+        }
+    }
+}