From bce67594596200dc4f7dc482effd1290db6575e2 Mon Sep 17 00:00:00 2001
From: Taeik Lim
Date: Wed, 10 Aug 2022 22:40:16 +0900
Subject: [PATCH 001/286] README: Bump version
---
README.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/README.md b/README.md
index 2bb15b16e..3e7546be2 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ Coordinates for core library (see all on [Maven Central]):
org.commonmarkcommonmark
- 0.18.1
+ 0.19.0
```
@@ -233,7 +233,7 @@ First, add an additional dependency (see [Maven Central] for others):
org.commonmarkcommonmark-ext-gfm-tables
- 0.18.1
+ 0.19.0
```
From 7584f008900752cd9a539d2ac0d216b848eb2a4b Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 18 Oct 2022 18:29:18 +1100
Subject: [PATCH 002/286] Fix GFM table parser to handle dangling pipe
correctly (fixes #255)
---
CHANGELOG.md | 6 ++
.../gfm/tables/internal/TableBlockParser.java | 18 ++++-
.../commonmark/ext/gfm/tables/TablesTest.java | 73 +++++++++++++++----
3 files changed, 78 insertions(+), 19 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cdfb65043..9a8f1b2ca 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html),
with the exception that 0.x versions can break between minor versions.
+## [Unreleased]
+### Fixed
+- A single pipe (optional whitespace) now ends a table instead of crashing or
+ being treated as an empty row, for consistency with GitHub (#255).
+
## [0.19.0] - 2022-06-02
### Added
- YAML front matter extension: Limited support for single and double
@@ -362,6 +367,7 @@ Initial release of commonmark-java, a port of commonmark.js with extensions
for autolinking URLs, GitHub flavored strikethrough and tables.
+[Unreleased]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.19.0...HEAD
[0.19.0]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.18.2...commonmark-parent-0.19.0
[0.18.2]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.18.1...commonmark-parent-0.18.2
[0.18.1]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.18.0...commonmark-parent-0.18.1
diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java
index a203164d3..b7cea14db 100644
--- a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java
+++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java
@@ -19,6 +19,8 @@ public class TableBlockParser extends AbstractBlockParser {
private final List rowLines = new ArrayList<>();
private final List columns;
+ private boolean canHaveLazyContinuationLines = true;
+
private TableBlockParser(List columns, SourceLine headerLine) {
this.columns = columns;
this.rowLines.add(headerLine);
@@ -26,7 +28,7 @@ private TableBlockParser(List columns, SourceLine headerLin
@Override
public boolean canHaveLazyContinuationLines() {
- return true;
+ return canHaveLazyContinuationLines;
}
@Override
@@ -36,7 +38,17 @@ public Block getBlock() {
@Override
public BlockContinue tryContinue(ParserState state) {
- if (Parsing.find('|', state.getLine().getContent(), 0) != -1) {
+ CharSequence content = state.getLine().getContent();
+ int pipe = Parsing.find('|', content, state.getNextNonSpaceIndex());
+ if (pipe != -1) {
+ if (pipe == state.getNextNonSpaceIndex()) {
+ // If we *only* have a pipe character (and whitespace), that is not a valid table row and ends the table.
+ if (Parsing.skipSpaceTab(content, pipe + 1, content.length()) == content.length()) {
+ // We also don't want the pipe to be added via lazy continuation.
+ canHaveLazyContinuationLines = false;
+ return BlockContinue.none();
+ }
+ }
return BlockContinue.atIndex(state.getIndex());
} else {
return BlockContinue.none();
@@ -128,7 +140,7 @@ private static List split(SourceLine line) {
// This row has leading/trailing pipes - skip the leading pipe
cellStart = nonSpace + 1;
// Strip whitespace from the end but not the pipe or we could miss an empty ("||") cell
- int nonSpaceEnd = Parsing.skipSpaceTabBackwards(row, row.length() - 1, cellStart + 1);
+ int nonSpaceEnd = Parsing.skipSpaceTabBackwards(row, row.length() - 1, cellStart);
cellEnd = nonSpaceEnd + 1;
}
List cells = new ArrayList<>();
diff --git a/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesTest.java b/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesTest.java
index 4d5153213..bef3b8b6c 100644
--- a/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesTest.java
+++ b/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesTest.java
@@ -261,21 +261,21 @@ public void pipesOnOutsideZeroLengthHeaders() {
"-|-------------|-\n" +
"1| 2 |3",
"
\n");
+ }
+
@Test
public void attributeProviderIsApplied() {
AttributeProviderFactory factory = new AttributeProviderFactory() {
@@ -718,7 +759,7 @@ public void sourceSpans() {
TableBlock block = (TableBlock) document.getFirstChild();
assertEquals(Arrays.asList(SourceSpan.of(0, 0, 7), SourceSpan.of(1, 0, 7),
- SourceSpan.of(2, 0, 4), SourceSpan.of(3, 0, 8), SourceSpan.of(4, 0, 3)),
+ SourceSpan.of(2, 0, 4), SourceSpan.of(3, 0, 8), SourceSpan.of(4, 0, 3)),
block.getSourceSpans());
TableHead head = (TableHead) block.getFirstChild();
From afb31c80deb5fc0fbf15b62ef2093ad12e165c41 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 19 Oct 2022 15:38:23 +1100
Subject: [PATCH 003/286] Move GFM spec out of tables ext, test strikethrough
as well
All tests passing already (there's not many).
---
.../strikethrough/StrikethroughSpecTest.java | 46 +++++++++++++++++++
.../ext/gfm/tables/TablesSpecTest.java | 9 +---
.../commonmark/testutil/TestResources.java | 4 ++
.../testutil/example/ExampleReader.java | 11 +++++
.../src/main}/resources/gfm-spec.txt | 3 +-
etc/update-spec.sh | 2 +-
6 files changed, 64 insertions(+), 11 deletions(-)
create mode 100644 commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughSpecTest.java
rename {commonmark-ext-gfm-tables/src/test => commonmark-test-util/src/main}/resources/gfm-spec.txt (99%)
diff --git a/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughSpecTest.java b/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughSpecTest.java
new file mode 100644
index 000000000..4b907cf41
--- /dev/null
+++ b/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughSpecTest.java
@@ -0,0 +1,46 @@
+package org.commonmark.ext.gfm.strikethrough;
+
+import org.commonmark.Extension;
+import org.commonmark.parser.Parser;
+import org.commonmark.renderer.html.HtmlRenderer;
+import org.commonmark.testutil.RenderingTestCase;
+import org.commonmark.testutil.TestResources;
+import org.commonmark.testutil.example.Example;
+import org.commonmark.testutil.example.ExampleReader;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+@RunWith(Parameterized.class)
+public class StrikethroughSpecTest extends RenderingTestCase {
+
+ private static final Set EXTENSIONS = Collections.singleton(StrikethroughExtension.create());
+ private static final Parser PARSER = Parser.builder().extensions(EXTENSIONS).build();
+ private static final HtmlRenderer RENDERER = HtmlRenderer.builder().extensions(EXTENSIONS).build();
+
+ private final Example example;
+
+ public StrikethroughSpecTest(Example example) {
+ this.example = example;
+ }
+
+ @Parameters(name = "{0}")
+ public static List
*/
public class StrikethroughExtension implements Parser.ParserExtension, HtmlRenderer.HtmlRendererExtension,
TextContentRenderer.TextContentRendererExtension {
- private StrikethroughExtension() {
+ private final boolean requireTwoTildes;
+
+ private StrikethroughExtension(Builder builder) {
+ this.requireTwoTildes = builder.requireTwoTildes;
}
+ /**
+ * @return the extension with default options
+ */
public static Extension create() {
- return new StrikethroughExtension();
+ return builder().build();
+ }
+
+ /**
+ * @return a builder to configure the behavior of the extension
+ */
+ public static Builder builder() {
+ return new Builder();
}
@Override
public void extend(Parser.Builder parserBuilder) {
- parserBuilder.customDelimiterProcessor(new StrikethroughDelimiterProcessor());
+ parserBuilder.customDelimiterProcessor(new StrikethroughDelimiterProcessor(requireTwoTildes));
}
@Override
@@ -58,4 +86,26 @@ public NodeRenderer create(TextContentNodeRendererContext context) {
}
});
}
+
+ public static class Builder {
+
+ private boolean requireTwoTildes = false;
+
+ /**
+ * @param requireTwoTildes Whether two tilde characters ({@code ~~}) are required for strikethrough or whether
+ * one is also enough. Default is {@code false}; both a single tilde and two tildes can be used for strikethrough.
+ * @return {@code this}
+ */
+ public Builder requireTwoTildes(boolean requireTwoTildes) {
+ this.requireTwoTildes = requireTwoTildes;
+ return this;
+ }
+
+ /**
+ * @return a configured extension
+ */
+ public Extension build() {
+ return new StrikethroughExtension(this);
+ }
+ }
}
diff --git a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java
index a26953d28..3dedff1b9 100644
--- a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java
+++ b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java
@@ -10,6 +10,16 @@
public class StrikethroughDelimiterProcessor implements DelimiterProcessor {
+ private final boolean requireTwoTildes;
+
+ public StrikethroughDelimiterProcessor() {
+ this(false);
+ }
+
+ public StrikethroughDelimiterProcessor(boolean requireTwoTildes) {
+ this.requireTwoTildes = requireTwoTildes;
+ }
+
@Override
public char getOpeningCharacter() {
return '~';
@@ -22,7 +32,7 @@ public char getClosingCharacter() {
@Override
public int getMinLength() {
- return 1;
+ return requireTwoTildes ? 2 : 1;
}
@Override
diff --git a/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughTest.java b/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughTest.java
index 76cf929e9..d8a754c72 100644
--- a/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughTest.java
+++ b/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughTest.java
@@ -4,22 +4,25 @@
import org.commonmark.node.Node;
import org.commonmark.node.Paragraph;
import org.commonmark.node.SourceSpan;
+import org.commonmark.node.Text;
import org.commonmark.parser.IncludeSourceSpans;
import org.commonmark.parser.Parser;
+import org.commonmark.parser.delimiter.DelimiterProcessor;
+import org.commonmark.parser.delimiter.DelimiterRun;
import org.commonmark.renderer.html.HtmlRenderer;
import org.commonmark.renderer.text.TextContentRenderer;
import org.commonmark.testutil.RenderingTestCase;
import org.junit.Test;
import java.util.Arrays;
-import java.util.Collections;
import java.util.Set;
+import static java.util.Collections.singleton;
import static org.junit.Assert.assertEquals;
public class StrikethroughTest extends RenderingTestCase {
- private static final Set EXTENSIONS = Collections.singleton(StrikethroughExtension.create());
+ private static final Set EXTENSIONS = singleton(StrikethroughExtension.create());
private static final Parser PARSER = Parser.builder().extensions(EXTENSIONS).build();
private static final HtmlRenderer HTML_RENDERER = HtmlRenderer.builder().extensions(EXTENSIONS).build();
private static final TextContentRenderer CONTENT_RENDERER = TextContentRenderer.builder()
@@ -92,6 +95,19 @@ public void textContentRenderer() {
assertEquals("/foo/", CONTENT_RENDERER.render(document));
}
+ @Test
+ public void requireTwoTildesOption() {
+ Parser parser = Parser.builder()
+ .extensions(singleton(StrikethroughExtension.builder()
+ .requireTwoTildes(true)
+ .build()))
+ .customDelimiterProcessor(new SubscriptDelimiterProcessor())
+ .build();
+
+ Node document = parser.parse("~foo~ ~~bar~~");
+ assertEquals("(sub)foo(/sub) /bar/", CONTENT_RENDERER.render(document));
+ }
+
@Test
public void sourceSpans() {
Parser parser = Parser.builder()
@@ -110,4 +126,29 @@ public void sourceSpans() {
protected String render(String source) {
return HTML_RENDERER.render(PARSER.parse(source));
}
+
+ private static class SubscriptDelimiterProcessor implements DelimiterProcessor {
+
+ @Override
+ public char getOpeningCharacter() {
+ return '~';
+ }
+
+ @Override
+ public char getClosingCharacter() {
+ return '~';
+ }
+
+ @Override
+ public int getMinLength() {
+ return 1;
+ }
+
+ @Override
+ public int process(DelimiterRun openingRun, DelimiterRun closingRun) {
+ openingRun.getOpener().insertAfter(new Text("(sub)"));
+ closingRun.getCloser().insertBefore(new Text("(/sub)"));
+ return 1;
+ }
+ }
}
diff --git a/commonmark/src/main/java/org/commonmark/internal/StaggeredDelimiterProcessor.java b/commonmark/src/main/java/org/commonmark/internal/StaggeredDelimiterProcessor.java
index 0fe8065bb..2836e346a 100644
--- a/commonmark/src/main/java/org/commonmark/internal/StaggeredDelimiterProcessor.java
+++ b/commonmark/src/main/java/org/commonmark/internal/StaggeredDelimiterProcessor.java
@@ -51,7 +51,7 @@ void add(DelimiterProcessor dp) {
added = true;
break;
} else if (len == pLen) {
- throw new IllegalArgumentException("Cannot add two delimiter processors for char '" + delim + "' and minimum length " + len);
+ throw new IllegalArgumentException("Cannot add two delimiter processors for char '" + delim + "' and minimum length " + len + "; conflicting processors: " + p + ", " + dp);
}
}
if (!added) {
diff --git a/commonmark/src/test/java/org/commonmark/test/DelimiterProcessorTest.java b/commonmark/src/test/java/org/commonmark/test/DelimiterProcessorTest.java
index 173d711d3..d2e20a64f 100644
--- a/commonmark/src/test/java/org/commonmark/test/DelimiterProcessorTest.java
+++ b/commonmark/src/test/java/org/commonmark/test/DelimiterProcessorTest.java
@@ -60,7 +60,7 @@ public void multipleDelimitersWithDifferentLengths() {
}
@Test(expected = IllegalArgumentException.class)
- public void multipleDelimitersWithSameLength() {
+ public void multipleDelimitersWithSameLengthConflict() {
Parser.builder()
.customDelimiterProcessor(new OneDelimiterProcessor())
.customDelimiterProcessor(new OneDelimiterProcessor())
From de1c59eccb38e6e2359a023dcffbe6507e0380a4 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Mon, 24 Oct 2022 17:13:01 +1100
Subject: [PATCH 011/286] Tweak Javadoc (meh)
---
.../ext/gfm/strikethrough/StrikethroughExtension.java | 2 ++
1 file changed, 2 insertions(+)
diff --git a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/StrikethroughExtension.java b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/StrikethroughExtension.java
index 7d277fd73..4f0228a1c 100644
--- a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/StrikethroughExtension.java
+++ b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/StrikethroughExtension.java
@@ -30,11 +30,13 @@
*
* If you have another extension that only uses a single tilde ({@code ~}) syntax, you will have to configure this
* {@link StrikethroughExtension} to only accept the double tilde syntax, like this:
+ *
* If you don't do that, there's a conflict between the two extensions and you will get an
* {@link IllegalArgumentException} when constructing the parser.
*
From 77c58536ab780a8ce582136206672f480e72c57a Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 3 Nov 2022 13:59:33 +1100
Subject: [PATCH 012/286] Add DingusApp for quickly testing out renderings
Named Dingus because of https://spec.commonmark.org/dingus/
---
.../java/org/commonmark/ui/DingusApp.java | 114 ++++++++++++++++++
1 file changed, 114 insertions(+)
create mode 100644 commonmark-integration-test/src/test/java/org/commonmark/ui/DingusApp.java
diff --git a/commonmark-integration-test/src/test/java/org/commonmark/ui/DingusApp.java b/commonmark-integration-test/src/test/java/org/commonmark/ui/DingusApp.java
new file mode 100644
index 000000000..0e98386bb
--- /dev/null
+++ b/commonmark-integration-test/src/test/java/org/commonmark/ui/DingusApp.java
@@ -0,0 +1,114 @@
+package org.commonmark.ui;
+
+import org.commonmark.parser.Parser;
+import org.commonmark.renderer.html.HtmlRenderer;
+import org.commonmark.renderer.text.TextContentRenderer;
+
+import java.awt.*;
+import javax.swing.*;
+import javax.swing.event.ChangeEvent;
+import javax.swing.event.ChangeListener;
+import javax.swing.event.DocumentEvent;
+import javax.swing.event.DocumentListener;
+
+/**
+ * Simple UI to quickly test out different rendering of CommonMark inputs.
+ * Similar to commonmark.js dingus.
+ **/
+public class DingusApp {
+
+ private final Parser parser = Parser.builder().build();
+ private final TextContentRenderer textRenderer = TextContentRenderer.builder().build();
+ private final HtmlRenderer htmlRenderer = HtmlRenderer.builder().build();
+
+ private final JTabbedPane tabbedPane;
+ private final JEditorPane htmlVisualRendererOutput;
+ private final JTextArea htmlSourceRendererOutput;
+ private final JTextArea textRendererOutput;
+
+ public static void main(String[] args) {
+ new DingusApp().run();
+ }
+
+ private DingusApp() {
+ tabbedPane = new JTabbedPane();
+
+ htmlVisualRendererOutput = new JEditorPane();
+ htmlVisualRendererOutput.setEnabled(false);
+ htmlVisualRendererOutput.setContentType("text/html");
+
+ htmlSourceRendererOutput = new JTextArea();
+ htmlSourceRendererOutput.setEnabled(false);
+ htmlSourceRendererOutput.setLineWrap(true);
+ htmlSourceRendererOutput.setFont(new Font(Font.MONOSPACED, Font.PLAIN, 12));
+
+ textRendererOutput = new JTextArea();
+ textRendererOutput.setEnabled(false);
+ textRendererOutput.setLineWrap(true);
+ textRendererOutput.setFont(new Font(Font.MONOSPACED, Font.PLAIN, 12));
+ }
+
+ private void run() {
+ JFrame frame = new JFrame("commonmark-java dingus");
+ frame.setDefaultCloseOperation(WindowConstants.EXIT_ON_CLOSE);
+ frame.setMinimumSize(new Dimension(400, 300));
+ frame.setSize(new Dimension(1200, 675));
+
+ final JTextArea input = new JTextArea();
+ input.setBorder(BorderFactory.createTitledBorder("Input"));
+ input.setLineWrap(true);
+ input.setFont(new Font(Font.MONOSPACED, Font.PLAIN, 12));
+
+ input.getDocument().addDocumentListener(new DocumentListener() {
+ @Override
+ public void insertUpdate(DocumentEvent e) {
+ updateOutput(input.getText());
+ }
+
+ @Override
+ public void removeUpdate(DocumentEvent e) {
+ updateOutput(input.getText());
+ }
+
+ @Override
+ public void changedUpdate(DocumentEvent e) {
+ }
+ });
+
+ tabbedPane.addTab("HTML rendered", htmlVisualRendererOutput);
+ tabbedPane.addTab("HTML source", htmlSourceRendererOutput);
+ tabbedPane.addTab("Plain text", textRendererOutput);
+
+ tabbedPane.addChangeListener(new ChangeListener() {
+ @Override
+ public void stateChanged(ChangeEvent e) {
+ updateOutput(input.getText());
+ }
+ });
+
+ input.setText("# Example\n" +
+ "Enter text *here* and see how it renders on the right.\n\n" +
+ "* Try\n* this\n\n" +
+ "```\nor this\n```");
+ updateOutput(input.getText());
+
+ frame.setLayout(new GridLayout());
+ frame.add(input);
+ frame.add(tabbedPane);
+
+ frame.setVisible(true);
+ }
+
+ private void updateOutput(String inputText) {
+ if (tabbedPane.getSelectedComponent() == htmlVisualRendererOutput) {
+ String rendered = htmlRenderer.render(parser.parse(inputText));
+ htmlVisualRendererOutput.setText(rendered);
+ } else if (tabbedPane.getSelectedComponent() == htmlSourceRendererOutput) {
+ String rendered = htmlRenderer.render(parser.parse(inputText));
+ htmlSourceRendererOutput.setText(rendered);
+ } else if (tabbedPane.getSelectedComponent() == textRendererOutput) {
+ String rendered = textRenderer.render(parser.parse(inputText));
+ textRendererOutput.setText(rendered);
+ }
+ }
+}
From 8636cf7f398c8a80c595037f44efa84c6e86758e Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 3 Nov 2022 14:35:28 +1100
Subject: [PATCH 013/286] Mention DingusApp
---
README.md | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/README.md b/README.md
index 31a7780da..bcf552c5b 100644
--- a/README.md
+++ b/README.md
@@ -48,7 +48,8 @@ be followed.
See the [spec.txt](commonmark-test-util/src/main/resources/spec.txt)
file if you're wondering which version of the spec is currently
implemented. Also check out the [CommonMark dingus] for getting familiar
-with the syntax or trying out edge cases.
+with the syntax or trying out edge cases. If you clone the repository,
+you can also use the `DingusApp` class to try out things interactively.
Usage
From 6bf916e1fa00ae7be9a692c5821bc91f0eba8957 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 10 Nov 2022 21:05:10 +1100
Subject: [PATCH 014/286] Bump maven plugins
---
pom.xml | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/pom.xml b/pom.xml
index 265b5f2d4..3cf35692e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -38,7 +38,7 @@
org.apache.maven.pluginsmaven-compiler-plugin
- 3.8.1
+ 3.10.177
@@ -47,17 +47,17 @@
org.apache.maven.pluginsmaven-jar-plugin
- 3.2.0
+ 3.3.0org.apache.maven.pluginsmaven-install-plugin
- 3.0.0-M1
+ 3.0.1org.apache.maven.pluginsmaven-javadoc-plugin
- 3.2.0
+ 3.4.1*.internal,*.internal.*
@@ -74,7 +74,7 @@
org.apache.maven.pluginsmaven-surefire-plugin
- 2.22.1
+ 2.22.2
@@ -83,7 +83,7 @@
org.sonatype.pluginsnexus-staging-maven-plugin
- 1.6.8
+ 1.6.13trueossrh
@@ -95,7 +95,7 @@
org.apache.maven.pluginsmaven-release-plugin
- 3.0.0-M1
+ 3.0.0-M7truefalse
@@ -212,7 +212,7 @@
org.apache.maven.pluginsmaven-gpg-plugin
- 1.6
+ 3.0.1sign-artifacts
@@ -239,7 +239,7 @@
org.jacocojacoco-maven-plugin
- 0.7.9
+ 0.8.8
From e146e4203324908de1b3797191b790d82b1ee679 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 17 Nov 2022 13:33:03 +1100
Subject: [PATCH 015/286] Prepare for version 0.21.0
Set the version using `mvn versions:set -DnewVersion=0.21.0-SNAPSHOT`
---
CHANGELOG.md | 4 ++--
commonmark-ext-autolink/pom.xml | 2 +-
commonmark-ext-gfm-strikethrough/pom.xml | 2 +-
commonmark-ext-gfm-tables/pom.xml | 2 +-
commonmark-ext-heading-anchor/pom.xml | 2 +-
commonmark-ext-image-attributes/pom.xml | 2 +-
commonmark-ext-ins/pom.xml | 2 +-
commonmark-ext-task-list-items/pom.xml | 2 +-
commonmark-ext-yaml-front-matter/pom.xml | 2 +-
commonmark-integration-test/pom.xml | 2 +-
commonmark-test-util/pom.xml | 2 +-
commonmark/pom.xml | 2 +-
pom.xml | 22 +++++++++++-----------
13 files changed, 24 insertions(+), 24 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 49ebab528..f558a3601 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html),
with the exception that 0.x versions can break between minor versions.
-## [Unreleased]
+## [0.21.0] - 2022-11-17
### Added
- GitHub strikethrough: With the previous version we adjusted the
extension to also accept the single tilde syntax. But if you use
@@ -379,7 +379,7 @@ API breaking changes (caused by changes in spec):
Initial release of commonmark-java, a port of commonmark.js with extensions
for autolinking URLs, GitHub flavored strikethrough and tables.
-[Unreleased]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.20.0...HEAD
+[0.21.0]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.20.0...commonmark-parent-0.21.0
[0.20.0]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.19.0...commonmark-parent-0.20.0
[0.19.0]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.18.2...commonmark-parent-0.19.0
[0.18.2]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.18.1...commonmark-parent-0.18.2
diff --git a/commonmark-ext-autolink/pom.xml b/commonmark-ext-autolink/pom.xml
index 43ebf31b1..c95d71383 100644
--- a/commonmark-ext-autolink/pom.xml
+++ b/commonmark-ext-autolink/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTcommonmark-ext-autolink
diff --git a/commonmark-ext-gfm-strikethrough/pom.xml b/commonmark-ext-gfm-strikethrough/pom.xml
index 3212541d0..41e358b08 100644
--- a/commonmark-ext-gfm-strikethrough/pom.xml
+++ b/commonmark-ext-gfm-strikethrough/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTcommonmark-ext-gfm-strikethrough
diff --git a/commonmark-ext-gfm-tables/pom.xml b/commonmark-ext-gfm-tables/pom.xml
index e017ab05d..940e491ee 100644
--- a/commonmark-ext-gfm-tables/pom.xml
+++ b/commonmark-ext-gfm-tables/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTcommonmark-ext-gfm-tables
diff --git a/commonmark-ext-heading-anchor/pom.xml b/commonmark-ext-heading-anchor/pom.xml
index 350fed676..40c8c91a0 100644
--- a/commonmark-ext-heading-anchor/pom.xml
+++ b/commonmark-ext-heading-anchor/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTcommonmark-ext-heading-anchor
diff --git a/commonmark-ext-image-attributes/pom.xml b/commonmark-ext-image-attributes/pom.xml
index b18b9d5ab..58ff6af45 100644
--- a/commonmark-ext-image-attributes/pom.xml
+++ b/commonmark-ext-image-attributes/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTcommonmark-ext-image-attributes
diff --git a/commonmark-ext-ins/pom.xml b/commonmark-ext-ins/pom.xml
index edacbd87c..475398615 100644
--- a/commonmark-ext-ins/pom.xml
+++ b/commonmark-ext-ins/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTcommonmark-ext-ins
diff --git a/commonmark-ext-task-list-items/pom.xml b/commonmark-ext-task-list-items/pom.xml
index cff7f974c..c56b00dbb 100644
--- a/commonmark-ext-task-list-items/pom.xml
+++ b/commonmark-ext-task-list-items/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTcommonmark-ext-task-list-items
diff --git a/commonmark-ext-yaml-front-matter/pom.xml b/commonmark-ext-yaml-front-matter/pom.xml
index 8337da4f5..57aa1fbd0 100644
--- a/commonmark-ext-yaml-front-matter/pom.xml
+++ b/commonmark-ext-yaml-front-matter/pom.xml
@@ -4,7 +4,7 @@
commonmark-parentorg.commonmark
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTcommonmark-ext-yaml-front-matter
diff --git a/commonmark-integration-test/pom.xml b/commonmark-integration-test/pom.xml
index a208f4de6..0745fe2d9 100644
--- a/commonmark-integration-test/pom.xml
+++ b/commonmark-integration-test/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTcommonmark-integration-test
diff --git a/commonmark-test-util/pom.xml b/commonmark-test-util/pom.xml
index 716d5243e..a9700153e 100644
--- a/commonmark-test-util/pom.xml
+++ b/commonmark-test-util/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTcommonmark-test-util
diff --git a/commonmark/pom.xml b/commonmark/pom.xml
index dc35608e4..a4b3368e2 100644
--- a/commonmark/pom.xml
+++ b/commonmark/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTcommonmark
diff --git a/pom.xml b/pom.xml
index 3cf35692e..1b8c1e82e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
org.commonmarkcommonmark-parent
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTcommonmark-java parent
Java implementation of CommonMark, a specification of the Markdown format for turning plain text into formatted
@@ -112,52 +112,52 @@
org.commonmarkcommonmark
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTorg.commonmarkcommonmark-ext-autolink
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTorg.commonmarkcommonmark-ext-image-attributes
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTorg.commonmarkcommonmark-ext-ins
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTorg.commonmarkcommonmark-ext-gfm-strikethrough
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTorg.commonmarkcommonmark-ext-gfm-tables
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTorg.commonmarkcommonmark-ext-heading-anchor
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTorg.commonmarkcommonmark-ext-task-list-items
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTorg.commonmarkcommonmark-ext-yaml-front-matter
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOTorg.commonmarkcommonmark-test-util
- 0.20.1-SNAPSHOT
+ 0.21.0-SNAPSHOT
From db8469c78cdbec6a5cb0375e5fce4844d9ad57d7 Mon Sep 17 00:00:00 2001
From: "Robin Stocker (GitHub Actions)"
<16778+robinst@users.noreply.github.com>
Date: Thu, 17 Nov 2022 02:37:39 +0000
Subject: [PATCH 016/286] [maven-release-plugin] prepare release
commonmark-parent-0.21.0
---
commonmark-ext-autolink/pom.xml | 2 +-
commonmark-ext-gfm-strikethrough/pom.xml | 2 +-
commonmark-ext-gfm-tables/pom.xml | 2 +-
commonmark-ext-heading-anchor/pom.xml | 2 +-
commonmark-ext-image-attributes/pom.xml | 2 +-
commonmark-ext-ins/pom.xml | 2 +-
commonmark-ext-task-list-items/pom.xml | 2 +-
commonmark-ext-yaml-front-matter/pom.xml | 2 +-
commonmark-integration-test/pom.xml | 2 +-
commonmark-test-util/pom.xml | 2 +-
commonmark/pom.xml | 2 +-
pom.xml | 24 ++++++++++++------------
12 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/commonmark-ext-autolink/pom.xml b/commonmark-ext-autolink/pom.xml
index c95d71383..3558b2231 100644
--- a/commonmark-ext-autolink/pom.xml
+++ b/commonmark-ext-autolink/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0-SNAPSHOT
+ 0.21.0commonmark-ext-autolink
diff --git a/commonmark-ext-gfm-strikethrough/pom.xml b/commonmark-ext-gfm-strikethrough/pom.xml
index 41e358b08..a1eb8f466 100644
--- a/commonmark-ext-gfm-strikethrough/pom.xml
+++ b/commonmark-ext-gfm-strikethrough/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0-SNAPSHOT
+ 0.21.0commonmark-ext-gfm-strikethrough
diff --git a/commonmark-ext-gfm-tables/pom.xml b/commonmark-ext-gfm-tables/pom.xml
index 940e491ee..adbc14e94 100644
--- a/commonmark-ext-gfm-tables/pom.xml
+++ b/commonmark-ext-gfm-tables/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0-SNAPSHOT
+ 0.21.0commonmark-ext-gfm-tables
diff --git a/commonmark-ext-heading-anchor/pom.xml b/commonmark-ext-heading-anchor/pom.xml
index 40c8c91a0..33dfa985c 100644
--- a/commonmark-ext-heading-anchor/pom.xml
+++ b/commonmark-ext-heading-anchor/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0-SNAPSHOT
+ 0.21.0commonmark-ext-heading-anchor
diff --git a/commonmark-ext-image-attributes/pom.xml b/commonmark-ext-image-attributes/pom.xml
index 58ff6af45..307597da9 100644
--- a/commonmark-ext-image-attributes/pom.xml
+++ b/commonmark-ext-image-attributes/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0-SNAPSHOT
+ 0.21.0commonmark-ext-image-attributes
diff --git a/commonmark-ext-ins/pom.xml b/commonmark-ext-ins/pom.xml
index 475398615..f9e095ea7 100644
--- a/commonmark-ext-ins/pom.xml
+++ b/commonmark-ext-ins/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0-SNAPSHOT
+ 0.21.0commonmark-ext-ins
diff --git a/commonmark-ext-task-list-items/pom.xml b/commonmark-ext-task-list-items/pom.xml
index c56b00dbb..772088d4a 100644
--- a/commonmark-ext-task-list-items/pom.xml
+++ b/commonmark-ext-task-list-items/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0-SNAPSHOT
+ 0.21.0commonmark-ext-task-list-items
diff --git a/commonmark-ext-yaml-front-matter/pom.xml b/commonmark-ext-yaml-front-matter/pom.xml
index 57aa1fbd0..9fd018c9a 100644
--- a/commonmark-ext-yaml-front-matter/pom.xml
+++ b/commonmark-ext-yaml-front-matter/pom.xml
@@ -4,7 +4,7 @@
commonmark-parentorg.commonmark
- 0.21.0-SNAPSHOT
+ 0.21.0commonmark-ext-yaml-front-matter
diff --git a/commonmark-integration-test/pom.xml b/commonmark-integration-test/pom.xml
index 0745fe2d9..2646e7456 100644
--- a/commonmark-integration-test/pom.xml
+++ b/commonmark-integration-test/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0-SNAPSHOT
+ 0.21.0commonmark-integration-test
diff --git a/commonmark-test-util/pom.xml b/commonmark-test-util/pom.xml
index a9700153e..6c43eb2ab 100644
--- a/commonmark-test-util/pom.xml
+++ b/commonmark-test-util/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0-SNAPSHOT
+ 0.21.0commonmark-test-util
diff --git a/commonmark/pom.xml b/commonmark/pom.xml
index a4b3368e2..5b69ddcf0 100644
--- a/commonmark/pom.xml
+++ b/commonmark/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0-SNAPSHOT
+ 0.21.0commonmark
diff --git a/pom.xml b/pom.xml
index 1b8c1e82e..f44082810 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
org.commonmarkcommonmark-parent
- 0.21.0-SNAPSHOT
+ 0.21.0commonmark-java parent
Java implementation of CommonMark, a specification of the Markdown format for turning plain text into formatted
@@ -112,52 +112,52 @@
org.commonmarkcommonmark
- 0.21.0-SNAPSHOT
+ 0.21.0org.commonmarkcommonmark-ext-autolink
- 0.21.0-SNAPSHOT
+ 0.21.0org.commonmarkcommonmark-ext-image-attributes
- 0.21.0-SNAPSHOT
+ 0.21.0org.commonmarkcommonmark-ext-ins
- 0.21.0-SNAPSHOT
+ 0.21.0org.commonmarkcommonmark-ext-gfm-strikethrough
- 0.21.0-SNAPSHOT
+ 0.21.0org.commonmarkcommonmark-ext-gfm-tables
- 0.21.0-SNAPSHOT
+ 0.21.0org.commonmarkcommonmark-ext-heading-anchor
- 0.21.0-SNAPSHOT
+ 0.21.0org.commonmarkcommonmark-ext-task-list-items
- 0.21.0-SNAPSHOT
+ 0.21.0org.commonmarkcommonmark-ext-yaml-front-matter
- 0.21.0-SNAPSHOT
+ 0.21.0org.commonmarkcommonmark-test-util
- 0.21.0-SNAPSHOT
+ 0.21.0
@@ -282,7 +282,7 @@
scm:git:https://github.com/commonmark/commonmark-javascm:git:https://github.com/commonmark/commonmark-javahttps://github.com/commonmark/commonmark-java
- HEAD
+ commonmark-parent-0.21.0
From dfd95d08a547d91bf10a6dea3d030906a5f976b7 Mon Sep 17 00:00:00 2001
From: "Robin Stocker (GitHub Actions)"
<16778+robinst@users.noreply.github.com>
Date: Thu, 17 Nov 2022 02:37:40 +0000
Subject: [PATCH 017/286] [maven-release-plugin] prepare for next development
iteration
---
commonmark-ext-autolink/pom.xml | 2 +-
commonmark-ext-gfm-strikethrough/pom.xml | 2 +-
commonmark-ext-gfm-tables/pom.xml | 2 +-
commonmark-ext-heading-anchor/pom.xml | 2 +-
commonmark-ext-image-attributes/pom.xml | 2 +-
commonmark-ext-ins/pom.xml | 2 +-
commonmark-ext-task-list-items/pom.xml | 2 +-
commonmark-ext-yaml-front-matter/pom.xml | 2 +-
commonmark-integration-test/pom.xml | 2 +-
commonmark-test-util/pom.xml | 2 +-
commonmark/pom.xml | 2 +-
pom.xml | 24 ++++++++++++------------
12 files changed, 23 insertions(+), 23 deletions(-)
diff --git a/commonmark-ext-autolink/pom.xml b/commonmark-ext-autolink/pom.xml
index 3558b2231..c82ecdae8 100644
--- a/commonmark-ext-autolink/pom.xml
+++ b/commonmark-ext-autolink/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0
+ 0.21.1-SNAPSHOTcommonmark-ext-autolink
diff --git a/commonmark-ext-gfm-strikethrough/pom.xml b/commonmark-ext-gfm-strikethrough/pom.xml
index a1eb8f466..878a1e586 100644
--- a/commonmark-ext-gfm-strikethrough/pom.xml
+++ b/commonmark-ext-gfm-strikethrough/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0
+ 0.21.1-SNAPSHOTcommonmark-ext-gfm-strikethrough
diff --git a/commonmark-ext-gfm-tables/pom.xml b/commonmark-ext-gfm-tables/pom.xml
index adbc14e94..c6448889f 100644
--- a/commonmark-ext-gfm-tables/pom.xml
+++ b/commonmark-ext-gfm-tables/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0
+ 0.21.1-SNAPSHOTcommonmark-ext-gfm-tables
diff --git a/commonmark-ext-heading-anchor/pom.xml b/commonmark-ext-heading-anchor/pom.xml
index 33dfa985c..49bc4a032 100644
--- a/commonmark-ext-heading-anchor/pom.xml
+++ b/commonmark-ext-heading-anchor/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0
+ 0.21.1-SNAPSHOTcommonmark-ext-heading-anchor
diff --git a/commonmark-ext-image-attributes/pom.xml b/commonmark-ext-image-attributes/pom.xml
index 307597da9..959b1406c 100644
--- a/commonmark-ext-image-attributes/pom.xml
+++ b/commonmark-ext-image-attributes/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0
+ 0.21.1-SNAPSHOTcommonmark-ext-image-attributes
diff --git a/commonmark-ext-ins/pom.xml b/commonmark-ext-ins/pom.xml
index f9e095ea7..708532472 100644
--- a/commonmark-ext-ins/pom.xml
+++ b/commonmark-ext-ins/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0
+ 0.21.1-SNAPSHOTcommonmark-ext-ins
diff --git a/commonmark-ext-task-list-items/pom.xml b/commonmark-ext-task-list-items/pom.xml
index 772088d4a..ec3ac1a1f 100644
--- a/commonmark-ext-task-list-items/pom.xml
+++ b/commonmark-ext-task-list-items/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0
+ 0.21.1-SNAPSHOTcommonmark-ext-task-list-items
diff --git a/commonmark-ext-yaml-front-matter/pom.xml b/commonmark-ext-yaml-front-matter/pom.xml
index 9fd018c9a..e5e120caa 100644
--- a/commonmark-ext-yaml-front-matter/pom.xml
+++ b/commonmark-ext-yaml-front-matter/pom.xml
@@ -4,7 +4,7 @@
commonmark-parentorg.commonmark
- 0.21.0
+ 0.21.1-SNAPSHOTcommonmark-ext-yaml-front-matter
diff --git a/commonmark-integration-test/pom.xml b/commonmark-integration-test/pom.xml
index 2646e7456..da5146a15 100644
--- a/commonmark-integration-test/pom.xml
+++ b/commonmark-integration-test/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0
+ 0.21.1-SNAPSHOTcommonmark-integration-test
diff --git a/commonmark-test-util/pom.xml b/commonmark-test-util/pom.xml
index 6c43eb2ab..c61000412 100644
--- a/commonmark-test-util/pom.xml
+++ b/commonmark-test-util/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0
+ 0.21.1-SNAPSHOTcommonmark-test-util
diff --git a/commonmark/pom.xml b/commonmark/pom.xml
index 5b69ddcf0..64c456192 100644
--- a/commonmark/pom.xml
+++ b/commonmark/pom.xml
@@ -4,7 +4,7 @@
org.commonmarkcommonmark-parent
- 0.21.0
+ 0.21.1-SNAPSHOTcommonmark
diff --git a/pom.xml b/pom.xml
index f44082810..f5c7749ef 100644
--- a/pom.xml
+++ b/pom.xml
@@ -5,7 +5,7 @@
org.commonmarkcommonmark-parent
- 0.21.0
+ 0.21.1-SNAPSHOTcommonmark-java parent
Java implementation of CommonMark, a specification of the Markdown format for turning plain text into formatted
@@ -112,52 +112,52 @@
org.commonmarkcommonmark
- 0.21.0
+ 0.21.1-SNAPSHOTorg.commonmarkcommonmark-ext-autolink
- 0.21.0
+ 0.21.1-SNAPSHOTorg.commonmarkcommonmark-ext-image-attributes
- 0.21.0
+ 0.21.1-SNAPSHOTorg.commonmarkcommonmark-ext-ins
- 0.21.0
+ 0.21.1-SNAPSHOTorg.commonmarkcommonmark-ext-gfm-strikethrough
- 0.21.0
+ 0.21.1-SNAPSHOTorg.commonmarkcommonmark-ext-gfm-tables
- 0.21.0
+ 0.21.1-SNAPSHOTorg.commonmarkcommonmark-ext-heading-anchor
- 0.21.0
+ 0.21.1-SNAPSHOTorg.commonmarkcommonmark-ext-task-list-items
- 0.21.0
+ 0.21.1-SNAPSHOTorg.commonmarkcommonmark-ext-yaml-front-matter
- 0.21.0
+ 0.21.1-SNAPSHOTorg.commonmarkcommonmark-test-util
- 0.21.0
+ 0.21.1-SNAPSHOT
@@ -282,7 +282,7 @@
scm:git:https://github.com/commonmark/commonmark-javascm:git:https://github.com/commonmark/commonmark-javahttps://github.com/commonmark/commonmark-java
- commonmark-parent-0.21.0
+ HEAD
From fbf87b8495d00723e8a62ba0824fd7df7fdf0ce5 Mon Sep 17 00:00:00 2001
From: Dan Wyand
Date: Fri, 14 Jul 2023 14:34:44 -0400
Subject: [PATCH 018/286] add column width to TableCell nodes
The width is determined by the number of dash and colon characters in the
separator line of a table, and passed on to the consumer as an integer.
This permits consumers of the AST to render tables that are more
proportional the width in the source markdown.
No changes have been made to the HTML render to make use of the width
information, but the test for this feature demonstrates a possible usage.
Should a cell be be created that is further to the right than the parsed
separator columns are (i.e., when alignment would be null), then the width
will be reported as 0.
---
.../commonmark/ext/gfm/tables/TableCell.java | 12 ++++++
.../gfm/tables/internal/TableBlockParser.java | 39 +++++++++++++++----
.../commonmark/ext/gfm/tables/TablesTest.java | 36 +++++++++++++++++
3 files changed, 80 insertions(+), 7 deletions(-)
diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TableCell.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TableCell.java
index 61880c6c3..623e30062 100644
--- a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TableCell.java
+++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TableCell.java
@@ -9,6 +9,7 @@ public class TableCell extends CustomNode {
private boolean header;
private Alignment alignment;
+ private int width;
/**
* @return whether the cell is a header or not
@@ -32,6 +33,17 @@ public void setAlignment(Alignment alignment) {
this.alignment = alignment;
}
+ /**
+ * @return the cell width
+ */
+ public int getWidth() {
+ return width;
+ }
+
+ public void setWidth(int width) {
+ this.width = width;
+ }
+
/**
* How the cell is aligned horizontally.
*/
diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java
index b7cea14db..2ffa53109 100644
--- a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java
+++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java
@@ -17,11 +17,11 @@ public class TableBlockParser extends AbstractBlockParser {
private final TableBlock block = new TableBlock();
private final List rowLines = new ArrayList<>();
- private final List columns;
+ private final List columns;
private boolean canHaveLazyContinuationLines = true;
- private TableBlockParser(List columns, SourceLine headerLine) {
+ private TableBlockParser(List columns, SourceLine headerLine) {
this.columns = columns;
this.rowLines.add(headerLine);
}
@@ -120,7 +120,9 @@ private TableCell parseCell(SourceLine cell, int column, InlineParser inlinePars
}
if (column < columns.size()) {
- tableCell.setAlignment(columns.get(column));
+ TableCellInfo cellInfo = columns.get(column);
+ tableCell.setAlignment(cellInfo.getAlignment());
+ tableCell.setWidth(cellInfo.getWidth());
}
CharSequence content = cell.getContent();
@@ -187,11 +189,12 @@ private static List split(SourceLine line) {
// -|-
// |-|-|
// --- | ---
- private static List parseSeparator(CharSequence s) {
- List columns = new ArrayList<>();
+ private static List parseSeparator(CharSequence s) {
+ List columns = new ArrayList<>();
int pipes = 0;
boolean valid = false;
int i = 0;
+ int width = 0;
while (i < s.length()) {
char c = s.charAt(i);
switch (c) {
@@ -216,10 +219,12 @@ private static List parseSeparator(CharSequence s) {
if (c == ':') {
left = true;
i++;
+ width++;
}
boolean haveDash = false;
while (i < s.length() && s.charAt(i) == '-') {
i++;
+ width++;
haveDash = true;
}
if (!haveDash) {
@@ -229,8 +234,10 @@ private static List parseSeparator(CharSequence s) {
if (i < s.length() && s.charAt(i) == ':') {
right = true;
i++;
+ width++;
}
- columns.add(getAlignment(left, right));
+ columns.add(new TableCellInfo(getAlignment(left, right), width));
+ width = 0;
// Next, need another pipe
pipes = 0;
break;
@@ -270,7 +277,7 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar
if (paragraphLines.size() == 1 && Parsing.find('|', paragraphLines.get(0).getContent(), 0) != -1) {
SourceLine line = state.getLine();
SourceLine separatorLine = line.substring(state.getIndex(), line.getContent().length());
- List columns = parseSeparator(separatorLine.getContent());
+ List columns = parseSeparator(separatorLine.getContent());
if (columns != null && !columns.isEmpty()) {
SourceLine paragraph = paragraphLines.get(0);
List headerCells = split(paragraph);
@@ -284,4 +291,22 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar
return BlockStart.none();
}
}
+
+ private static class TableCellInfo {
+ private final TableCell.Alignment alignment;
+ private final int width;
+
+ public TableCell.Alignment getAlignment() {
+ return alignment;
+ }
+
+ public int getWidth() {
+ return width;
+ }
+
+ public TableCellInfo(TableCell.Alignment alignment, int width) {
+ this.alignment = alignment;
+ this.width = width;
+ }
+ }
}
diff --git a/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesTest.java b/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesTest.java
index bef3b8b6c..b03714d9a 100644
--- a/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesTest.java
+++ b/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesTest.java
@@ -749,6 +749,42 @@ public void setAttributes(Node node, String tagName, Map attribu
"\n"));
}
+ @Test
+ public void columnWidthIsRecorded() {
+ AttributeProviderFactory factory = new AttributeProviderFactory() {
+ @Override
+ public AttributeProvider create(AttributeProviderContext context) {
+ return new AttributeProvider() {
+ @Override
+ public void setAttributes(Node node, String tagName, Map attributes) {
+ if (node instanceof TableCell && "th".equals(tagName)) {
+ attributes.put("width", ((TableCell) node).getWidth() + "em");
+ }
+ }
+ };
+ }
+ };
+ HtmlRenderer renderer = HtmlRenderer.builder()
+ .attributeProviderFactory(factory)
+ .extensions(EXTENSIONS)
+ .build();
+ String rendered = renderer.render(PARSER.parse("Abc|Def\n-----|---\n1|2"));
+ assertThat(rendered, is("
\n" +
+ "\n" +
+ "
\n" +
+ "
Abc
\n" +
+ "
Def
\n" +
+ "
\n" +
+ "\n" +
+ "\n" +
+ "
\n" +
+ "
1
\n" +
+ "
2
\n" +
+ "
\n" +
+ "\n" +
+ "
\n"));
+ }
+
@Test
public void sourceSpans() {
Parser parser = Parser.builder()
From 654203286f8e6b1900a4e564613ed361e8a36449 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 9 Jan 2024 16:48:57 +1100
Subject: [PATCH 019/286] Add comment to TextContentRenderer, fix typo
---
.../org/commonmark/renderer/text/TextContentRenderer.java | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/text/TextContentRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/text/TextContentRenderer.java
index aacfbb82a..9dd5918af 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/text/TextContentRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/text/TextContentRenderer.java
@@ -9,6 +9,9 @@
import java.util.ArrayList;
import java.util.List;
+/**
+ * Renders nodes to plain text content with minimal markup-like additions.
+ */
public class TextContentRenderer implements Renderer {
private final boolean stripNewlines;
@@ -30,7 +33,7 @@ public NodeRenderer create(TextContentNodeRendererContext context) {
}
/**
- * Create a new builder for configuring an {@link TextContentRenderer}.
+ * Create a new builder for configuring a {@link TextContentRenderer}.
*
* @return a builder
*/
@@ -52,7 +55,7 @@ public String render(Node node) {
}
/**
- * Builder for configuring an {@link TextContentRenderer}. See methods for default configuration.
+ * Builder for configuring a {@link TextContentRenderer}. See methods for default configuration.
*/
public static class Builder {
From 320266c0af42f4672144adfc726ba3edad221a4e Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 9 Jan 2024 16:50:07 +1100
Subject: [PATCH 020/286] Ignore .DS_Store
---
.gitignore | 3 +++
1 file changed, 3 insertions(+)
diff --git a/.gitignore b/.gitignore
index a156931f0..d998d8890 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,3 +9,6 @@
# Maven
target/
+
+# macOS
+.DS_Store
From 1ba567cd09bc3296b84d06dbfc4d3af9d5eff797 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 10 Jan 2024 14:36:09 +1100
Subject: [PATCH 021/286] Make SpecTestCase not extend RenderingTestCase
That allows it to be used outside of HTML rendering test cases.
With JUnit 5, it might not even need to be a base class anymore.
---
.../integration/SourceSpanIntegrationTest.java | 2 +-
.../integration/SpecIntegrationTest.java | 9 +++++----
.../java/org/commonmark/testutil/Asserts.java | 17 +++++++++++++++++
.../commonmark/testutil/RenderingTestCase.java | 14 +-------------
.../org/commonmark/testutil/SpecTestCase.java | 9 +++------
.../java/org/commonmark/test/SpecCoreTest.java | 9 +++++++--
.../org/commonmark/test/SpecCrLfCoreTest.java | 11 +++++++++--
7 files changed, 43 insertions(+), 28 deletions(-)
create mode 100644 commonmark-test-util/src/main/java/org/commonmark/testutil/Asserts.java
diff --git a/commonmark-integration-test/src/test/java/org/commonmark/integration/SourceSpanIntegrationTest.java b/commonmark-integration-test/src/test/java/org/commonmark/integration/SourceSpanIntegrationTest.java
index 6e51b0af5..b6fa4922a 100644
--- a/commonmark-integration-test/src/test/java/org/commonmark/integration/SourceSpanIntegrationTest.java
+++ b/commonmark-integration-test/src/test/java/org/commonmark/integration/SourceSpanIntegrationTest.java
@@ -5,7 +5,7 @@
import org.commonmark.testutil.example.Example;
/**
- * Spec and all extensions, with source spans enabed.
+ * Spec and all extensions, with source spans enabled.
*/
public class SourceSpanIntegrationTest extends SpecIntegrationTest {
diff --git a/commonmark-integration-test/src/test/java/org/commonmark/integration/SpecIntegrationTest.java b/commonmark-integration-test/src/test/java/org/commonmark/integration/SpecIntegrationTest.java
index 13d5918b8..f434f65d2 100644
--- a/commonmark-integration-test/src/test/java/org/commonmark/integration/SpecIntegrationTest.java
+++ b/commonmark-integration-test/src/test/java/org/commonmark/integration/SpecIntegrationTest.java
@@ -12,10 +12,13 @@
import org.commonmark.parser.Parser;
import org.commonmark.testutil.example.Example;
import org.commonmark.testutil.SpecTestCase;
+import org.junit.Assert;
import org.junit.Test;
import java.util.*;
+import static org.commonmark.testutil.Asserts.assertRendering;
+
/**
* Tests that the spec examples still render the same with all extensions enabled.
*/
@@ -39,17 +42,15 @@ public SpecIntegrationTest(Example example) {
}
@Test
- @Override
public void testHtmlRendering() {
String expectedHtml = OVERRIDDEN_EXAMPLES.get(example.getSource());
if (expectedHtml != null) {
- assertRendering(example.getSource(), expectedHtml);
+ assertRendering(example.getSource(), expectedHtml, render(example.getSource()));
} else {
- super.testHtmlRendering();
+ assertRendering(example.getSource(), example.getHtml(), render(example.getSource()));
}
}
- @Override
protected String render(String source) {
return RENDERER.render(PARSER.parse(source));
}
diff --git a/commonmark-test-util/src/main/java/org/commonmark/testutil/Asserts.java b/commonmark-test-util/src/main/java/org/commonmark/testutil/Asserts.java
new file mode 100644
index 000000000..64124b129
--- /dev/null
+++ b/commonmark-test-util/src/main/java/org/commonmark/testutil/Asserts.java
@@ -0,0 +1,17 @@
+package org.commonmark.testutil;
+
+import static org.junit.Assert.assertEquals;
+
+public class Asserts {
+ public static void assertRendering(String source, String expectedRendering, String actualRendering) {
+ // include source for better assertion errors
+ String expected = showTabs(expectedRendering + "\n\n" + source);
+ String actual = showTabs(actualRendering + "\n\n" + source);
+ assertEquals(expected, actual);
+ }
+
+ private static String showTabs(String s) {
+ // Tabs are shown as "rightwards arrow" for easier comparison
+ return s.replace("\t", "\u2192");
+ }
+}
diff --git a/commonmark-test-util/src/main/java/org/commonmark/testutil/RenderingTestCase.java b/commonmark-test-util/src/main/java/org/commonmark/testutil/RenderingTestCase.java
index 682123494..b585f4604 100644
--- a/commonmark-test-util/src/main/java/org/commonmark/testutil/RenderingTestCase.java
+++ b/commonmark-test-util/src/main/java/org/commonmark/testutil/RenderingTestCase.java
@@ -1,22 +1,10 @@
package org.commonmark.testutil;
-import static org.junit.Assert.assertEquals;
-
public abstract class RenderingTestCase {
protected abstract String render(String source);
protected void assertRendering(String source, String expectedResult) {
- String renderedContent = render(source);
-
- // include source for better assertion errors
- String expected = showTabs(expectedResult + "\n\n" + source);
- String actual = showTabs(renderedContent + "\n\n" + source);
- assertEquals(expected, actual);
- }
-
- private static String showTabs(String s) {
- // Tabs are shown as "rightwards arrow" for easier comparison
- return s.replace("\t", "\u2192");
+ Asserts.assertRendering(source, expectedResult, render(source));
}
}
diff --git a/commonmark-test-util/src/main/java/org/commonmark/testutil/SpecTestCase.java b/commonmark-test-util/src/main/java/org/commonmark/testutil/SpecTestCase.java
index 1c35b7c28..3be768682 100644
--- a/commonmark-test-util/src/main/java/org/commonmark/testutil/SpecTestCase.java
+++ b/commonmark-test-util/src/main/java/org/commonmark/testutil/SpecTestCase.java
@@ -10,8 +10,10 @@
import java.util.ArrayList;
import java.util.List;
+import static org.commonmark.testutil.Asserts.assertRendering;
+
@RunWith(Parameterized.class)
-public abstract class SpecTestCase extends RenderingTestCase {
+public abstract class SpecTestCase {
protected final Example example;
@@ -29,9 +31,4 @@ public static List data() {
return data;
}
- @Test
- public void testHtmlRendering() {
- assertRendering(example.getSource(), example.getHtml());
- }
-
}
diff --git a/commonmark/src/test/java/org/commonmark/test/SpecCoreTest.java b/commonmark/src/test/java/org/commonmark/test/SpecCoreTest.java
index e4820f09c..8d284c6f9 100644
--- a/commonmark/src/test/java/org/commonmark/test/SpecCoreTest.java
+++ b/commonmark/src/test/java/org/commonmark/test/SpecCoreTest.java
@@ -9,6 +9,7 @@
import org.commonmark.testutil.example.Example;
import org.junit.Test;
+import static org.commonmark.testutil.Asserts.assertRendering;
import static org.junit.Assert.fail;
public class SpecCoreTest extends SpecTestCase {
@@ -49,8 +50,12 @@ protected void visitChildren(Node parent) {
});
}
- @Override
- protected String render(String source) {
+ @Test
+ public void testHtmlRendering() {
+ assertRendering(example.getSource(), example.getHtml(), render(example.getSource()));
+ }
+
+ private String render(String source) {
return RENDERER.render(PARSER.parse(source));
}
}
diff --git a/commonmark/src/test/java/org/commonmark/test/SpecCrLfCoreTest.java b/commonmark/src/test/java/org/commonmark/test/SpecCrLfCoreTest.java
index 6424ab659..ab0103e5a 100644
--- a/commonmark/src/test/java/org/commonmark/test/SpecCrLfCoreTest.java
+++ b/commonmark/src/test/java/org/commonmark/test/SpecCrLfCoreTest.java
@@ -4,6 +4,9 @@
import org.commonmark.renderer.html.HtmlRenderer;
import org.commonmark.testutil.SpecTestCase;
import org.commonmark.testutil.example.Example;
+import org.junit.Test;
+
+import static org.commonmark.testutil.Asserts.assertRendering;
/**
* Same as {@link SpecCoreTest} but converts line endings to Windows-style CR+LF endings before parsing.
@@ -18,8 +21,12 @@ public SpecCrLfCoreTest(Example example) {
super(example);
}
- @Override
- protected String render(String source) {
+ @Test
+ public void testHtmlRendering() {
+ assertRendering(example.getSource(), example.getHtml(), render(example.getSource()));
+ }
+
+ private String render(String source) {
String windowsStyle = source.replace("\n", "\r\n");
return RENDERER.render(PARSER.parse(windowsStyle));
}
From ec66829b7541b2539cc2126b8f4df4414e2607b8 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 16 Jan 2024 22:30:52 +1100
Subject: [PATCH 022/286] Add markerIndent and contentIndent to ListItem
This information is required for a renderer producing Markdown.
---
.../commonmark/internal/ListBlockParser.java | 2 +-
.../commonmark/internal/ListItemParser.java | 4 +-
.../java/org/commonmark/node/ListItem.java | 45 +++++++++++++
.../commonmark/test/ListBlockParserTest.java | 67 +++++++++++++++++++
.../test/java/org/commonmark/test/Nodes.java | 23 +++++++
5 files changed, 139 insertions(+), 2 deletions(-)
create mode 100644 commonmark/src/test/java/org/commonmark/test/ListBlockParserTest.java
diff --git a/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java
index f6702518b..0ff644a47 100644
--- a/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/ListBlockParser.java
@@ -217,7 +217,7 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar
}
int newColumn = listData.contentColumn;
- ListItemParser listItemParser = new ListItemParser(newColumn - state.getColumn());
+ ListItemParser listItemParser = new ListItemParser(state.getIndent(), newColumn - state.getColumn());
// prepend the list block if needed
if (!(matched instanceof ListBlockParser) ||
diff --git a/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java b/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java
index 6f03770b3..49722dff2 100644
--- a/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/ListItemParser.java
@@ -20,8 +20,10 @@ public class ListItemParser extends AbstractBlockParser {
private boolean hadBlankLine;
- public ListItemParser(int contentIndent) {
+ public ListItemParser(int markerIndent, int contentIndent) {
this.contentIndent = contentIndent;
+ block.setMarkerIndent(markerIndent);
+ block.setContentIndent(contentIndent);
}
@Override
diff --git a/commonmark/src/main/java/org/commonmark/node/ListItem.java b/commonmark/src/main/java/org/commonmark/node/ListItem.java
index aa526be01..21f4e2b82 100644
--- a/commonmark/src/main/java/org/commonmark/node/ListItem.java
+++ b/commonmark/src/main/java/org/commonmark/node/ListItem.java
@@ -2,8 +2,53 @@
public class ListItem extends Block {
+ private int markerIndent;
+ private int contentIndent;
+
@Override
public void accept(Visitor visitor) {
visitor.visit(this);
}
+
+ /**
+ * Returns the indent of the marker such as "-" or "1." in columns (spaces or tab stop of 4).
+ *
+ * Some examples and their marker indent:
+ *
- Foo
+ * Marker indent: 0
+ *
- Foo
+ * Marker indent: 1
+ *
1. Foo
+ * Marker indent: 2
+ */
+ public int getMarkerIndent() {
+ return markerIndent;
+ }
+
+ public void setMarkerIndent(int markerIndent) {
+ this.markerIndent = markerIndent;
+ }
+
+ /**
+ * Returns the indent of the content in columns (spaces or tab stop of 4). The content indent is counted from the
+ * beginning of the line and includes the marker on the first line.
+ *
+ * Some examples and their content indent:
+ *
- Foo
+ * Content indent: 2
+ *
- Foo
+ * Content indent: 3
+ *
1. Foo
+ * Content indent: 5
+ *
+ * Note that subsequent lines in the same list item need to be indented by at least the content indent to be counted
+ * as part of the list item.
+ */
+ public int getContentIndent() {
+ return contentIndent;
+ }
+
+ public void setContentIndent(int contentIndent) {
+ this.contentIndent = contentIndent;
+ }
}
diff --git a/commonmark/src/test/java/org/commonmark/test/ListBlockParserTest.java b/commonmark/src/test/java/org/commonmark/test/ListBlockParserTest.java
new file mode 100644
index 000000000..a8a03fb74
--- /dev/null
+++ b/commonmark/src/test/java/org/commonmark/test/ListBlockParserTest.java
@@ -0,0 +1,67 @@
+package org.commonmark.test;
+
+import org.commonmark.node.ListItem;
+import org.commonmark.node.Node;
+import org.commonmark.parser.Parser;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+public class ListBlockParserTest {
+
+ private static final Parser PARSER = Parser.builder().build();
+
+ @Test
+ public void testBulletListIndents() {
+ assertListItemIndents("* foo", 0, 2);
+ assertListItemIndents(" * foo", 1, 3);
+ assertListItemIndents(" * foo", 2, 4);
+ assertListItemIndents(" * foo", 3, 5);
+
+ assertListItemIndents("* foo", 0, 3);
+ assertListItemIndents("* foo", 0, 4);
+ assertListItemIndents("* foo", 0, 5);
+ assertListItemIndents(" * foo", 1, 4);
+ assertListItemIndents(" * foo", 3, 8);
+
+ // The indent is relative to any containing blocks
+ assertListItemIndents("> * foo", 0, 2);
+ assertListItemIndents("> * foo", 1, 3);
+ assertListItemIndents("> * foo", 1, 4);
+
+ // Tab counts as 3 spaces here (to the next tab stop column of 4) -> content indent is 1+3
+ assertListItemIndents("*\tfoo", 0, 4);
+
+ // Empty list, content indent is expected to be 2
+ assertListItemIndents("-\n", 0, 2);
+ }
+
+ @Test
+ public void testOrderedListIndents() {
+ assertListItemIndents("1. foo", 0, 3);
+ assertListItemIndents(" 1. foo", 1, 4);
+ assertListItemIndents(" 1. foo", 2, 5);
+ assertListItemIndents(" 1. foo", 3, 6);
+
+ assertListItemIndents("1. foo", 0, 4);
+ assertListItemIndents("1. foo", 0, 5);
+ assertListItemIndents("1. foo", 0, 6);
+ assertListItemIndents(" 1. foo", 1, 5);
+ assertListItemIndents(" 1. foo", 2, 8);
+
+ assertListItemIndents("> 1. foo", 0, 3);
+ assertListItemIndents("> 1. foo", 1, 4);
+ assertListItemIndents("> 1. foo", 1, 5);
+
+ assertListItemIndents("1.\tfoo", 0, 4);
+ }
+
+ private void assertListItemIndents(String input, int expectedMarkerIndent, int expectedContentIndent) {
+ Node doc = PARSER.parse(input);
+ ListItem listItem = Nodes.find(doc, ListItem.class);
+ assertNotNull(listItem);
+ assertEquals(expectedMarkerIndent, listItem.getMarkerIndent());
+ assertEquals(expectedContentIndent, listItem.getContentIndent());
+ }
+}
diff --git a/commonmark/src/test/java/org/commonmark/test/Nodes.java b/commonmark/src/test/java/org/commonmark/test/Nodes.java
index bbc019a6a..25ce75836 100644
--- a/commonmark/src/test/java/org/commonmark/test/Nodes.java
+++ b/commonmark/src/test/java/org/commonmark/test/Nodes.java
@@ -14,4 +14,27 @@ public static List getChildren(Node parent) {
}
return children;
}
+
+ /**
+ * Recursively try to find a node with the given type within the children of the specified node.
+ *
+ * @param parent The node to get children from (node itself will not be checked)
+ * @param nodeClass The type of node to find
+ */
+ public static T find(Node parent, Class nodeClass) {
+ Node node = parent.getFirstChild();
+ while (node != null) {
+ Node next = node.getNext();
+ if (nodeClass.isInstance(node)) {
+ //noinspection unchecked
+ return (T) node;
+ }
+ T result = find(node, nodeClass);
+ if (result != null) {
+ return result;
+ }
+ node = next;
+ }
+ return null;
+ }
}
From 9ec3935d7fc4982d9c407d89d6810a13fa3a8a22 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 9 Jan 2024 16:53:44 +1100
Subject: [PATCH 023/286] Beginnings of a MarkdownRenderer
Structure mostly copied from TextContentRenderer (for extension related bits).
Started with leaf blocks (in spec order), but probably good to do some inlines
too before going too far down all the blocks.
Also, container blocks might be the most interesting.
---
.../markdown/CoreMarkdownNodeRenderer.java | 152 ++++++++++++++++++
.../markdown/MarkdownNodeRendererContext.java | 19 +++
.../markdown/MarkdownNodeRendererFactory.java | 17 ++
.../renderer/markdown/MarkdownRenderer.java | 134 +++++++++++++++
.../renderer/markdown/MarkdownWriter.java | 52 ++++++
.../markdown/MarkdownRendererTest.java | 50 ++++++
6 files changed, 424 insertions(+)
create mode 100644 commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
create mode 100644 commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererContext.java
create mode 100644 commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererFactory.java
create mode 100644 commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
create mode 100644 commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
create mode 100644 commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
new file mode 100644
index 000000000..ee4fbbe95
--- /dev/null
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -0,0 +1,152 @@
+package org.commonmark.renderer.markdown;
+
+import org.commonmark.node.*;
+import org.commonmark.renderer.NodeRenderer;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * The node renderer that renders all the core nodes (comes last in the order of node renderers).
+ */
+public class CoreMarkdownNodeRenderer extends AbstractVisitor implements NodeRenderer {
+
+ protected final MarkdownNodeRendererContext context;
+ private final MarkdownWriter writer;
+
+ public CoreMarkdownNodeRenderer(MarkdownNodeRendererContext context) {
+ this.context = context;
+ this.writer = context.getWriter();
+ }
+
+ @Override
+ public Set> getNodeTypes() {
+ return new HashSet<>(Arrays.asList(
+ Document.class,
+ Heading.class,
+ Paragraph.class,
+ BlockQuote.class,
+ BulletList.class,
+ FencedCodeBlock.class,
+ HtmlBlock.class,
+ ThematicBreak.class,
+ IndentedCodeBlock.class,
+ Link.class,
+ ListItem.class,
+ OrderedList.class,
+ Image.class,
+ Emphasis.class,
+ StrongEmphasis.class,
+ Text.class,
+ Code.class,
+ HtmlInline.class,
+ SoftLineBreak.class,
+ HardLineBreak.class
+ ));
+ }
+
+ @Override
+ public void render(Node node) {
+ node.accept(this);
+ }
+
+ @Override
+ public void visit(Document document) {
+ // No rendering itself
+ visitChildren(document);
+ }
+
+ @Override
+ public void visit(BlockQuote blockQuote) {
+ }
+
+ @Override
+ public void visit(BulletList bulletList) {
+ }
+
+ @Override
+ public void visit(Code code) {
+ }
+
+ @Override
+ public void visit(FencedCodeBlock fencedCodeBlock) {
+ }
+
+ @Override
+ public void visit(HardLineBreak hardLineBreak) {
+ }
+
+ @Override
+ public void visit(Heading heading) {
+ for (int i = 0; i < heading.getLevel(); i++) {
+ writer.write('#');
+ }
+ writer.write(' ');
+ visitChildren(heading);
+ writer.block();
+ }
+
+ @Override
+ public void visit(ThematicBreak thematicBreak) {
+ writer.write("***");
+ writer.block();
+ }
+
+ @Override
+ public void visit(HtmlInline htmlInline) {
+ }
+
+ @Override
+ public void visit(HtmlBlock htmlBlock) {
+ }
+
+ @Override
+ public void visit(Image image) {
+ }
+
+ @Override
+ public void visit(IndentedCodeBlock indentedCodeBlock) {
+ }
+
+ @Override
+ public void visit(Link link) {
+ }
+
+ @Override
+ public void visit(ListItem listItem) {
+ }
+
+ @Override
+ public void visit(OrderedList orderedList) {
+ }
+
+ @Override
+ public void visit(Paragraph paragraph) {
+ visitChildren(paragraph);
+ writer.block();
+ }
+
+ @Override
+ public void visit(SoftLineBreak softLineBreak) {
+ }
+
+ @Override
+ public void visit(Text text) {
+ writeText(text.getLiteral());
+ }
+
+ @Override
+ protected void visitChildren(Node parent) {
+ Node node = parent.getFirstChild();
+ while (node != null) {
+ Node next = node.getNext();
+ context.render(node);
+ node = next;
+ }
+ }
+
+ private void writeText(String text) {
+ writer.write(text);
+ }
+}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererContext.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererContext.java
new file mode 100644
index 000000000..8fe0f73d5
--- /dev/null
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererContext.java
@@ -0,0 +1,19 @@
+package org.commonmark.renderer.markdown;
+
+import org.commonmark.node.Node;
+
+public interface MarkdownNodeRendererContext {
+
+ /**
+ * @return the writer to use
+ */
+ MarkdownWriter getWriter();
+
+ /**
+ * Render the specified node and its children using the configured renderers. This should be used to render child
+ * nodes; be careful not to pass the node that is being rendered, that would result in an endless loop.
+ *
+ * @param node the node to render
+ */
+ void render(Node node);
+}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererFactory.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererFactory.java
new file mode 100644
index 000000000..7b3134277
--- /dev/null
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererFactory.java
@@ -0,0 +1,17 @@
+package org.commonmark.renderer.markdown;
+
+import org.commonmark.renderer.NodeRenderer;
+
+/**
+ * Factory for instantiating new node renderers Æ’or rendering.
+ */
+public interface MarkdownNodeRendererFactory {
+
+ /**
+ * Create a new node renderer for the specified rendering context.
+ *
+ * @param context the context for rendering (normally passed on to the node renderer)
+ * @return a node renderer
+ */
+ NodeRenderer create(MarkdownNodeRendererContext context);
+}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
new file mode 100644
index 000000000..6466b97a2
--- /dev/null
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
@@ -0,0 +1,134 @@
+package org.commonmark.renderer.markdown;
+
+import org.commonmark.Extension;
+import org.commonmark.internal.renderer.NodeRendererMap;
+import org.commonmark.node.Node;
+import org.commonmark.renderer.NodeRenderer;
+import org.commonmark.renderer.Renderer;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Renders nodes to CommonMark Markdown.
+ *
+ * Note that it does not currently attempt to preserve the exact syntax of the original input Markdown (if any):
+ *
+ *
Headings are always output as ATX headings for simplicity
+ *
+ */
+public class MarkdownRenderer implements Renderer {
+
+ private final List nodeRendererFactories;
+
+ private MarkdownRenderer(Builder builder) {
+ this.nodeRendererFactories = new ArrayList<>(builder.nodeRendererFactories.size() + 1);
+ this.nodeRendererFactories.addAll(builder.nodeRendererFactories);
+ // Add as last. This means clients can override the rendering of core nodes if they want.
+ this.nodeRendererFactories.add(new MarkdownNodeRendererFactory() {
+ @Override
+ public NodeRenderer create(MarkdownNodeRendererContext context) {
+ return new CoreMarkdownNodeRenderer(context);
+ }
+ });
+ }
+
+ /**
+ * Create a new builder for configuring a {@link MarkdownRenderer}.
+ *
+ * @return a builder
+ */
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ @Override
+ public void render(Node node, Appendable output) {
+ RendererContext context = new RendererContext(new MarkdownWriter(output));
+ context.render(node);
+ }
+
+ @Override
+ public String render(Node node) {
+ StringBuilder sb = new StringBuilder();
+ render(node, sb);
+ return sb.toString();
+ }
+
+ /**
+ * Builder for configuring a {@link MarkdownRenderer}. See methods for default configuration.
+ */
+ public static class Builder {
+
+ private List nodeRendererFactories = new ArrayList<>();
+
+ /**
+ * @return the configured {@link MarkdownRenderer}
+ */
+ public MarkdownRenderer build() {
+ return new MarkdownRenderer(this);
+ }
+
+ /**
+ * Add a factory for instantiating a node renderer (done when rendering). This allows to override the rendering
+ * of node types or define rendering for custom node types.
+ *
+ * If multiple node renderers for the same node type are created, the one from the factory that was added first
+ * "wins". (This is how the rendering for core node types can be overridden; the default rendering comes last.)
+ *
+ * @param nodeRendererFactory the factory for creating a node renderer
+ * @return {@code this}
+ */
+ public Builder nodeRendererFactory(MarkdownNodeRendererFactory nodeRendererFactory) {
+ this.nodeRendererFactories.add(nodeRendererFactory);
+ return this;
+ }
+
+ /**
+ * @param extensions extensions to use on this renderer
+ * @return {@code this}
+ */
+ public Builder extensions(Iterable extends Extension> extensions) {
+ for (Extension extension : extensions) {
+ if (extension instanceof MarkdownRendererExtension) {
+ MarkdownRendererExtension markdownRendererExtension = (MarkdownRendererExtension) extension;
+ markdownRendererExtension.extend(this);
+ }
+ }
+ return this;
+ }
+ }
+
+ /**
+ * Extension for {@link MarkdownRenderer}.
+ */
+ public interface MarkdownRendererExtension extends Extension {
+ void extend(Builder rendererBuilder);
+ }
+
+ private class RendererContext implements MarkdownNodeRendererContext {
+ private final MarkdownWriter writer;
+ private final NodeRendererMap nodeRendererMap = new NodeRendererMap();
+
+ private RendererContext(MarkdownWriter writer) {
+ this.writer = writer;
+
+ // The first node renderer for a node type "wins".
+ for (int i = nodeRendererFactories.size() - 1; i >= 0; i--) {
+ MarkdownNodeRendererFactory nodeRendererFactory = nodeRendererFactories.get(i);
+ NodeRenderer nodeRenderer = nodeRendererFactory.create(this);
+ nodeRendererMap.add(nodeRenderer);
+ }
+ }
+
+ @Override
+ public MarkdownWriter getWriter() {
+ return writer;
+ }
+
+ @Override
+ public void render(Node node) {
+ nodeRendererMap.render(node);
+ }
+ }
+}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
new file mode 100644
index 000000000..1377d86f9
--- /dev/null
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
@@ -0,0 +1,52 @@
+package org.commonmark.renderer.markdown;
+
+import java.io.IOException;
+
+public class MarkdownWriter {
+
+ private final Appendable buffer;
+
+ private boolean prependLine = false;
+
+ public MarkdownWriter(Appendable out) {
+ buffer = out;
+ }
+
+ public void block() {
+ append('\n');
+ prependLine = true;
+ }
+
+ public void write(String s) {
+ append(s);
+ }
+
+ public void write(char c) {
+ append(c);
+ }
+
+ private void append(String s) {
+ try {
+ appendLineIfNeeded();
+ buffer.append(s);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void append(char c) {
+ try {
+ appendLineIfNeeded();
+ buffer.append(c);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void appendLineIfNeeded() throws IOException {
+ if (prependLine) {
+ buffer.append('\n');
+ prependLine = false;
+ }
+ }
+}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
new file mode 100644
index 000000000..297dfa206
--- /dev/null
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -0,0 +1,50 @@
+package org.commonmark.renderer.markdown;
+
+import org.commonmark.node.Node;
+import org.commonmark.parser.Parser;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class MarkdownRendererTest {
+
+ @Test
+ public void testThematicBreaks() {
+ assertRoundTrip("***\n");
+ // TODO: spec: If you want a thematic break in a list item, use a different bullet:
+
+ assertRoundTrip("***\n\nfoo\n");
+ }
+
+ @Test
+ public void testHeadings() {
+ // Type of heading is currently not preserved
+ assertRoundTrip("# foo\n");
+ assertRoundTrip("## foo\n");
+ assertRoundTrip("### foo\n");
+ assertRoundTrip("#### foo\n");
+ assertRoundTrip("##### foo\n");
+ assertRoundTrip("###### foo\n");
+
+ assertRoundTrip("# foo\n\nbar\n");
+ }
+
+ @Test
+ public void testParagraphs() {
+ assertRoundTrip("foo\n");
+ assertRoundTrip("foo\n\nbar\n");
+ }
+
+ private Node parse(String source) {
+ return Parser.builder().build().parse(source);
+ }
+
+ private String render(String source) {
+ return MarkdownRenderer.builder().build().render(parse(source));
+ }
+
+ private void assertRoundTrip(String input) {
+ String rendered = render(input);
+ assertEquals(input, rendered);
+ }
+}
From b25b7ead8fd8f3066b1361ff8faf2a679021a43b Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 10 Jan 2024 12:04:12 +1100
Subject: [PATCH 024/286] Code spans
---
.../markdown/CoreMarkdownNodeRenderer.java | 34 +++++++++++++++++++
.../markdown/MarkdownRendererTest.java | 9 +++++
2 files changed, 43 insertions(+)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index ee4fbbe95..9addf72fa 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -1,5 +1,6 @@
package org.commonmark.renderer.markdown;
+import org.commonmark.internal.util.Parsing;
import org.commonmark.node.*;
import org.commonmark.renderer.NodeRenderer;
@@ -67,6 +68,24 @@ public void visit(BulletList bulletList) {
@Override
public void visit(Code code) {
+ String literal = code.getLiteral();
+ // If the literal includes backticks, we can surround them by using one more backtick.
+ int backticks = findMaxRunLength('`', literal);
+ for (int i = 0; i < backticks + 1; i++) {
+ writer.write('`');
+ }
+ // If the literal starts or ends with a backtick, surround it with a single space.
+ boolean addSpace = literal.startsWith("`") || literal.endsWith("`");
+ if (addSpace) {
+ writer.write(' ');
+ }
+ writer.write(literal);
+ if (addSpace) {
+ writer.write(' ');
+ }
+ for (int i = 0; i < backticks + 1; i++) {
+ writer.write('`');
+ }
}
@Override
@@ -146,6 +165,21 @@ protected void visitChildren(Node parent) {
}
}
+ private static int findMaxRunLength(char c, CharSequence s) {
+ int backticks = 0;
+ int start = 0;
+ while (start < s.length()) {
+ int index = Parsing.find(c, s, start);
+ if (index != -1) {
+ start = Parsing.skip(c, s, index + 1, s.length());
+ backticks = Math.max(backticks, start - index);
+ } else {
+ break;
+ }
+ }
+ return backticks;
+ }
+
private void writeText(String text) {
writer.write(text);
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index 297dfa206..caa4bdf88 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -35,6 +35,15 @@ public void testParagraphs() {
assertRoundTrip("foo\n\nbar\n");
}
+ @Test
+ public void testCodeSpans() {
+ assertRoundTrip("`foo`\n");
+ assertRoundTrip("``foo ` bar``\n");
+ assertRoundTrip("```foo `` ` bar```\n");
+
+ assertRoundTrip("`` `foo ``\n");
+ }
+
private Node parse(String source) {
return Parser.builder().build().parse(source);
}
From aa7ac8b5183a8a3715de0eebfe49feec72c98306 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 10 Jan 2024 15:43:49 +1100
Subject: [PATCH 025/286] Add test that renders the spec examples through
Markdown
---
.../markdown/SpecMarkdownRendererTest.java | 56 +++++++++++++++++++
1 file changed, 56 insertions(+)
create mode 100644 commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
new file mode 100644
index 000000000..87d7507c6
--- /dev/null
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -0,0 +1,56 @@
+package org.commonmark.renderer.markdown;
+
+import org.commonmark.node.Node;
+import org.commonmark.parser.Parser;
+import org.commonmark.renderer.html.HtmlRenderer;
+import org.commonmark.testutil.TestResources;
+import org.commonmark.testutil.example.Example;
+import org.commonmark.testutil.example.ExampleReader;
+import org.junit.Test;
+
+import java.util.List;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests Markdown rendering using the examples in the spec like this:
+ *
+ *
Parses the source to an AST and then renders it back to Markdown
+ *
Parses that to an AST and then renders it to HTML
+ *
Compares that HTML to the expected HTML of the example:
+ * If it's the same, then the expected elements were preserved in the Markdown rendering
+ *
+ */
+public class SpecMarkdownRendererTest {
+
+ public static final MarkdownRenderer MARKDOWN_RENDERER = MarkdownRenderer.builder().build();
+ public static final HtmlRenderer HTML_RENDERER = HtmlRenderer.builder().build();
+
+ @Test
+ public void testCoverage() {
+ List examples = ExampleReader.readExamples(TestResources.getSpec());
+ int passed = 0;
+ for (Example example : examples) {
+ String markdown = renderMarkdown(example.getSource());
+ String rendered = renderHtml(markdown);
+ if (rendered.equals(example.getHtml())) {
+ passed++;
+ }
+ }
+
+ int expectedPassed = 151;
+ assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passed, passed >= expectedPassed);
+ }
+
+ private Node parse(String source) {
+ return Parser.builder().build().parse(source);
+ }
+
+ private String renderMarkdown(String source) {
+ return MARKDOWN_RENDERER.render(parse(source));
+ }
+
+ private String renderHtml(String source) {
+ return HTML_RENDERER.render(parse(source));
+ }
+}
From bd8fa6a348f74e6a3c6d0fb7762d23bc030caeab Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 10 Jan 2024 15:57:58 +1100
Subject: [PATCH 026/286] Count how many examples pass/fail
---
.../commonmark/testutil/example/Example.java | 4 +++
.../markdown/SpecMarkdownRendererTest.java | 33 +++++++++++++++++--
2 files changed, 34 insertions(+), 3 deletions(-)
diff --git a/commonmark-test-util/src/main/java/org/commonmark/testutil/example/Example.java b/commonmark-test-util/src/main/java/org/commonmark/testutil/example/Example.java
index 417a66097..11e87d0aa 100644
--- a/commonmark-test-util/src/main/java/org/commonmark/testutil/example/Example.java
+++ b/commonmark-test-util/src/main/java/org/commonmark/testutil/example/Example.java
@@ -30,6 +30,10 @@ public String getHtml() {
return html;
}
+ public String getSection() {
+ return section;
+ }
+
@Override
public String toString() {
return "File \"" + filename + "\" section \"" + section + "\" example " + exampleNumber;
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index 87d7507c6..06791e20f 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -8,7 +8,10 @@
import org.commonmark.testutil.example.ExampleReader;
import org.junit.Test;
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import static org.junit.Assert.assertTrue;
@@ -29,17 +32,41 @@ public class SpecMarkdownRendererTest {
@Test
public void testCoverage() {
List examples = ExampleReader.readExamples(TestResources.getSpec());
- int passed = 0;
+ List passes = new ArrayList<>();
+ List fails = new ArrayList<>();
for (Example example : examples) {
String markdown = renderMarkdown(example.getSource());
String rendered = renderHtml(markdown);
if (rendered.equals(example.getHtml())) {
- passed++;
+ passes.add(example);
+ } else {
+ fails.add(example);
}
}
+ System.out.println("Failed examples by section:");
+ printCountsBySection(fails);
+ System.out.println();
+
+ System.out.println("Passed examples by section:");
+ printCountsBySection(passes);
+
int expectedPassed = 151;
- assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passed, passed >= expectedPassed);
+ assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
+ }
+
+ private static void printCountsBySection(List examples) {
+ Map bySection = new LinkedHashMap<>();
+ for (Example example : examples) {
+ Integer count = bySection.get(example.getSection());
+ if (count == null) {
+ count = 0;
+ }
+ bySection.put(example.getSection(), count + 1);
+ }
+ for (Map.Entry entry : bySection.entrySet()) {
+ System.out.println(entry.getKey() + ": " + entry.getValue());
+ }
}
private Node parse(String source) {
From 25abdad8665e0652c1f98cf4add17946294b5bd8 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 10 Jan 2024 16:21:44 +1100
Subject: [PATCH 027/286] Emphasis and strong emphasis
---
.../markdown/CoreMarkdownNodeRenderer.java | 16 ++++++++++++++++
.../renderer/markdown/MarkdownWriter.java | 12 ++++++++++++
.../markdown/MarkdownRendererTest.java | 18 ++++++++++++++++++
.../markdown/SpecMarkdownRendererTest.java | 10 +++++-----
4 files changed, 51 insertions(+), 5 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 9addf72fa..c968da850 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -88,6 +88,15 @@ public void visit(Code code) {
}
}
+ @Override
+ public void visit(Emphasis emphasis) {
+ // When emphasis is nested, a different delimiter needs to be used
+ char delimiter = writer.getLastChar() == '*' ? '_' : '*';
+ writer.write(delimiter);
+ super.visit(emphasis);
+ writer.write(delimiter);
+ }
+
@Override
public void visit(FencedCodeBlock fencedCodeBlock) {
}
@@ -150,6 +159,13 @@ public void visit(Paragraph paragraph) {
public void visit(SoftLineBreak softLineBreak) {
}
+ @Override
+ public void visit(StrongEmphasis strongEmphasis) {
+ writer.write("**");
+ super.visit(strongEmphasis);
+ writer.write("**");
+ }
+
@Override
public void visit(Text text) {
writeText(text.getLiteral());
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
index 1377d86f9..fd79ce6af 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
@@ -7,11 +7,16 @@ public class MarkdownWriter {
private final Appendable buffer;
private boolean prependLine = false;
+ private char lastChar;
public MarkdownWriter(Appendable out) {
buffer = out;
}
+ public char getLastChar() {
+ return lastChar;
+ }
+
public void block() {
append('\n');
prependLine = true;
@@ -32,6 +37,11 @@ private void append(String s) {
} catch (IOException e) {
throw new RuntimeException(e);
}
+
+ int length = s.length();
+ if (length != 0) {
+ lastChar = s.charAt(length - 1);
+ }
}
private void append(char c) {
@@ -41,6 +51,8 @@ private void append(char c) {
} catch (IOException e) {
throw new RuntimeException(e);
}
+
+ lastChar = c;
}
private void appendLineIfNeeded() throws IOException {
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index caa4bdf88..0a2983a90 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -44,6 +44,24 @@ public void testCodeSpans() {
assertRoundTrip("`` `foo ``\n");
}
+ @Test
+ public void testEmphasis() {
+ assertRoundTrip("*foo*\n");
+ assertRoundTrip("foo*bar*\n");
+ // When nesting, a different delimiter needs to be used
+ assertRoundTrip("*_foo_*\n");
+ assertRoundTrip("*_*foo*_*\n");
+
+ // Not emphasis (needs * inside words)
+ assertRoundTrip("foo_bar_\n");
+ }
+
+ @Test
+ public void testStrongEmphasis() {
+ assertRoundTrip("**foo**\n");
+ assertRoundTrip("foo**bar**\n");
+ }
+
private Node parse(String source) {
return Parser.builder().build().parse(source);
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index 06791e20f..c94465e7a 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -44,14 +44,14 @@ public void testCoverage() {
}
}
- System.out.println("Failed examples by section:");
- printCountsBySection(fails);
+ System.out.println("Passed examples by section (total " + passes.size() + "):");
+ printCountsBySection(passes);
System.out.println();
- System.out.println("Passed examples by section:");
- printCountsBySection(passes);
+ System.out.println("Failed examples by section (total " + fails.size() + "):");
+ printCountsBySection(fails);
- int expectedPassed = 151;
+ int expectedPassed = 226;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From 3e99b4886e4aac77856d05aaaf56e3868cb05c78 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 10 Jan 2024 16:27:36 +1100
Subject: [PATCH 028/286] Line breaks
---
.../renderer/markdown/CoreMarkdownNodeRenderer.java | 3 +++
.../commonmark/renderer/markdown/MarkdownWriter.java | 4 ++++
.../renderer/markdown/MarkdownRendererTest.java | 10 ++++++++++
.../renderer/markdown/SpecMarkdownRendererTest.java | 2 +-
4 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index c968da850..1f2943951 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -103,6 +103,8 @@ public void visit(FencedCodeBlock fencedCodeBlock) {
@Override
public void visit(HardLineBreak hardLineBreak) {
+ writer.write(" ");
+ writer.line();
}
@Override
@@ -157,6 +159,7 @@ public void visit(Paragraph paragraph) {
@Override
public void visit(SoftLineBreak softLineBreak) {
+ writer.line();
}
@Override
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
index fd79ce6af..5753026a8 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
@@ -22,6 +22,10 @@ public void block() {
prependLine = true;
}
+ public void line() {
+ append('\n');
+ }
+
public void write(String s) {
append(s);
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index 0a2983a90..619830aec 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -62,6 +62,16 @@ public void testStrongEmphasis() {
assertRoundTrip("foo**bar**\n");
}
+ @Test
+ public void testHardLineBreaks() {
+ assertRoundTrip("foo \nbar\n");
+ }
+
+ @Test
+ public void testSoftLineBreaks() {
+ assertRoundTrip("foo\nbar\n");
+ }
+
private Node parse(String source) {
return Parser.builder().build().parse(source);
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index c94465e7a..11a077b13 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -51,7 +51,7 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
- int expectedPassed = 226;
+ int expectedPassed = 263;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From 43006b2d572e30b1e054352fcf10600a297fc8cc Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 10 Jan 2024 17:37:45 +1100
Subject: [PATCH 029/286] Links and images
---
.../markdown/CoreMarkdownNodeRenderer.java | 41 +++++++++++++++++++
.../renderer/markdown/MarkdownRenderer.java | 2 +
.../renderer/markdown/MarkdownWriter.java | 22 ++++++++++
.../markdown/MarkdownRendererTest.java | 24 +++++++++++
.../markdown/SpecMarkdownRendererTest.java | 4 +-
5 files changed, 91 insertions(+), 2 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 1f2943951..376841ec7 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -1,5 +1,7 @@
package org.commonmark.renderer.markdown;
+import org.commonmark.internal.util.AsciiMatcher;
+import org.commonmark.internal.util.CharMatcher;
import org.commonmark.internal.util.Parsing;
import org.commonmark.node.*;
import org.commonmark.renderer.NodeRenderer;
@@ -13,6 +15,13 @@
*/
public class CoreMarkdownNodeRenderer extends AbstractVisitor implements NodeRenderer {
+ private final CharMatcher linkDestinationNeedsAngleBrackets =
+ AsciiMatcher.builder().c(' ').c('(').c(')').c('<').c('>').c('\\').build();
+ private final CharMatcher linkDestinationEscapeInAngleBrackets =
+ AsciiMatcher.builder().c('<').c('>').build();
+ private final CharMatcher linkTitleEscapeInQuotes =
+ AsciiMatcher.builder().c('"').build();
+
protected final MarkdownNodeRendererContext context;
private final MarkdownWriter writer;
@@ -133,6 +142,7 @@ public void visit(HtmlBlock htmlBlock) {
@Override
public void visit(Image image) {
+ writeLinkLike(image.getTitle(), image.getDestination(), image, "![");
}
@Override
@@ -141,6 +151,7 @@ public void visit(IndentedCodeBlock indentedCodeBlock) {
@Override
public void visit(Link link) {
+ writeLinkLike(link.getTitle(), link.getDestination(), link, "[");
}
@Override
@@ -199,7 +210,37 @@ private static int findMaxRunLength(char c, CharSequence s) {
return backticks;
}
+ private static boolean contains(String s, CharMatcher charMatcher) {
+ for (int i = 0; i < s.length(); i++) {
+ if (charMatcher.matches(s.charAt(i))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private void writeText(String text) {
writer.write(text);
}
+
+ private void writeLinkLike(String title, String destination, Node node, String opener) {
+ writer.write(opener);
+ visitChildren(node);
+ writer.write(']');
+ writer.write('(');
+ if (contains(destination, linkDestinationNeedsAngleBrackets)) {
+ writer.write('<');
+ writer.writeEscaped(destination, linkDestinationEscapeInAngleBrackets);
+ writer.write('>');
+ } else {
+ writer.write(destination);
+ }
+ if (title != null) {
+ writer.write(' ');
+ writer.write('"');
+ writer.writeEscaped(title, linkTitleEscapeInQuotes);
+ writer.write('"');
+ }
+ writer.write(')');
+ }
}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
index 6466b97a2..4dc8dbff9 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
@@ -15,6 +15,8 @@
* Note that it does not currently attempt to preserve the exact syntax of the original input Markdown (if any):
*
*
Headings are always output as ATX headings for simplicity
+ *
Escaping might be over-eager, e.g. a plain {@code *} might be escaped
+ * even though it doesn't need to be in that particular context
\n");
+ }
+
@Test
public void testParagraphs() {
assertRoundTrip("foo\n");
@@ -86,6 +91,11 @@ public void testImages() {
assertRoundTrip("\n");
}
+ @Test
+ public void testHtmlInline() {
+ assertRoundTrip("*foo*\n");
+ }
+
@Test
public void testHardLineBreaks() {
assertRoundTrip("foo \nbar\n");
From e4ced9b071a32c2c14c614a0cbcd3c2ae4b4c9e6 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 10 Jan 2024 18:15:53 +1100
Subject: [PATCH 031/286] Block quotes
---
.../markdown/CoreMarkdownNodeRenderer.java | 6 +++
.../renderer/markdown/MarkdownWriter.java | 41 ++++++++++++++-----
.../markdown/MarkdownRendererTest.java | 14 +++++++
.../markdown/SpecMarkdownRendererTest.java | 2 +-
4 files changed, 52 insertions(+), 11 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 77b6b0c17..04be11214 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -65,10 +65,16 @@ public void render(Node node) {
public void visit(Document document) {
// No rendering itself
visitChildren(document);
+ writer.line();
}
@Override
public void visit(BlockQuote blockQuote) {
+ writer.write("> ");
+ writer.pushPrefix("> ");
+ visitChildren(blockQuote);
+ writer.popPrefix();
+ writer.block();
}
@Override
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
index fd88b73ef..71c820559 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
@@ -3,13 +3,15 @@
import org.commonmark.internal.util.CharMatcher;
import java.io.IOException;
+import java.util.LinkedList;
public class MarkdownWriter {
private final Appendable buffer;
- private boolean prependLine = false;
+ private boolean finishBlock = false;
private char lastChar;
+ private final LinkedList prefixes = new LinkedList<>();
public MarkdownWriter(Appendable out) {
buffer = out;
@@ -20,19 +22,21 @@ public char getLastChar() {
}
public void block() {
- append('\n');
- prependLine = true;
+ finishBlock = true;
}
public void line() {
append('\n');
+ writePrefixes();
}
public void write(String s) {
+ finishBlockIfNeeded();
append(s);
}
public void write(char c) {
+ finishBlockIfNeeded();
append(c);
}
@@ -40,8 +44,8 @@ public void writeEscaped(String s, CharMatcher escape) {
if (s.isEmpty()) {
return;
}
+ finishBlockIfNeeded();
try {
- appendLineIfNeeded();
for (int i = 0; i < s.length(); i++) {
char ch = s.charAt(i);
if (ch == '\\' || escape.matches(ch)) {
@@ -56,9 +60,16 @@ public void writeEscaped(String s, CharMatcher escape) {
lastChar = s.charAt(s.length() - 1);
}
+ public void pushPrefix(String prefix) {
+ prefixes.addLast(prefix);
+ }
+
+ public void popPrefix() {
+ prefixes.removeLast();
+ }
+
private void append(String s) {
try {
- appendLineIfNeeded();
buffer.append(s);
} catch (IOException e) {
throw new RuntimeException(e);
@@ -72,7 +83,6 @@ private void append(String s) {
private void append(char c) {
try {
- appendLineIfNeeded();
buffer.append(c);
} catch (IOException e) {
throw new RuntimeException(e);
@@ -81,10 +91,21 @@ private void append(char c) {
lastChar = c;
}
- private void appendLineIfNeeded() throws IOException {
- if (prependLine) {
- buffer.append('\n');
- prependLine = false;
+ private void finishBlockIfNeeded() {
+ if (finishBlock) {
+ finishBlock = false;
+ append('\n');
+ writePrefixes();
+ append('\n');
+ writePrefixes();
+ }
+ }
+
+ private void writePrefixes() {
+ if (!prefixes.isEmpty()) {
+ for (String prefix : prefixes) {
+ append(prefix);
+ }
}
}
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index e9c300cbb..d2243d1b8 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -8,6 +8,8 @@
public class MarkdownRendererTest {
+ // Leaf blocks
+
@Test
public void testThematicBreaks() {
assertRoundTrip("***\n");
@@ -40,6 +42,18 @@ public void testParagraphs() {
assertRoundTrip("foo\n\nbar\n");
}
+ // Container blocks
+
+ @Test
+ public void testBlockQuotes() {
+ assertRoundTrip("> test\n");
+ assertRoundTrip("> foo\n> bar\n");
+ assertRoundTrip("> > foo\n> > bar\n");
+ assertRoundTrip("> # Foo\n> \n> bar\n> baz\n");
+ }
+
+ // Inlines
+
@Test
public void testCodeSpans() {
assertRoundTrip("`foo`\n");
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index c05aef846..e40ec457a 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -51,7 +51,7 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
- int expectedPassed = 372;
+ int expectedPassed = 459;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From f640e94142873b69729e7bdee54810b5d3c8a82b Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 10 Jan 2024 21:17:00 +1100
Subject: [PATCH 032/286] Indented code blocks
---
.../markdown/CoreMarkdownNodeRenderer.java | 15 +++++++++++++++
.../renderer/markdown/MarkdownRendererTest.java | 7 +++++++
.../markdown/SpecMarkdownRendererTest.java | 2 +-
3 files changed, 23 insertions(+), 1 deletion(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 04be11214..ad0d276ae 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -156,6 +156,21 @@ public void visit(Image image) {
@Override
public void visit(IndentedCodeBlock indentedCodeBlock) {
+ String literal = indentedCodeBlock.getLiteral();
+ String[] lines = literal.split("\n");
+ // We need to respect line prefixes which is why we need to write it line by line (e.g. an indented code block
+ // within a block quote)
+ writer.pushPrefix(" ");
+ writer.write(" ");
+ for (int i = 0; i < lines.length; i++) {
+ String line = lines[i];
+ writer.write(line);
+ if (i != lines.length - 1) {
+ writer.line();
+ }
+ }
+ writer.popPrefix();
+ writer.block();
}
@Override
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index d2243d1b8..e8f0bd7ac 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -31,6 +31,13 @@ public void testHeadings() {
assertRoundTrip("# foo\n\nbar\n");
}
+ @Test
+ public void testIndentedCodeBlocks() {
+ assertRoundTrip(" hi\n");
+ assertRoundTrip(" hi\n code\n");
+ assertRoundTrip("> hi\n> code\n");
+ }
+
@Test
public void testHtmlBlocks() {
assertRoundTrip("
test
\n");
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index e40ec457a..225c34525 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -51,7 +51,7 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
- int expectedPassed = 459;
+ int expectedPassed = 481;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From 31e36645fb3585dc500726c9ef775c2192f7edbb Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 10 Jan 2024 21:29:03 +1100
Subject: [PATCH 033/286] Fenced code blocks
---
.../markdown/CoreMarkdownNodeRenderer.java | 36 ++++++++++++++++++-
.../markdown/MarkdownRendererTest.java | 8 +++++
.../markdown/SpecMarkdownRendererTest.java | 2 +-
3 files changed, 44 insertions(+), 2 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index ad0d276ae..cec810e34 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -114,6 +114,32 @@ public void visit(Emphasis emphasis) {
@Override
public void visit(FencedCodeBlock fencedCodeBlock) {
+ String literal = fencedCodeBlock.getLiteral();
+ String fence = repeat(String.valueOf(fencedCodeBlock.getFenceChar()), fencedCodeBlock.getFenceLength());
+ int indent = fencedCodeBlock.getFenceIndent();
+
+ if (indent > 0) {
+ String indentPrefix = repeat(" ", indent);
+ writer.write(indentPrefix);
+ writer.pushPrefix(indentPrefix);
+ }
+
+ writer.write(fence);
+ if (fencedCodeBlock.getInfo() != null) {
+ writer.write(fencedCodeBlock.getInfo());
+ }
+ writer.line();
+ String[] lines = literal.split("\n");
+ for (int i = 0; i < lines.length; i++) {
+ String line = lines[i];
+ writer.write(line);
+ writer.line();
+ }
+ writer.write(fence);
+ if (indent > 0) {
+ writer.popPrefix();
+ }
+ writer.block();
}
@Override
@@ -160,8 +186,8 @@ public void visit(IndentedCodeBlock indentedCodeBlock) {
String[] lines = literal.split("\n");
// We need to respect line prefixes which is why we need to write it line by line (e.g. an indented code block
// within a block quote)
- writer.pushPrefix(" ");
writer.write(" ");
+ writer.pushPrefix(" ");
for (int i = 0; i < lines.length; i++) {
String line = lines[i];
writer.write(line);
@@ -243,6 +269,14 @@ private static boolean contains(String s, CharMatcher charMatcher) {
return false;
}
+ private static String repeat(String s, int count) {
+ StringBuilder sb = new StringBuilder(s.length() * count);
+ for (int i = 0; i < count; i++) {
+ sb.append(s);
+ }
+ return sb.toString();
+ }
+
private void writeText(String text) {
writer.write(text);
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index e8f0bd7ac..04b6cc262 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -38,6 +38,14 @@ public void testIndentedCodeBlocks() {
assertRoundTrip("> hi\n> code\n");
}
+ @Test
+ public void testFencedCodeBlocks() {
+ assertRoundTrip("```\ntest\n```\n");
+ assertRoundTrip("~~~~\ntest\n~~~~\n");
+ assertRoundTrip("```info\ntest\n```\n");
+ assertRoundTrip(" ```\n test\n ```\n");
+ }
+
@Test
public void testHtmlBlocks() {
assertRoundTrip("
test
\n");
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index 225c34525..703b52bfc 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -51,7 +51,7 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
- int expectedPassed = 481;
+ int expectedPassed = 507;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From bbde769f5723ab2b2ae5a0016670e82b1e608196 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 11 Jan 2024 16:08:42 +1100
Subject: [PATCH 034/286] Lists (not tight lists yet)
---
.../markdown/CoreMarkdownNodeRenderer.java | 67 ++++++++++++++++++-
.../markdown/MarkdownRendererTest.java | 19 ++++++
.../markdown/SpecMarkdownRendererTest.java | 2 +-
3 files changed, 84 insertions(+), 4 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index cec810e34..77bd855ea 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -24,6 +24,11 @@ public class CoreMarkdownNodeRenderer extends AbstractVisitor implements NodeRen
protected final MarkdownNodeRendererContext context;
private final MarkdownWriter writer;
+ /**
+ * If we're currently within a {@link BulletList} or {@link OrderedList}, this keeps the context of that list.
+ * It has a parent field so that it can represent a stack (for nested lists).
+ */
+ private ListHolder listHolder;
public CoreMarkdownNodeRenderer(MarkdownNodeRendererContext context) {
this.context = context;
@@ -79,6 +84,9 @@ public void visit(BlockQuote blockQuote) {
@Override
public void visit(BulletList bulletList) {
+ listHolder = new BulletListHolder(listHolder, bulletList);
+ visitChildren(bulletList);
+ listHolder = listHolder.parent;
}
@Override
@@ -130,8 +138,7 @@ public void visit(FencedCodeBlock fencedCodeBlock) {
}
writer.line();
String[] lines = literal.split("\n");
- for (int i = 0; i < lines.length; i++) {
- String line = lines[i];
+ for (String line : lines) {
writer.write(line);
writer.line();
}
@@ -206,16 +213,42 @@ public void visit(Link link) {
@Override
public void visit(ListItem listItem) {
+ boolean pushedPrefix = false;
+ if (listHolder instanceof BulletListHolder) {
+ BulletListHolder bulletListHolder = (BulletListHolder) listHolder;
+ String prefix = bulletListHolder.bulletMarker + " ";
+ writer.write(prefix);
+ writer.pushPrefix(repeat(" ", prefix.length()));
+ pushedPrefix = true;
+ } else if (listHolder instanceof OrderedListHolder) {
+ OrderedListHolder orderedListHolder = (OrderedListHolder) listHolder;
+ String prefix = String.valueOf(orderedListHolder.number) + orderedListHolder.delimiter + " ";
+ orderedListHolder.number++;
+ writer.write(prefix);
+ writer.pushPrefix(repeat(" ", prefix.length()));
+ pushedPrefix = true;
+ }
+ visitChildren(listItem);
+ if (pushedPrefix) {
+ writer.popPrefix();
+ }
}
@Override
public void visit(OrderedList orderedList) {
+ listHolder = new OrderedListHolder(listHolder, orderedList);
+ visitChildren(orderedList);
+ listHolder = listHolder.parent;
}
@Override
public void visit(Paragraph paragraph) {
visitChildren(paragraph);
- writer.block();
+ if (paragraph.getParent() instanceof ListItem) {
+ writer.block();
+ } else {
+ writer.block();
+ }
}
@Override
@@ -301,4 +334,32 @@ private void writeLinkLike(String title, String destination, Node node, String o
}
writer.write(')');
}
+
+ private static class ListHolder {
+ final ListHolder parent;
+
+ protected ListHolder(ListHolder parent) {
+ this.parent = parent;
+ }
+ }
+
+ private static class BulletListHolder extends ListHolder {
+ final char bulletMarker;
+
+ public BulletListHolder(ListHolder parent, BulletList bulletList) {
+ super(parent);
+ this.bulletMarker = bulletList.getBulletMarker();
+ }
+ }
+
+ private static class OrderedListHolder extends ListHolder {
+ final char delimiter;
+ private int number;
+
+ protected OrderedListHolder(ListHolder parent, OrderedList orderedList) {
+ super(parent);
+ delimiter = orderedList.getDelimiter();
+ number = orderedList.getStartNumber();
+ }
+ }
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index 04b6cc262..16719e8fc 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -67,6 +67,25 @@ public void testBlockQuotes() {
assertRoundTrip("> # Foo\n> \n> bar\n> baz\n");
}
+ @Test
+ public void testBulletListItems() {
+ assertRoundTrip("* foo\n");
+ assertRoundTrip("- foo\n");
+ assertRoundTrip("+ foo\n");
+ assertRoundTrip("* foo\n bar\n");
+ assertRoundTrip("* ```\n code\n ```\n");
+ assertRoundTrip("* foo\n\n* bar\n");
+
+ // Tight list
+// assertRoundTrip("* foo\n* bar\n");
+ }
+
+ @Test
+ public void testOrderedListItems() {
+ assertRoundTrip("1. foo\n");
+ assertRoundTrip("2. foo\n\n3. bar\n");
+ }
+
// Inlines
@Test
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index 703b52bfc..c980c8682 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -51,7 +51,7 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
- int expectedPassed = 507;
+ int expectedPassed = 564;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From b82d7cee637fe322a6afc726c60562f91021c37d Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 11 Jan 2024 16:24:17 +1100
Subject: [PATCH 035/286] Percent encode URLs in spec test (like in other spec
tests)
---
.../renderer/markdown/SpecMarkdownRendererTest.java | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index c980c8682..d2ab3c110 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -27,7 +27,8 @@
public class SpecMarkdownRendererTest {
public static final MarkdownRenderer MARKDOWN_RENDERER = MarkdownRenderer.builder().build();
- public static final HtmlRenderer HTML_RENDERER = HtmlRenderer.builder().build();
+ // The spec says URL-escaping is optional, but the examples assume that it's enabled.
+ public static final HtmlRenderer HTML_RENDERER = HtmlRenderer.builder().percentEncodeUrls(true).build();
@Test
public void testCoverage() {
@@ -51,7 +52,7 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
- int expectedPassed = 564;
+ int expectedPassed = 574;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From d0e89f4ea7bc6a0b870abe8006911fb32ebcc2b0 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 11 Jan 2024 16:44:00 +1100
Subject: [PATCH 036/286] Fix code spans with spaces
---
.../renderer/markdown/CoreMarkdownNodeRenderer.java | 5 ++++-
.../commonmark/renderer/markdown/MarkdownRendererTest.java | 2 ++
.../renderer/markdown/SpecMarkdownRendererTest.java | 2 +-
3 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 77bd855ea..682545c65 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -98,7 +98,10 @@ public void visit(Code code) {
writer.write('`');
}
// If the literal starts or ends with a backtick, surround it with a single space.
- boolean addSpace = literal.startsWith("`") || literal.endsWith("`");
+ // If it starts and ends with a space (but is not only spaces), add an additional space (otherwise they would
+ // get removed on parsing).
+ boolean addSpace = literal.startsWith("`") || literal.endsWith("`") ||
+ (literal.startsWith(" ") && literal.endsWith(" ") && Parsing.hasNonSpace(literal));
if (addSpace) {
writer.write(' ');
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index 16719e8fc..a0cea3878 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -95,6 +95,8 @@ public void testCodeSpans() {
assertRoundTrip("```foo `` ` bar```\n");
assertRoundTrip("`` `foo ``\n");
+ assertRoundTrip("`` ` ``\n");
+ assertRoundTrip("` `\n");
}
@Test
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index d2ab3c110..e03c991fe 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -52,7 +52,7 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
- int expectedPassed = 574;
+ int expectedPassed = 575;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From a4eecb5e29d6d12e0562a1eca9e788c4618239ac Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 11 Jan 2024 16:48:32 +1100
Subject: [PATCH 037/286] Escape special characters in normal text
---
.../renderer/markdown/CoreMarkdownNodeRenderer.java | 8 +++-----
.../renderer/markdown/MarkdownRendererTest.java | 8 ++++++++
.../renderer/markdown/SpecMarkdownRendererTest.java | 2 +-
3 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 682545c65..22aa171df 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -15,6 +15,8 @@
*/
public class CoreMarkdownNodeRenderer extends AbstractVisitor implements NodeRenderer {
+ private final CharMatcher textEscape =
+ AsciiMatcher.builder().c('[').c(']').c('<').c('>').c('`').build();
private final CharMatcher linkDestinationNeedsAngleBrackets =
AsciiMatcher.builder().c(' ').c('(').c(')').c('<').c('>').c('\\').build();
private final CharMatcher linkDestinationEscapeInAngleBrackets =
@@ -268,7 +270,7 @@ public void visit(StrongEmphasis strongEmphasis) {
@Override
public void visit(Text text) {
- writeText(text.getLiteral());
+ writer.writeEscaped(text.getLiteral(), textEscape);
}
@Override
@@ -313,10 +315,6 @@ private static String repeat(String s, int count) {
return sb.toString();
}
- private void writeText(String text) {
- writer.write(text);
- }
-
private void writeLinkLike(String title, String destination, Node node, String opener) {
writer.write(opener);
visitChildren(node);
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index a0cea3878..3c6925c03 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -88,6 +88,14 @@ public void testOrderedListItems() {
// Inlines
+ @Test
+ public void testEscaping() {
+ // These are a bit tricky. We always escape some characters, even though they only need escaping if they would
+ // otherwise result in a different parse result (e.g. a link):
+ assertRoundTrip("\\[a\\](/uri)\n");
+ assertRoundTrip("\\`abc\\`\n");
+ }
+
@Test
public void testCodeSpans() {
assertRoundTrip("`foo`\n");
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index e03c991fe..37253fb6f 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -52,7 +52,7 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
- int expectedPassed = 575;
+ int expectedPassed = 590;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From 9f7d9499a0b7e9ddc8fd880b3c5f1bf801ad59da Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 11 Jan 2024 17:03:45 +1100
Subject: [PATCH 038/286] Fix empty fenced code blocks
---
.../renderer/markdown/CoreMarkdownNodeRenderer.java | 10 ++++++----
.../renderer/markdown/MarkdownRendererTest.java | 1 +
.../renderer/markdown/SpecMarkdownRendererTest.java | 2 +-
3 files changed, 8 insertions(+), 5 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 22aa171df..34533c633 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -142,10 +142,12 @@ public void visit(FencedCodeBlock fencedCodeBlock) {
writer.write(fencedCodeBlock.getInfo());
}
writer.line();
- String[] lines = literal.split("\n");
- for (String line : lines) {
- writer.write(line);
- writer.line();
+ if (!literal.isEmpty()) {
+ String[] lines = literal.split("\n");
+ for (String line : lines) {
+ writer.write(line);
+ writer.line();
+ }
}
writer.write(fence);
if (indent > 0) {
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index 3c6925c03..a1c848ccd 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -44,6 +44,7 @@ public void testFencedCodeBlocks() {
assertRoundTrip("~~~~\ntest\n~~~~\n");
assertRoundTrip("```info\ntest\n```\n");
assertRoundTrip(" ```\n test\n ```\n");
+ assertRoundTrip("```\n```\n");
}
@Test
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index 37253fb6f..14b9b5196 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -52,7 +52,7 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
- int expectedPassed = 590;
+ int expectedPassed = 594;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From 29145aac3da21fb49286b46e7a7119a21bbed1d8 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 11 Jan 2024 17:13:28 +1100
Subject: [PATCH 039/286] Tight lists
---
.../markdown/CoreMarkdownNodeRenderer.java | 6 ++++++
.../renderer/markdown/MarkdownWriter.java | 15 +++++++++++++--
.../renderer/markdown/MarkdownRendererTest.java | 5 ++++-
.../markdown/SpecMarkdownRendererTest.java | 2 +-
4 files changed, 24 insertions(+), 4 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 34533c633..7068227dd 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -86,9 +86,12 @@ public void visit(BlockQuote blockQuote) {
@Override
public void visit(BulletList bulletList) {
+ boolean oldTight = writer.getTight();
+ writer.setTight(bulletList.isTight());
listHolder = new BulletListHolder(listHolder, bulletList);
visitChildren(bulletList);
listHolder = listHolder.parent;
+ writer.setTight(oldTight);
}
@Override
@@ -243,9 +246,12 @@ public void visit(ListItem listItem) {
@Override
public void visit(OrderedList orderedList) {
+ boolean oldTight = writer.getTight();
+ writer.setTight(orderedList.isTight());
listHolder = new OrderedListHolder(listHolder, orderedList);
visitChildren(orderedList);
listHolder = listHolder.parent;
+ writer.setTight(oldTight);
}
@Override
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
index 71c820559..c2c826195 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
@@ -10,6 +10,7 @@ public class MarkdownWriter {
private final Appendable buffer;
private boolean finishBlock = false;
+ private boolean tight;
private char lastChar;
private final LinkedList prefixes = new LinkedList<>();
@@ -96,8 +97,10 @@ private void finishBlockIfNeeded() {
finishBlock = false;
append('\n');
writePrefixes();
- append('\n');
- writePrefixes();
+ if (!tight) {
+ append('\n');
+ writePrefixes();
+ }
}
}
@@ -108,4 +111,12 @@ private void writePrefixes() {
}
}
}
+
+ public boolean getTight() {
+ return tight;
+ }
+
+ public void setTight(boolean tight) {
+ this.tight = tight;
+ }
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index a1c848ccd..b99209ef9 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -78,13 +78,16 @@ public void testBulletListItems() {
assertRoundTrip("* foo\n\n* bar\n");
// Tight list
-// assertRoundTrip("* foo\n* bar\n");
+ assertRoundTrip("* foo\n* bar\n");
}
@Test
public void testOrderedListItems() {
assertRoundTrip("1. foo\n");
assertRoundTrip("2. foo\n\n3. bar\n");
+
+ // Tight list
+ assertRoundTrip("1. foo\n2. bar\n");
}
// Inlines
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index 14b9b5196..e004366f6 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -52,7 +52,7 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
- int expectedPassed = 594;
+ int expectedPassed = 611;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From 8171519f9b89a1257eabffde7e68edeb832fc72b Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 17 Jan 2024 16:58:26 +1100
Subject: [PATCH 040/286] Preserve indent and content indent for lists
Makes use of #303 which was split out.
---
.../markdown/CoreMarkdownNodeRenderer.java | 15 +++++++++------
.../renderer/markdown/MarkdownRendererTest.java | 16 +++++++++++++++-
.../markdown/SpecMarkdownRendererTest.java | 2 +-
3 files changed, 25 insertions(+), 8 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 7068227dd..c583e6579 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -223,19 +223,22 @@ public void visit(Link link) {
@Override
public void visit(ListItem listItem) {
+ int contentIndent = listItem.getContentIndent();
boolean pushedPrefix = false;
if (listHolder instanceof BulletListHolder) {
BulletListHolder bulletListHolder = (BulletListHolder) listHolder;
- String prefix = bulletListHolder.bulletMarker + " ";
- writer.write(prefix);
- writer.pushPrefix(repeat(" ", prefix.length()));
+ String marker = repeat(" ", listItem.getMarkerIndent()) + bulletListHolder.bulletMarker;
+ writer.write(marker);
+ writer.write(repeat(" ", contentIndent - marker.length()));
+ writer.pushPrefix(repeat(" ", contentIndent));
pushedPrefix = true;
} else if (listHolder instanceof OrderedListHolder) {
OrderedListHolder orderedListHolder = (OrderedListHolder) listHolder;
- String prefix = String.valueOf(orderedListHolder.number) + orderedListHolder.delimiter + " ";
+ String marker = repeat(" ", listItem.getMarkerIndent()) + orderedListHolder.number + orderedListHolder.delimiter;
orderedListHolder.number++;
- writer.write(prefix);
- writer.pushPrefix(repeat(" ", prefix.length()));
+ writer.write(marker);
+ writer.write(repeat(" ", contentIndent - marker.length()));
+ writer.pushPrefix(repeat(" ", contentIndent));
pushedPrefix = true;
}
visitChildren(listItem);
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index b99209ef9..5b3148a2d 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -76,9 +76,20 @@ public void testBulletListItems() {
assertRoundTrip("* foo\n bar\n");
assertRoundTrip("* ```\n code\n ```\n");
assertRoundTrip("* foo\n\n* bar\n");
+ // Note that the " " in the second line is not necessary, but it's not wrong either.
+ // We could try to avoid it in a future change, but not sure if necessary.
+ assertRoundTrip("* foo\n \n bar\n");
// Tight list
assertRoundTrip("* foo\n* bar\n");
+
+ // List item indent. This is a tricky one, but here the amount of space between the list marker and "one"
+ // determines whether "two" is part of the list item or an indented code block.
+ // In this case, it's an indented code block because it's not indented enough to be part of the list item.
+ // If the renderer would just use "- one", then "two" would change from being an indented code block to being
+ // a paragraph in the list item! So it is important for the renderer to preserve the content indent of the list
+ // item.
+ assertRoundTrip(" - one\n\n two\n");
}
@Test
@@ -88,6 +99,8 @@ public void testOrderedListItems() {
// Tight list
assertRoundTrip("1. foo\n2. bar\n");
+
+ assertRoundTrip(" 1. one\n\n two\n");
}
// Inlines
@@ -173,7 +186,8 @@ private Node parse(String source) {
}
private String render(String source) {
- return MarkdownRenderer.builder().build().render(parse(source));
+ Node parsed = parse(source);
+ return MarkdownRenderer.builder().build().render(parsed);
}
private void assertRoundTrip(String input) {
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index e004366f6..fbd3bc1a6 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -52,7 +52,7 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
- int expectedPassed = 611;
+ int expectedPassed = 613;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From 41eeede32bb189d41b4c7f1fb7dfef12057d438e Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 17 Jan 2024 17:01:49 +1100
Subject: [PATCH 041/286] Fix empty list items
---
.../renderer/markdown/CoreMarkdownNodeRenderer.java | 7 ++++++-
.../commonmark/renderer/markdown/MarkdownRendererTest.java | 3 +++
.../renderer/markdown/SpecMarkdownRendererTest.java | 2 +-
3 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index c583e6579..28461c870 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -241,7 +241,12 @@ public void visit(ListItem listItem) {
writer.pushPrefix(repeat(" ", contentIndent));
pushedPrefix = true;
}
- visitChildren(listItem);
+ if (listItem.getFirstChild() == null) {
+ // Empty list item
+ writer.block();
+ } else {
+ visitChildren(listItem);
+ }
if (pushedPrefix) {
writer.popPrefix();
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index 5b3148a2d..06ad79c25 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -90,6 +90,9 @@ public void testBulletListItems() {
// a paragraph in the list item! So it is important for the renderer to preserve the content indent of the list
// item.
assertRoundTrip(" - one\n\n two\n");
+
+ // Empty list
+ assertRoundTrip("- \n\nFoo\n");
}
@Test
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index fbd3bc1a6..682fee9d8 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -52,7 +52,7 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
- int expectedPassed = 613;
+ int expectedPassed = 618;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From 12ce6bc4244d9f0c06b2ff3bab753e6a42c0907e Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 17 Jan 2024 22:43:30 +1100
Subject: [PATCH 042/286] Fix tight list with nested loose list
---
.../markdown/CoreMarkdownNodeRenderer.java | 90 +++++++++----------
.../renderer/markdown/MarkdownWriter.java | 70 +++++++++------
.../markdown/MarkdownRendererTest.java | 4 +
.../markdown/SpecMarkdownRendererTest.java | 2 +-
4 files changed, 94 insertions(+), 72 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 28461c870..5e8833331 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -92,6 +92,49 @@ public void visit(BulletList bulletList) {
visitChildren(bulletList);
listHolder = listHolder.parent;
writer.setTight(oldTight);
+ writer.block();
+ }
+
+ @Override
+ public void visit(OrderedList orderedList) {
+ boolean oldTight = writer.getTight();
+ writer.setTight(orderedList.isTight());
+ listHolder = new OrderedListHolder(listHolder, orderedList);
+ visitChildren(orderedList);
+ listHolder = listHolder.parent;
+ writer.setTight(oldTight);
+ writer.block();
+ }
+
+ @Override
+ public void visit(ListItem listItem) {
+ int contentIndent = listItem.getContentIndent();
+ boolean pushedPrefix = false;
+ if (listHolder instanceof BulletListHolder) {
+ BulletListHolder bulletListHolder = (BulletListHolder) listHolder;
+ String marker = repeat(" ", listItem.getMarkerIndent()) + bulletListHolder.bulletMarker;
+ writer.write(marker);
+ writer.write(repeat(" ", contentIndent - marker.length()));
+ writer.pushPrefix(repeat(" ", contentIndent));
+ pushedPrefix = true;
+ } else if (listHolder instanceof OrderedListHolder) {
+ OrderedListHolder orderedListHolder = (OrderedListHolder) listHolder;
+ String marker = repeat(" ", listItem.getMarkerIndent()) + orderedListHolder.number + orderedListHolder.delimiter;
+ orderedListHolder.number++;
+ writer.write(marker);
+ writer.write(repeat(" ", contentIndent - marker.length()));
+ writer.pushPrefix(repeat(" ", contentIndent));
+ pushedPrefix = true;
+ }
+ if (listItem.getFirstChild() == null) {
+ // Empty list item
+ writer.block();
+ } else {
+ visitChildren(listItem);
+ }
+ if (pushedPrefix) {
+ writer.popPrefix();
+ }
}
@Override
@@ -221,55 +264,10 @@ public void visit(Link link) {
writeLinkLike(link.getTitle(), link.getDestination(), link, "[");
}
- @Override
- public void visit(ListItem listItem) {
- int contentIndent = listItem.getContentIndent();
- boolean pushedPrefix = false;
- if (listHolder instanceof BulletListHolder) {
- BulletListHolder bulletListHolder = (BulletListHolder) listHolder;
- String marker = repeat(" ", listItem.getMarkerIndent()) + bulletListHolder.bulletMarker;
- writer.write(marker);
- writer.write(repeat(" ", contentIndent - marker.length()));
- writer.pushPrefix(repeat(" ", contentIndent));
- pushedPrefix = true;
- } else if (listHolder instanceof OrderedListHolder) {
- OrderedListHolder orderedListHolder = (OrderedListHolder) listHolder;
- String marker = repeat(" ", listItem.getMarkerIndent()) + orderedListHolder.number + orderedListHolder.delimiter;
- orderedListHolder.number++;
- writer.write(marker);
- writer.write(repeat(" ", contentIndent - marker.length()));
- writer.pushPrefix(repeat(" ", contentIndent));
- pushedPrefix = true;
- }
- if (listItem.getFirstChild() == null) {
- // Empty list item
- writer.block();
- } else {
- visitChildren(listItem);
- }
- if (pushedPrefix) {
- writer.popPrefix();
- }
- }
-
- @Override
- public void visit(OrderedList orderedList) {
- boolean oldTight = writer.getTight();
- writer.setTight(orderedList.isTight());
- listHolder = new OrderedListHolder(listHolder, orderedList);
- visitChildren(orderedList);
- listHolder = listHolder.parent;
- writer.setTight(oldTight);
- }
-
@Override
public void visit(Paragraph paragraph) {
visitChildren(paragraph);
- if (paragraph.getParent() instanceof ListItem) {
- writer.block();
- } else {
- writer.block();
- }
+ writer.block();
}
@Override
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
index c2c826195..ba682d465 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
@@ -9,7 +9,7 @@ public class MarkdownWriter {
private final Appendable buffer;
- private boolean finishBlock = false;
+ private int blockSeparator = 0;
private boolean tight;
private char lastChar;
private final LinkedList prefixes = new LinkedList<>();
@@ -22,22 +22,13 @@ public char getLastChar() {
return lastChar;
}
- public void block() {
- finishBlock = true;
- }
-
- public void line() {
- append('\n');
- writePrefixes();
- }
-
public void write(String s) {
- finishBlockIfNeeded();
+ flushBlockSeparator();
append(s);
}
public void write(char c) {
- finishBlockIfNeeded();
+ flushBlockSeparator();
append(c);
}
@@ -45,7 +36,7 @@ public void writeEscaped(String s, CharMatcher escape) {
if (s.isEmpty()) {
return;
}
- finishBlockIfNeeded();
+ flushBlockSeparator();
try {
for (int i = 0; i < s.length(); i++) {
char ch = s.charAt(i);
@@ -61,6 +52,21 @@ public void writeEscaped(String s, CharMatcher escape) {
lastChar = s.charAt(s.length() - 1);
}
+ public void line() {
+ append('\n');
+ writePrefixes();
+ }
+
+ /**
+ * Enqueue a block separator to be written before the next text is written. Block separators are not written
+ * straight away because if there are no more blocks to write we don't want a separator (at the end of the document).
+ */
+ public void block() {
+ // Remember whether this should be a tight or loose separator now because tight could get changed in between
+ // this and the next flush.
+ blockSeparator = tight ? 1 : 2;
+ }
+
public void pushPrefix(String prefix) {
prefixes.addLast(prefix);
}
@@ -92,18 +98,6 @@ private void append(char c) {
lastChar = c;
}
- private void finishBlockIfNeeded() {
- if (finishBlock) {
- finishBlock = false;
- append('\n');
- writePrefixes();
- if (!tight) {
- append('\n');
- writePrefixes();
- }
- }
- }
-
private void writePrefixes() {
if (!prefixes.isEmpty()) {
for (String prefix : prefixes) {
@@ -112,10 +106,36 @@ private void writePrefixes() {
}
}
+ /**
+ * If a block separator has been enqueued with {@link #block()} but not yet written, write it now.
+ */
+ private void flushBlockSeparator() {
+ if (blockSeparator != 0) {
+ append('\n');
+ writePrefixes();
+ if (blockSeparator > 1) {
+ append('\n');
+ writePrefixes();
+ }
+ blockSeparator = 0;
+ }
+ }
+
+ /**
+ * @return whether blocks are currently set to tight or loose, see {@link #setTight(boolean)}
+ */
public boolean getTight() {
return tight;
}
+ /**
+ * Change whether blocks are tight or loose. Loose is the default where blocks are separated by a blank line. Tight
+ * is where blocks are not separated by a blank line. Tight blocks are used in lists, if there are no blank lines
+ * within the list.
+ *
+ * Note that changing this does not affect block separators that have already been enqueued (with {@link #block()},
+ * only future ones.
+ */
public void setTight(boolean tight) {
this.tight = tight;
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index 06ad79c25..f6ce1b4e0 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -82,6 +82,8 @@ public void testBulletListItems() {
// Tight list
assertRoundTrip("* foo\n* bar\n");
+ // Tight list where the second item contains a loose list
+ assertRoundTrip("- Foo\n - Bar\n \n - Baz\n");
// List item indent. This is a tricky one, but here the amount of space between the list marker and "one"
// determines whether "two" is part of the list item or an indented code block.
@@ -102,6 +104,8 @@ public void testOrderedListItems() {
// Tight list
assertRoundTrip("1. foo\n2. bar\n");
+ // Tight list where the second item contains a loose list
+ assertRoundTrip("1. Foo\n 1. Bar\n \n 2. Baz\n");
assertRoundTrip(" 1. one\n\n two\n");
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index 682fee9d8..e21ec8cc7 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -52,7 +52,7 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
- int expectedPassed = 618;
+ int expectedPassed = 621;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From 6417d012e5476b8abeaf89b5b19f35c2d9477a32 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 17 Jan 2024 22:54:06 +1100
Subject: [PATCH 043/286] Print failing cases
---
.../renderer/markdown/SpecMarkdownRendererTest.java | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index e21ec8cc7..d50349d34 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -51,6 +51,16 @@ public void testCoverage() {
System.out.println("Failed examples by section (total " + fails.size() + "):");
printCountsBySection(fails);
+ System.out.println();
+
+ System.out.println("Failed examples:");
+ for (Example fail : fails) {
+ System.out.println("Failed: " + fail);
+ System.out.println("````````````````````````````````");
+ System.out.print(fail.getSource());
+ System.out.println("````````````````````````````````");
+ System.out.println();
+ }
int expectedPassed = 621;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
From 26ac7e1c42821cd166e7cf91d657936dcc63accb Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Mon, 22 Jan 2024 22:09:49 +1100
Subject: [PATCH 044/286] Don't discard trailing empty lines in code blocks
---
.../markdown/CoreMarkdownNodeRenderer.java | 25 +++++++++++++++----
.../markdown/SpecMarkdownRendererTest.java | 2 +-
2 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 5e8833331..a135d73ee 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -8,6 +8,7 @@
import java.util.Arrays;
import java.util.HashSet;
+import java.util.List;
import java.util.Set;
/**
@@ -189,7 +190,7 @@ public void visit(FencedCodeBlock fencedCodeBlock) {
}
writer.line();
if (!literal.isEmpty()) {
- String[] lines = literal.split("\n");
+ List lines = getLines(literal);
for (String line : lines) {
writer.write(line);
writer.line();
@@ -243,15 +244,15 @@ public void visit(Image image) {
@Override
public void visit(IndentedCodeBlock indentedCodeBlock) {
String literal = indentedCodeBlock.getLiteral();
- String[] lines = literal.split("\n");
// We need to respect line prefixes which is why we need to write it line by line (e.g. an indented code block
// within a block quote)
writer.write(" ");
writer.pushPrefix(" ");
- for (int i = 0; i < lines.length; i++) {
- String line = lines[i];
+ List lines = getLines(literal);
+ for (int i = 0; i < lines.size(); i++) {
+ String line = lines.get(i);
writer.write(line);
- if (i != lines.length - 1) {
+ if (i != lines.size() - 1) {
writer.line();
}
}
@@ -329,6 +330,20 @@ private static String repeat(String s, int count) {
return sb.toString();
}
+ private static List getLines(String literal) {
+ // Without -1, split would discard all trailing empty strings, which is not what we want, e.g. it would
+ // return the same result for "abc", "abc\n" and "abc\n\n".
+ // With -1, it returns ["abc"], ["abc", ""] and ["abc", "", ""].
+ String[] parts = literal.split("\n", -1);
+ if (parts[parts.length - 1].isEmpty()) {
+ // But we don't want the last empty string, as "\n" is used as a line terminator (not a separator),
+ // so return without the last element.
+ return Arrays.asList(parts).subList(0, parts.length - 1);
+ } else {
+ return Arrays.asList(parts);
+ }
+ }
+
private void writeLinkLike(String title, String destination, Node node, String opener) {
writer.write(opener);
visitChildren(node);
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index d50349d34..4e9396e77 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -62,7 +62,7 @@ public void testCoverage() {
System.out.println();
}
- int expectedPassed = 621;
+ int expectedPassed = 622;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From 724619b0efc5937b4fba974769cea7fa6049dffc Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Mon, 22 Jan 2024 22:46:33 +1100
Subject: [PATCH 045/286] Use Setext heading if necessary
---
.../markdown/CoreMarkdownNodeRenderer.java | 47 +++++++++++++++++++
.../renderer/markdown/MarkdownRenderer.java | 2 +-
.../markdown/MarkdownRendererTest.java | 3 ++
.../markdown/SpecMarkdownRendererTest.java | 2 +-
4 files changed, 52 insertions(+), 2 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index a135d73ee..dbaa9a47c 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -13,6 +13,10 @@
/**
* The node renderer that renders all the core nodes (comes last in the order of node renderers).
+ *
+ * Note that while sometimes it would be easier to record what kind of syntax was used on parsing (e.g. ATX vs Setext
+ * heading), this renderer is intended to also work for documents that were created by directly creating
+ * {@link Node Nodes} instead. So in order to support that, it sometimes needs to do a bit more work.
*/
public class CoreMarkdownNodeRenderer extends AbstractVisitor implements NodeRenderer {
@@ -211,11 +215,34 @@ public void visit(HardLineBreak hardLineBreak) {
@Override
public void visit(Heading heading) {
+ if (heading.getLevel() <= 2) {
+ LineBreakVisitor lineBreakVisitor = new LineBreakVisitor();
+ heading.accept(lineBreakVisitor);
+ boolean isMultipleLines = lineBreakVisitor.hasLineBreak();
+
+ if (isMultipleLines) {
+ // Setext headings: Can have multiple lines, but only level 1 or 2
+ visitChildren(heading);
+ writer.line();
+ if (heading.getLevel() == 1) {
+ // Note that it would be nice to match the length of the contents instead of just using 3, but that's
+ // not easy.
+ writer.write("===");
+ } else {
+ writer.write("---");
+ }
+ writer.block();
+ return;
+ }
+ }
+
+ // ATX headings: Can't have multiple lines, but up to level 6.
for (int i = 0; i < heading.getLevel(); i++) {
writer.write('#');
}
writer.write(' ');
visitChildren(heading);
+
writer.block();
}
@@ -392,4 +419,24 @@ protected OrderedListHolder(ListHolder parent, OrderedList orderedList) {
number = orderedList.getStartNumber();
}
}
+
+ private static class LineBreakVisitor extends AbstractVisitor {
+ private boolean lineBreak = false;
+
+ public boolean hasLineBreak() {
+ return lineBreak;
+ }
+
+ @Override
+ public void visit(SoftLineBreak softLineBreak) {
+ super.visit(softLineBreak);
+ lineBreak = true;
+ }
+
+ @Override
+ public void visit(HardLineBreak hardLineBreak) {
+ super.visit(hardLineBreak);
+ lineBreak = true;
+ }
+ }
}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
index 4dc8dbff9..4dee17ed6 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
@@ -14,7 +14,7 @@
*
* Note that it does not currently attempt to preserve the exact syntax of the original input Markdown (if any):
*
- *
Headings are always output as ATX headings for simplicity
+ *
Headings are output as ATX headings if possible (multi-line headings need Setext headings)
*
Escaping might be over-eager, e.g. a plain {@code *} might be escaped
* even though it doesn't need to be in that particular context
*
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index f6ce1b4e0..05a253fde 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -28,6 +28,9 @@ public void testHeadings() {
assertRoundTrip("##### foo\n");
assertRoundTrip("###### foo\n");
+ assertRoundTrip("Foo\nbar\n===\n");
+ assertRoundTrip("[foo\nbar](/url)\n===\n");
+
assertRoundTrip("# foo\n\nbar\n");
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index 4e9396e77..1dd064414 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -62,7 +62,7 @@ public void testCoverage() {
System.out.println();
}
- int expectedPassed = 622;
+ int expectedPassed = 625;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From 02670c91eebb750dd1e2b65fea40382e9fe1be57 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Mon, 22 Jan 2024 23:02:59 +1100
Subject: [PATCH 046/286] Test tabs properly
---
.../commonmark/renderer/markdown/MarkdownRendererTest.java | 5 +++++
.../renderer/markdown/SpecMarkdownRendererTest.java | 5 +++--
2 files changed, 8 insertions(+), 2 deletions(-)
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index 05a253fde..ae3396680 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -115,6 +115,11 @@ public void testOrderedListItems() {
// Inlines
+ @Test
+ public void testTabs() {
+ assertRoundTrip("a\tb\n");
+ }
+
@Test
public void testEscaping() {
// These are a bit tricky. We always escape some characters, even though they only need escaping if they would
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index 1dd064414..de657e2cd 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -62,7 +62,7 @@ public void testCoverage() {
System.out.println();
}
- int expectedPassed = 625;
+ int expectedPassed = 629;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
@@ -89,6 +89,7 @@ private String renderMarkdown(String source) {
}
private String renderHtml(String source) {
- return HTML_RENDERER.render(parse(source));
+ // The spec uses "rightwards arrow" to show tabs
+ return HTML_RENDERER.render(parse(source)).replace("\t", "\u2192");
}
}
From 4918d64d592280490b54c77de066ec15c8582b1a Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Mon, 22 Jan 2024 23:10:31 +1100
Subject: [PATCH 047/286] Write HTML block in lines to make use of prefix
---
.../renderer/markdown/CoreMarkdownNodeRenderer.java | 9 ++++++++-
.../renderer/markdown/MarkdownRendererTest.java | 1 +
.../renderer/markdown/SpecMarkdownRendererTest.java | 2 +-
3 files changed, 10 insertions(+), 2 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index dbaa9a47c..813efb164 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -259,7 +259,14 @@ public void visit(HtmlInline htmlInline) {
@Override
public void visit(HtmlBlock htmlBlock) {
- writer.write(htmlBlock.getLiteral());
+ List lines = getLines(htmlBlock.getLiteral());
+ for (int i = 0; i < lines.size(); i++) {
+ String line = lines.get(i);
+ writer.write(line);
+ if (i != lines.size() - 1) {
+ writer.line();
+ }
+ }
writer.block();
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index ae3396680..76a77dbb5 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -53,6 +53,7 @@ public void testFencedCodeBlocks() {
@Test
public void testHtmlBlocks() {
assertRoundTrip("
test
\n");
+ assertRoundTrip(">
\n> test\n>
\n");
}
@Test
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index de657e2cd..2a0c46c00 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -62,7 +62,7 @@ public void testCoverage() {
System.out.println();
}
- int expectedPassed = 629;
+ int expectedPassed = 630;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From db5f055f0c456bd852601bd0de61336c75bd9bd7 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 23 Jan 2024 22:43:20 +1100
Subject: [PATCH 048/286] Escape special characters at beginning of line
---
.../internal/util/AsciiMatcher.java | 7 +++
.../markdown/CoreMarkdownNodeRenderer.java | 58 ++++++++++++++++++-
.../renderer/markdown/MarkdownWriter.java | 6 +-
.../markdown/MarkdownRendererTest.java | 16 +++++
.../markdown/SpecMarkdownRendererTest.java | 2 +-
5 files changed, 84 insertions(+), 5 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java b/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java
index 82d83ca46..35769f82d 100644
--- a/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java
+++ b/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java
@@ -29,6 +29,13 @@ private Builder(BitSet set) {
this.set = set;
}
+ public Builder anyOf(String s) {
+ for (int i = 0; i < s.length(); i++) {
+ c(s.charAt(i));
+ }
+ return this;
+ }
+
public Builder c(char c) {
if (c > 127) {
throw new IllegalArgumentException("Can only match ASCII characters");
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 813efb164..fe5725aee 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -10,6 +10,8 @@
import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
* The node renderer that renders all the core nodes (comes last in the order of node renderers).
@@ -20,8 +22,8 @@
*/
public class CoreMarkdownNodeRenderer extends AbstractVisitor implements NodeRenderer {
- private final CharMatcher textEscape =
- AsciiMatcher.builder().c('[').c(']').c('<').c('>').c('`').build();
+ private final AsciiMatcher textEscape =
+ AsciiMatcher.builder().anyOf("[]<>`*&").build();
private final CharMatcher linkDestinationNeedsAngleBrackets =
AsciiMatcher.builder().c(' ').c('(').c(')').c('<').c('>').c('\\').build();
private final CharMatcher linkDestinationEscapeInAngleBrackets =
@@ -29,6 +31,8 @@ public class CoreMarkdownNodeRenderer extends AbstractVisitor implements NodeRen
private final CharMatcher linkTitleEscapeInQuotes =
AsciiMatcher.builder().c('"').build();
+ private final Pattern orderedListMarkerPattern = Pattern.compile("^([0-9]{1,9})([.)])");
+
protected final MarkdownNodeRendererContext context;
private final MarkdownWriter writer;
/**
@@ -319,7 +323,55 @@ public void visit(StrongEmphasis strongEmphasis) {
@Override
public void visit(Text text) {
- writer.writeEscaped(text.getLiteral(), textEscape);
+ String literal = text.getLiteral();
+ if (writer.isAtLineStart() && !literal.isEmpty()) {
+ char c = literal.charAt(0);
+ switch (c) {
+ case '-': {
+ // Would be ambiguous with a bullet list marker, escape
+ writer.write("\\-");
+ literal = literal.substring(1);
+ break;
+ }
+ case '#': {
+ // Would be ambiguous with an ATX heading, escape
+ writer.write("\\#");
+ literal = literal.substring(1);
+ break;
+ }
+ case '=': {
+ // Would be ambiguous with a Setext heading, escape
+ writer.write("\\=");
+ literal = literal.substring(1);
+ break;
+ }
+ case '0':
+ case '1':
+ case '2':
+ case '3':
+ case '4':
+ case '5':
+ case '6':
+ case '7':
+ case '8':
+ case '9': {
+ // Check for ordered list marker
+ Matcher m = orderedListMarkerPattern.matcher(literal);
+ if (m.find()) {
+ writer.write(m.group(1));
+ writer.write("\\" + m.group(2));
+ literal = literal.substring(m.end());
+ }
+ }
+ }
+ }
+
+ if (literal.endsWith("!") && text.getNext() instanceof Link) {
+ writer.writeEscaped(literal.substring(0, literal.length() - 1), textEscape);
+ writer.write("\\!");
+ } else {
+ writer.writeEscaped(literal, textEscape);
+ }
}
@Override
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
index ba682d465..bc10f020e 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
@@ -11,7 +11,7 @@ public class MarkdownWriter {
private int blockSeparator = 0;
private boolean tight;
- private char lastChar;
+ private char lastChar = '\n';
private final LinkedList prefixes = new LinkedList<>();
public MarkdownWriter(Appendable out) {
@@ -22,6 +22,10 @@ public char getLastChar() {
return lastChar;
}
+ public boolean isAtLineStart() {
+ return lastChar == '\n' || blockSeparator > 0;
+ }
+
public void write(String s) {
flushBlockSeparator();
append(s);
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index 76a77dbb5..59039799a 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -127,6 +127,22 @@ public void testEscaping() {
// otherwise result in a different parse result (e.g. a link):
assertRoundTrip("\\[a\\](/uri)\n");
assertRoundTrip("\\`abc\\`\n");
+
+ // Some characters only need to be escaped at the beginning of the line
+ assertRoundTrip("\\- Test\n");
+ assertRoundTrip("\\-\n");
+ assertRoundTrip("Test -\n");
+ assertRoundTrip("Abc\n\n\\- Test\n");
+ assertRoundTrip("\\# Test\n");
+ assertRoundTrip("\\## Test\n");
+ assertRoundTrip("\\#\n");
+ assertRoundTrip("Foo\n\\===\n");
+
+ // This is a bit more tricky as we need to check for a list start
+ assertRoundTrip("1\\. Foo\n");
+ assertRoundTrip("999\\. Foo\n");
+ assertRoundTrip("1\\.\n");
+ assertRoundTrip("1\\) Foo\n");
}
@Test
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index 2a0c46c00..632f4acfc 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -62,7 +62,7 @@ public void testCoverage() {
System.out.println();
}
- int expectedPassed = 630;
+ int expectedPassed = 646;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From 64d163d2e6b2719affe4fd40d31eefa44bf52711 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 23 Jan 2024 22:45:29 +1100
Subject: [PATCH 049/286] Escape # in headings too
---
.../java/org/commonmark/internal/util/AsciiMatcher.java | 4 ++++
.../renderer/markdown/CoreMarkdownNodeRenderer.java | 8 ++++++--
.../renderer/markdown/SpecMarkdownRendererTest.java | 2 +-
3 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java b/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java
index 35769f82d..d31020fa3 100644
--- a/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java
+++ b/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java
@@ -22,6 +22,10 @@ public static Builder builder() {
return new Builder(new BitSet());
}
+ public static Builder builder(AsciiMatcher matcher) {
+ return new Builder((BitSet) matcher.set.clone());
+ }
+
public static class Builder {
private final BitSet set;
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index fe5725aee..8cb67e0fd 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -24,6 +24,8 @@ public class CoreMarkdownNodeRenderer extends AbstractVisitor implements NodeRen
private final AsciiMatcher textEscape =
AsciiMatcher.builder().anyOf("[]<>`*&").build();
+ private final CharMatcher textEscapeInHeading =
+ AsciiMatcher.builder(textEscape).anyOf("#").build();
private final CharMatcher linkDestinationNeedsAngleBrackets =
AsciiMatcher.builder().c(' ').c('(').c(')').c('<').c('>').c('\\').build();
private final CharMatcher linkDestinationEscapeInAngleBrackets =
@@ -366,11 +368,13 @@ public void visit(Text text) {
}
}
+ CharMatcher escape = text.getParent() instanceof Heading ? textEscapeInHeading : textEscape;
+
if (literal.endsWith("!") && text.getNext() instanceof Link) {
- writer.writeEscaped(literal.substring(0, literal.length() - 1), textEscape);
+ writer.writeEscaped(literal.substring(0, literal.length() - 1), escape);
writer.write("\\!");
} else {
- writer.writeEscaped(literal, textEscape);
+ writer.writeEscaped(literal, escape);
}
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index 632f4acfc..d6bdf9cfb 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -62,7 +62,7 @@ public void testCoverage() {
System.out.println();
}
- int expectedPassed = 646;
+ int expectedPassed = 647;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From f5b04efdae2c48e46f8f108df71308a7521ac223 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 23 Jan 2024 23:35:48 +1100
Subject: [PATCH 050/286] Disregard prefixes for line start logic
---
.../markdown/CoreMarkdownNodeRenderer.java | 14 +++++++-------
.../renderer/markdown/MarkdownWriter.java | 19 +++++++++++++++++--
.../markdown/MarkdownRendererTest.java | 5 +++++
.../markdown/SpecMarkdownRendererTest.java | 2 +-
4 files changed, 30 insertions(+), 10 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 8cb67e0fd..11e7ae5f0 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -88,7 +88,7 @@ public void visit(Document document) {
@Override
public void visit(BlockQuote blockQuote) {
- writer.write("> ");
+ writer.writePrefix("> ");
writer.pushPrefix("> ");
visitChildren(blockQuote);
writer.popPrefix();
@@ -124,16 +124,16 @@ public void visit(ListItem listItem) {
if (listHolder instanceof BulletListHolder) {
BulletListHolder bulletListHolder = (BulletListHolder) listHolder;
String marker = repeat(" ", listItem.getMarkerIndent()) + bulletListHolder.bulletMarker;
- writer.write(marker);
- writer.write(repeat(" ", contentIndent - marker.length()));
+ writer.writePrefix(marker);
+ writer.writePrefix(repeat(" ", contentIndent - marker.length()));
writer.pushPrefix(repeat(" ", contentIndent));
pushedPrefix = true;
} else if (listHolder instanceof OrderedListHolder) {
OrderedListHolder orderedListHolder = (OrderedListHolder) listHolder;
String marker = repeat(" ", listItem.getMarkerIndent()) + orderedListHolder.number + orderedListHolder.delimiter;
orderedListHolder.number++;
- writer.write(marker);
- writer.write(repeat(" ", contentIndent - marker.length()));
+ writer.writePrefix(marker);
+ writer.writePrefix(repeat(" ", contentIndent - marker.length()));
writer.pushPrefix(repeat(" ", contentIndent));
pushedPrefix = true;
}
@@ -190,7 +190,7 @@ public void visit(FencedCodeBlock fencedCodeBlock) {
if (indent > 0) {
String indentPrefix = repeat(" ", indent);
- writer.write(indentPrefix);
+ writer.writePrefix(indentPrefix);
writer.pushPrefix(indentPrefix);
}
@@ -286,7 +286,7 @@ public void visit(IndentedCodeBlock indentedCodeBlock) {
String literal = indentedCodeBlock.getLiteral();
// We need to respect line prefixes which is why we need to write it line by line (e.g. an indented code block
// within a block quote)
- writer.write(" ");
+ writer.writePrefix(" ");
writer.pushPrefix(" ");
List lines = getLines(literal);
for (int i = 0; i < lines.size(); i++) {
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
index bc10f020e..d95c0bd87 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
@@ -11,7 +11,8 @@ public class MarkdownWriter {
private int blockSeparator = 0;
private boolean tight;
- private char lastChar = '\n';
+ private char lastChar;
+ private boolean atLineStart = true;
private final LinkedList prefixes = new LinkedList<>();
public MarkdownWriter(Appendable out) {
@@ -22,8 +23,11 @@ public char getLastChar() {
return lastChar;
}
+ /**
+ * @return whether we're at the line start (not counting any prefixes), i.e. after a {@link #line} or {@link #block}.
+ */
public boolean isAtLineStart() {
- return lastChar == '\n' || blockSeparator > 0;
+ return atLineStart;
}
public void write(String s) {
@@ -54,11 +58,13 @@ public void writeEscaped(String s, CharMatcher escape) {
}
lastChar = s.charAt(s.length() - 1);
+ atLineStart = false;
}
public void line() {
append('\n');
writePrefixes();
+ atLineStart = true;
}
/**
@@ -69,12 +75,19 @@ public void block() {
// Remember whether this should be a tight or loose separator now because tight could get changed in between
// this and the next flush.
blockSeparator = tight ? 1 : 2;
+ atLineStart = true;
}
public void pushPrefix(String prefix) {
prefixes.addLast(prefix);
}
+ public void writePrefix(String prefix) {
+ boolean tmp = atLineStart;
+ write(prefix);
+ atLineStart = tmp;
+ }
+
public void popPrefix() {
prefixes.removeLast();
}
@@ -90,6 +103,7 @@ private void append(String s) {
if (length != 0) {
lastChar = s.charAt(length - 1);
}
+ atLineStart = false;
}
private void append(char c) {
@@ -100,6 +114,7 @@ private void append(char c) {
}
lastChar = c;
+ atLineStart = false;
}
private void writePrefixes() {
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index 59039799a..6f9e1e6f4 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -137,6 +137,11 @@ public void testEscaping() {
assertRoundTrip("\\## Test\n");
assertRoundTrip("\\#\n");
assertRoundTrip("Foo\n\\===\n");
+ // The beginning of the line within the block, so disregarding prefixes
+ assertRoundTrip("> \\- Test\n");
+ assertRoundTrip("- \\- Test\n");
+ // That's not the beginning of the line
+ assertRoundTrip("`a`- foo\n");
// This is a bit more tricky as we need to check for a list start
assertRoundTrip("1\\. Foo\n");
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index d6bdf9cfb..39269368e 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -62,7 +62,7 @@ public void testCoverage() {
System.out.println();
}
- int expectedPassed = 647;
+ int expectedPassed = 650;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
}
From 59d5c030969c57ce0f23b8eb619283cb2d7bf5ea Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 24 Jan 2024 14:53:33 +1100
Subject: [PATCH 051/286] Escape whitespace too (weird)
---
.../renderer/markdown/CoreMarkdownNodeRenderer.java | 11 +++++++++++
.../commonmark/renderer/markdown/MarkdownWriter.java | 11 ++++++++---
.../renderer/markdown/MarkdownRendererTest.java | 5 +++++
.../renderer/markdown/SpecMarkdownRendererTest.java | 4 +++-
4 files changed, 27 insertions(+), 4 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 11e7ae5f0..207851113 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -364,6 +364,17 @@ public void visit(Text text) {
writer.write("\\" + m.group(2));
literal = literal.substring(m.end());
}
+ break;
+ }
+ case '\t': {
+ writer.write(" ");
+ literal = literal.substring(1);
+ break;
+ }
+ case ' ': {
+ writer.write(" ");
+ literal = literal.substring(1);
+ break;
}
}
}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
index d95c0bd87..ef2945359 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
@@ -48,10 +48,15 @@ public void writeEscaped(String s, CharMatcher escape) {
try {
for (int i = 0; i < s.length(); i++) {
char ch = s.charAt(i);
- if (ch == '\\' || escape.matches(ch)) {
- buffer.append('\\');
+ if (ch == '\n') {
+ // Can't escape this with \, use numeric character reference
+ buffer.append("
");
+ } else {
+ if (ch == '\\' || escape.matches(ch)) {
+ buffer.append('\\');
+ }
+ buffer.append(ch);
}
- buffer.append(ch);
}
} catch (IOException e) {
throw new RuntimeException(e);
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index 6f9e1e6f4..bc6d9b696 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -148,6 +148,11 @@ public void testEscaping() {
assertRoundTrip("999\\. Foo\n");
assertRoundTrip("1\\.\n");
assertRoundTrip("1\\) Foo\n");
+
+ // Escaped whitespace, wow
+ assertRoundTrip(" foo\n");
+ assertRoundTrip(" foo\n");
+ assertRoundTrip("foo
bar\n");
}
@Test
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
index 39269368e..5df2e5c80 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/SpecMarkdownRendererTest.java
@@ -13,6 +13,7 @@
import java.util.List;
import java.util.Map;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
/**
@@ -62,8 +63,9 @@ public void testCoverage() {
System.out.println();
}
- int expectedPassed = 650;
+ int expectedPassed = 652;
assertTrue("Expected at least " + expectedPassed + " examples to pass but was " + passes.size(), passes.size() >= expectedPassed);
+ assertEquals(0, fails.size());
}
private static void printCountsBySection(List examples) {
From 37e507ab08912d3fd8e9753f86ed1dd7e84f9b69 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 24 Jan 2024 21:57:17 +1100
Subject: [PATCH 052/286] Reorder methods in spec order
---
.../markdown/CoreMarkdownNodeRenderer.java | 238 +++++++++---------
1 file changed, 119 insertions(+), 119 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 207851113..30c6332b9 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -86,6 +86,114 @@ public void visit(Document document) {
writer.line();
}
+ @Override
+ public void visit(ThematicBreak thematicBreak) {
+ writer.write("***");
+ writer.block();
+ }
+
+ @Override
+ public void visit(Heading heading) {
+ if (heading.getLevel() <= 2) {
+ LineBreakVisitor lineBreakVisitor = new LineBreakVisitor();
+ heading.accept(lineBreakVisitor);
+ boolean isMultipleLines = lineBreakVisitor.hasLineBreak();
+
+ if (isMultipleLines) {
+ // Setext headings: Can have multiple lines, but only level 1 or 2
+ visitChildren(heading);
+ writer.line();
+ if (heading.getLevel() == 1) {
+ // Note that it would be nice to match the length of the contents instead of just using 3, but that's
+ // not easy.
+ writer.write("===");
+ } else {
+ writer.write("---");
+ }
+ writer.block();
+ return;
+ }
+ }
+
+ // ATX headings: Can't have multiple lines, but up to level 6.
+ for (int i = 0; i < heading.getLevel(); i++) {
+ writer.write('#');
+ }
+ writer.write(' ');
+ visitChildren(heading);
+
+ writer.block();
+ }
+
+ @Override
+ public void visit(IndentedCodeBlock indentedCodeBlock) {
+ String literal = indentedCodeBlock.getLiteral();
+ // We need to respect line prefixes which is why we need to write it line by line (e.g. an indented code block
+ // within a block quote)
+ writer.writePrefix(" ");
+ writer.pushPrefix(" ");
+ List lines = getLines(literal);
+ for (int i = 0; i < lines.size(); i++) {
+ String line = lines.get(i);
+ writer.write(line);
+ if (i != lines.size() - 1) {
+ writer.line();
+ }
+ }
+ writer.popPrefix();
+ writer.block();
+ }
+
+ @Override
+ public void visit(FencedCodeBlock fencedCodeBlock) {
+ String literal = fencedCodeBlock.getLiteral();
+ String fence = repeat(String.valueOf(fencedCodeBlock.getFenceChar()), fencedCodeBlock.getFenceLength());
+ int indent = fencedCodeBlock.getFenceIndent();
+
+ if (indent > 0) {
+ String indentPrefix = repeat(" ", indent);
+ writer.writePrefix(indentPrefix);
+ writer.pushPrefix(indentPrefix);
+ }
+
+ writer.write(fence);
+ if (fencedCodeBlock.getInfo() != null) {
+ writer.write(fencedCodeBlock.getInfo());
+ }
+ writer.line();
+ if (!literal.isEmpty()) {
+ List lines = getLines(literal);
+ for (String line : lines) {
+ writer.write(line);
+ writer.line();
+ }
+ }
+ writer.write(fence);
+ if (indent > 0) {
+ writer.popPrefix();
+ }
+ writer.block();
+ }
+
+ @Override
+ public void visit(HtmlBlock htmlBlock) {
+ List lines = getLines(htmlBlock.getLiteral());
+ for (int i = 0; i < lines.size(); i++) {
+ String line = lines.get(i);
+ writer.write(line);
+ if (i != lines.size() - 1) {
+ writer.line();
+ }
+ }
+ writer.block();
+ }
+
+ @Override
+ public void visit(Paragraph paragraph) {
+ visitChildren(paragraph);
+ writer.block();
+ }
+
@Override
public void visit(BlockQuote blockQuote) {
writer.writePrefix("> ");
@@ -183,97 +291,15 @@ public void visit(Emphasis emphasis) {
}
@Override
- public void visit(FencedCodeBlock fencedCodeBlock) {
- String literal = fencedCodeBlock.getLiteral();
- String fence = repeat(String.valueOf(fencedCodeBlock.getFenceChar()), fencedCodeBlock.getFenceLength());
- int indent = fencedCodeBlock.getFenceIndent();
-
- if (indent > 0) {
- String indentPrefix = repeat(" ", indent);
- writer.writePrefix(indentPrefix);
- writer.pushPrefix(indentPrefix);
- }
-
- writer.write(fence);
- if (fencedCodeBlock.getInfo() != null) {
- writer.write(fencedCodeBlock.getInfo());
- }
- writer.line();
- if (!literal.isEmpty()) {
- List lines = getLines(literal);
- for (String line : lines) {
- writer.write(line);
- writer.line();
- }
- }
- writer.write(fence);
- if (indent > 0) {
- writer.popPrefix();
- }
- writer.block();
- }
-
- @Override
- public void visit(HardLineBreak hardLineBreak) {
- writer.write(" ");
- writer.line();
- }
-
- @Override
- public void visit(Heading heading) {
- if (heading.getLevel() <= 2) {
- LineBreakVisitor lineBreakVisitor = new LineBreakVisitor();
- heading.accept(lineBreakVisitor);
- boolean isMultipleLines = lineBreakVisitor.hasLineBreak();
-
- if (isMultipleLines) {
- // Setext headings: Can have multiple lines, but only level 1 or 2
- visitChildren(heading);
- writer.line();
- if (heading.getLevel() == 1) {
- // Note that it would be nice to match the length of the contents instead of just using 3, but that's
- // not easy.
- writer.write("===");
- } else {
- writer.write("---");
- }
- writer.block();
- return;
- }
- }
-
- // ATX headings: Can't have multiple lines, but up to level 6.
- for (int i = 0; i < heading.getLevel(); i++) {
- writer.write('#');
- }
- writer.write(' ');
- visitChildren(heading);
-
- writer.block();
- }
-
- @Override
- public void visit(ThematicBreak thematicBreak) {
- writer.write("***");
- writer.block();
- }
-
- @Override
- public void visit(HtmlInline htmlInline) {
- writer.write(htmlInline.getLiteral());
+ public void visit(StrongEmphasis strongEmphasis) {
+ writer.write("**");
+ super.visit(strongEmphasis);
+ writer.write("**");
}
@Override
- public void visit(HtmlBlock htmlBlock) {
- List lines = getLines(htmlBlock.getLiteral());
- for (int i = 0; i < lines.size(); i++) {
- String line = lines.get(i);
- writer.write(line);
- if (i != lines.size() - 1) {
- writer.line();
- }
- }
- writer.block();
+ public void visit(Link link) {
+ writeLinkLike(link.getTitle(), link.getDestination(), link, "[");
}
@Override
@@ -282,33 +308,14 @@ public void visit(Image image) {
}
@Override
- public void visit(IndentedCodeBlock indentedCodeBlock) {
- String literal = indentedCodeBlock.getLiteral();
- // We need to respect line prefixes which is why we need to write it line by line (e.g. an indented code block
- // within a block quote)
- writer.writePrefix(" ");
- writer.pushPrefix(" ");
- List lines = getLines(literal);
- for (int i = 0; i < lines.size(); i++) {
- String line = lines.get(i);
- writer.write(line);
- if (i != lines.size() - 1) {
- writer.line();
- }
- }
- writer.popPrefix();
- writer.block();
- }
-
- @Override
- public void visit(Link link) {
- writeLinkLike(link.getTitle(), link.getDestination(), link, "[");
+ public void visit(HtmlInline htmlInline) {
+ writer.write(htmlInline.getLiteral());
}
@Override
- public void visit(Paragraph paragraph) {
- visitChildren(paragraph);
- writer.block();
+ public void visit(HardLineBreak hardLineBreak) {
+ writer.write(" ");
+ writer.line();
}
@Override
@@ -316,13 +323,6 @@ public void visit(SoftLineBreak softLineBreak) {
writer.line();
}
- @Override
- public void visit(StrongEmphasis strongEmphasis) {
- writer.write("**");
- super.visit(strongEmphasis);
- writer.write("**");
- }
-
@Override
public void visit(Text text) {
String literal = text.getLiteral();
From edfc3d025503b38799b63db261196f6fe4668add Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Mon, 29 Jan 2024 17:30:36 +1100
Subject: [PATCH 053/286] Some more comments
---
.../markdown/CoreMarkdownNodeRenderer.java | 32 +++++++++++++------
1 file changed, 22 insertions(+), 10 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 30c6332b9..199add94d 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -51,26 +51,26 @@ public CoreMarkdownNodeRenderer(MarkdownNodeRendererContext context) {
@Override
public Set> getNodeTypes() {
return new HashSet<>(Arrays.asList(
- Document.class,
- Heading.class,
- Paragraph.class,
BlockQuote.class,
BulletList.class,
+ Code.class,
+ Document.class,
+ Emphasis.class,
FencedCodeBlock.class,
+ HardLineBreak.class,
+ Heading.class,
HtmlBlock.class,
- ThematicBreak.class,
+ HtmlInline.class,
+ Image.class,
IndentedCodeBlock.class,
Link.class,
ListItem.class,
OrderedList.class,
- Image.class,
- Emphasis.class,
+ Paragraph.class,
+ SoftLineBreak.class,
StrongEmphasis.class,
Text.class,
- Code.class,
- HtmlInline.class,
- SoftLineBreak.class,
- HardLineBreak.class
+ ThematicBreak.class
));
}
@@ -325,6 +325,14 @@ public void visit(SoftLineBreak softLineBreak) {
@Override
public void visit(Text text) {
+ // Text is tricky. In Markdown special characters (`-`, `#` etc.) can be escaped (`\-`, `\#` etc.) so that
+ // they're parsed as plain text. Currently, whether a character was escaped or not is not recorded in the Node,
+ // so here we don't know. If we just wrote out those characters unescaped, the resulting Markdown would change
+ // meaning (turn into a list item, heading, etc.).
+ // You might say "Why not store that in the Node when parsing", but that wouldn't work for the use case where
+ // nodes are constructed directly instead of via parsing. This renderer needs to work for that too.
+ // So currently, when in doubt, we escape. For special characters only occurring at the beginning of a line,
+ // we only escape them then (we wouldn't want to escape every `.` for example).
String literal = text.getLiteral();
if (writer.isAtLineStart() && !literal.isEmpty()) {
char c = literal.charAt(0);
@@ -382,6 +390,7 @@ public void visit(Text text) {
CharMatcher escape = text.getParent() instanceof Heading ? textEscapeInHeading : textEscape;
if (literal.endsWith("!") && text.getNext() instanceof Link) {
+ // If we wrote the `!` unescaped, it would turn the link into an image instead.
writer.writeEscaped(literal.substring(0, literal.length() - 1), escape);
writer.write("\\!");
} else {
@@ -494,6 +503,9 @@ protected OrderedListHolder(ListHolder parent, OrderedList orderedList) {
}
}
+ /**
+ * Visits nodes to check if there are any soft or hard line breaks.
+ */
private static class LineBreakVisitor extends AbstractVisitor {
private boolean lineBreak = false;
From 08a60f11ce3c69042e2ce9f532a77c0cb5a0733f Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Mon, 29 Jan 2024 17:39:58 +1100
Subject: [PATCH 054/286] Reorder methods in writer, add comments
---
.../markdown/CoreMarkdownNodeRenderer.java | 88 ++++++++---------
.../renderer/markdown/MarkdownWriter.java | 99 +++++++++++++------
2 files changed, 111 insertions(+), 76 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 199add94d..f5a9377f6 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -88,7 +88,7 @@ public void visit(Document document) {
@Override
public void visit(ThematicBreak thematicBreak) {
- writer.write("***");
+ writer.raw("***");
writer.block();
}
@@ -106,9 +106,9 @@ public void visit(Heading heading) {
if (heading.getLevel() == 1) {
// Note that it would be nice to match the length of the contents instead of just using 3, but that's
// not easy.
- writer.write("===");
+ writer.raw("===");
} else {
- writer.write("---");
+ writer.raw("---");
}
writer.block();
return;
@@ -117,9 +117,9 @@ public void visit(Heading heading) {
// ATX headings: Can't have multiple lines, but up to level 6.
for (int i = 0; i < heading.getLevel(); i++) {
- writer.write('#');
+ writer.raw('#');
}
- writer.write(' ');
+ writer.raw(' ');
visitChildren(heading);
writer.block();
@@ -135,7 +135,7 @@ public void visit(IndentedCodeBlock indentedCodeBlock) {
List lines = getLines(literal);
for (int i = 0; i < lines.size(); i++) {
String line = lines.get(i);
- writer.write(line);
+ writer.raw(line);
if (i != lines.size() - 1) {
writer.line();
}
@@ -156,19 +156,19 @@ public void visit(FencedCodeBlock fencedCodeBlock) {
writer.pushPrefix(indentPrefix);
}
- writer.write(fence);
+ writer.raw(fence);
if (fencedCodeBlock.getInfo() != null) {
- writer.write(fencedCodeBlock.getInfo());
+ writer.raw(fencedCodeBlock.getInfo());
}
writer.line();
if (!literal.isEmpty()) {
List lines = getLines(literal);
for (String line : lines) {
- writer.write(line);
+ writer.raw(line);
writer.line();
}
}
- writer.write(fence);
+ writer.raw(fence);
if (indent > 0) {
writer.popPrefix();
}
@@ -180,7 +180,7 @@ public void visit(HtmlBlock htmlBlock) {
List lines = getLines(htmlBlock.getLiteral());
for (int i = 0; i < lines.size(); i++) {
String line = lines.get(i);
- writer.write(line);
+ writer.raw(line);
if (i != lines.size() - 1) {
writer.line();
}
@@ -262,7 +262,7 @@ public void visit(Code code) {
// If the literal includes backticks, we can surround them by using one more backtick.
int backticks = findMaxRunLength('`', literal);
for (int i = 0; i < backticks + 1; i++) {
- writer.write('`');
+ writer.raw('`');
}
// If the literal starts or ends with a backtick, surround it with a single space.
// If it starts and ends with a space (but is not only spaces), add an additional space (otherwise they would
@@ -270,14 +270,14 @@ public void visit(Code code) {
boolean addSpace = literal.startsWith("`") || literal.endsWith("`") ||
(literal.startsWith(" ") && literal.endsWith(" ") && Parsing.hasNonSpace(literal));
if (addSpace) {
- writer.write(' ');
+ writer.raw(' ');
}
- writer.write(literal);
+ writer.raw(literal);
if (addSpace) {
- writer.write(' ');
+ writer.raw(' ');
}
for (int i = 0; i < backticks + 1; i++) {
- writer.write('`');
+ writer.raw('`');
}
}
@@ -285,16 +285,16 @@ public void visit(Code code) {
public void visit(Emphasis emphasis) {
// When emphasis is nested, a different delimiter needs to be used
char delimiter = writer.getLastChar() == '*' ? '_' : '*';
- writer.write(delimiter);
+ writer.raw(delimiter);
super.visit(emphasis);
- writer.write(delimiter);
+ writer.raw(delimiter);
}
@Override
public void visit(StrongEmphasis strongEmphasis) {
- writer.write("**");
+ writer.raw("**");
super.visit(strongEmphasis);
- writer.write("**");
+ writer.raw("**");
}
@Override
@@ -309,12 +309,12 @@ public void visit(Image image) {
@Override
public void visit(HtmlInline htmlInline) {
- writer.write(htmlInline.getLiteral());
+ writer.raw(htmlInline.getLiteral());
}
@Override
public void visit(HardLineBreak hardLineBreak) {
- writer.write(" ");
+ writer.raw(" ");
writer.line();
}
@@ -339,19 +339,19 @@ public void visit(Text text) {
switch (c) {
case '-': {
// Would be ambiguous with a bullet list marker, escape
- writer.write("\\-");
+ writer.raw("\\-");
literal = literal.substring(1);
break;
}
case '#': {
// Would be ambiguous with an ATX heading, escape
- writer.write("\\#");
+ writer.raw("\\#");
literal = literal.substring(1);
break;
}
case '=': {
// Would be ambiguous with a Setext heading, escape
- writer.write("\\=");
+ writer.raw("\\=");
literal = literal.substring(1);
break;
}
@@ -368,19 +368,19 @@ public void visit(Text text) {
// Check for ordered list marker
Matcher m = orderedListMarkerPattern.matcher(literal);
if (m.find()) {
- writer.write(m.group(1));
- writer.write("\\" + m.group(2));
+ writer.raw(m.group(1));
+ writer.raw("\\" + m.group(2));
literal = literal.substring(m.end());
}
break;
}
case '\t': {
- writer.write(" ");
+ writer.raw(" ");
literal = literal.substring(1);
break;
}
case ' ': {
- writer.write(" ");
+ writer.raw(" ");
literal = literal.substring(1);
break;
}
@@ -391,10 +391,10 @@ public void visit(Text text) {
if (literal.endsWith("!") && text.getNext() instanceof Link) {
// If we wrote the `!` unescaped, it would turn the link into an image instead.
- writer.writeEscaped(literal.substring(0, literal.length() - 1), escape);
- writer.write("\\!");
+ writer.text(literal.substring(0, literal.length() - 1), escape);
+ writer.raw("\\!");
} else {
- writer.writeEscaped(literal, escape);
+ writer.text(literal, escape);
}
}
@@ -455,24 +455,24 @@ private static List getLines(String literal) {
}
private void writeLinkLike(String title, String destination, Node node, String opener) {
- writer.write(opener);
+ writer.raw(opener);
visitChildren(node);
- writer.write(']');
- writer.write('(');
+ writer.raw(']');
+ writer.raw('(');
if (contains(destination, linkDestinationNeedsAngleBrackets)) {
- writer.write('<');
- writer.writeEscaped(destination, linkDestinationEscapeInAngleBrackets);
- writer.write('>');
+ writer.raw('<');
+ writer.text(destination, linkDestinationEscapeInAngleBrackets);
+ writer.raw('>');
} else {
- writer.write(destination);
+ writer.raw(destination);
}
if (title != null) {
- writer.write(' ');
- writer.write('"');
- writer.writeEscaped(title, linkTitleEscapeInQuotes);
- writer.write('"');
+ writer.raw(' ');
+ writer.raw('"');
+ writer.text(title, linkTitleEscapeInQuotes);
+ writer.raw('"');
}
- writer.write(')');
+ writer.raw(')');
}
private static class ListHolder {
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
index ef2945359..3b2ae18b0 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
@@ -5,6 +5,9 @@
import java.io.IOException;
import java.util.LinkedList;
+/**
+ * Writer for Markdown (CommonMark) text.
+ */
public class MarkdownWriter {
private final Appendable buffer;
@@ -19,28 +22,29 @@ public MarkdownWriter(Appendable out) {
buffer = out;
}
- public char getLastChar() {
- return lastChar;
- }
-
/**
- * @return whether we're at the line start (not counting any prefixes), i.e. after a {@link #line} or {@link #block}.
+ * Write the supplied string (raw/unescaped).
*/
- public boolean isAtLineStart() {
- return atLineStart;
- }
-
- public void write(String s) {
+ public void raw(String s) {
flushBlockSeparator();
append(s);
}
- public void write(char c) {
+ /**
+ * Write the supplied character (raw/unescaped).
+ */
+ public void raw(char c) {
flushBlockSeparator();
append(c);
}
- public void writeEscaped(String s, CharMatcher escape) {
+ /**
+ * Write the supplied string with escaping.
+ *
+ * @param s the string to write
+ * @param escape which characters to escape
+ */
+ public void text(String s, CharMatcher escape) {
if (s.isEmpty()) {
return;
}
@@ -66,6 +70,9 @@ public void writeEscaped(String s, CharMatcher escape) {
atLineStart = false;
}
+ /**
+ * Write a newline (line terminator).
+ */
public void line() {
append('\n');
writePrefixes();
@@ -83,20 +90,67 @@ public void block() {
atLineStart = true;
}
+ /**
+ * Push a prefix onto the top of the stack. All prefixes are written at the beginning of each line, until the
+ * prefix is popped again.
+ *
+ * @param prefix the raw prefix string
+ */
public void pushPrefix(String prefix) {
prefixes.addLast(prefix);
}
+ /**
+ * Write a prefix.
+ *
+ * @param prefix the raw prefix string to write
+ */
public void writePrefix(String prefix) {
boolean tmp = atLineStart;
- write(prefix);
+ raw(prefix);
atLineStart = tmp;
}
+ /**
+ * Remove the last prefix from the top of the stack.
+ */
public void popPrefix() {
prefixes.removeLast();
}
+ /**
+ * @return the last character that was written
+ */
+ public char getLastChar() {
+ return lastChar;
+ }
+
+ /**
+ * @return whether we're at the line start (not counting any prefixes), i.e. after a {@link #line} or {@link #block}.
+ */
+ public boolean isAtLineStart() {
+ return atLineStart;
+ }
+
+ /**
+ * @return whether blocks are currently set to tight or loose, see {@link #setTight(boolean)}
+ */
+ public boolean getTight() {
+ return tight;
+ }
+
+ /**
+ * Change whether blocks are tight or loose. Loose is the default where blocks are separated by a blank line. Tight
+ * is where blocks are not separated by a blank line. Tight blocks are used in lists, if there are no blank lines
+ * within the list.
+ *
+ * Note that changing this does not affect block separators that have already been enqueued (with {@link #block()},
+ * only future ones.
+ */
+ public void setTight(boolean tight) {
+ this.tight = tight;
+ }
+
private void append(String s) {
try {
buffer.append(s);
@@ -144,23 +198,4 @@ private void flushBlockSeparator() {
blockSeparator = 0;
}
}
-
- /**
- * @return whether blocks are currently set to tight or loose, see {@link #setTight(boolean)}
- */
- public boolean getTight() {
- return tight;
- }
-
- /**
- * Change whether blocks are tight or loose. Loose is the default where blocks are separated by a blank line. Tight
- * is where blocks are not separated by a blank line. Tight blocks are used in lists, if there are no blank lines
- * within the list.
- *
- * Note that changing this does not affect block separators that have already been enqueued (with {@link #block()},
- * only future ones.
- */
- public void setTight(boolean tight) {
- this.tight = tight;
- }
}
From e6c82d533516553935b8f2a18d60314231cb8adb Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 6 Feb 2024 12:44:26 +1100
Subject: [PATCH 055/286] Tables extension markdown renderer
---
.../commonmark/ext/gfm/tables/TableCell.java | 2 +-
.../ext/gfm/tables/TablesExtension.java | 16 +++-
.../internal/TableMarkdownNodeRenderer.java | 96 +++++++++++++++++++
.../tables/TablesMarkdownRenderingTest.java | 70 ++++++++++++++
.../markdown/CoreMarkdownNodeRenderer.java | 8 +-
.../renderer/markdown/MarkdownWriter.java | 94 ++++++++++++------
.../markdown/MarkdownRendererTest.java | 3 +
7 files changed, 255 insertions(+), 34 deletions(-)
create mode 100644 commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableMarkdownNodeRenderer.java
create mode 100644 commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesMarkdownRenderingTest.java
diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TableCell.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TableCell.java
index 61880c6c3..bd7b40e02 100644
--- a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TableCell.java
+++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TableCell.java
@@ -22,7 +22,7 @@ public void setHeader(boolean header) {
}
/**
- * @return the cell alignment
+ * @return the cell alignment or {@code null} if no specific alignment
*/
public Alignment getAlignment() {
return alignment;
diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TablesExtension.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TablesExtension.java
index 5707b0f14..d23f6f5fc 100644
--- a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TablesExtension.java
+++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TablesExtension.java
@@ -3,12 +3,16 @@
import org.commonmark.Extension;
import org.commonmark.ext.gfm.tables.internal.TableBlockParser;
import org.commonmark.ext.gfm.tables.internal.TableHtmlNodeRenderer;
+import org.commonmark.ext.gfm.tables.internal.TableMarkdownNodeRenderer;
import org.commonmark.ext.gfm.tables.internal.TableTextContentNodeRenderer;
import org.commonmark.parser.Parser;
import org.commonmark.renderer.NodeRenderer;
import org.commonmark.renderer.html.HtmlNodeRendererContext;
import org.commonmark.renderer.html.HtmlNodeRendererFactory;
import org.commonmark.renderer.html.HtmlRenderer;
+import org.commonmark.renderer.markdown.MarkdownNodeRendererContext;
+import org.commonmark.renderer.markdown.MarkdownNodeRendererFactory;
+import org.commonmark.renderer.markdown.MarkdownRenderer;
import org.commonmark.renderer.text.TextContentNodeRendererContext;
import org.commonmark.renderer.text.TextContentNodeRendererFactory;
import org.commonmark.renderer.text.TextContentRenderer;
@@ -27,7 +31,7 @@
* @see Tables (extension) in GitHub Flavored Markdown Spec
*/
public class TablesExtension implements Parser.ParserExtension, HtmlRenderer.HtmlRendererExtension,
- TextContentRenderer.TextContentRendererExtension {
+ TextContentRenderer.TextContentRendererExtension, MarkdownRenderer.MarkdownRendererExtension {
private TablesExtension() {
}
@@ -60,4 +64,14 @@ public NodeRenderer create(TextContentNodeRendererContext context) {
}
});
}
+
+ @Override
+ public void extend(MarkdownRenderer.Builder rendererBuilder) {
+ rendererBuilder.nodeRendererFactory(new MarkdownNodeRendererFactory() {
+ @Override
+ public NodeRenderer create(MarkdownNodeRendererContext context) {
+ return new TableMarkdownNodeRenderer(context);
+ }
+ });
+ }
}
diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableMarkdownNodeRenderer.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableMarkdownNodeRenderer.java
new file mode 100644
index 000000000..bf94db5bb
--- /dev/null
+++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableMarkdownNodeRenderer.java
@@ -0,0 +1,96 @@
+package org.commonmark.ext.gfm.tables.internal;
+
+import org.commonmark.ext.gfm.tables.*;
+import org.commonmark.internal.util.AsciiMatcher;
+import org.commonmark.internal.util.CharMatcher;
+import org.commonmark.node.Node;
+import org.commonmark.renderer.NodeRenderer;
+import org.commonmark.renderer.markdown.MarkdownNodeRendererContext;
+import org.commonmark.renderer.markdown.MarkdownWriter;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * The Table node renderer that is needed for rendering GFM tables (GitHub Flavored Markdown) to text content.
+ */
+public class TableMarkdownNodeRenderer extends TableNodeRenderer implements NodeRenderer {
+ private final MarkdownWriter writer;
+ private final MarkdownNodeRendererContext context;
+
+ private final CharMatcher pipe = AsciiMatcher.builder().c('|').build();
+
+ private final List columns = new ArrayList<>();
+
+ public TableMarkdownNodeRenderer(MarkdownNodeRendererContext context) {
+ this.writer = context.getWriter();
+ this.context = context;
+ }
+
+ @Override
+ protected void renderBlock(TableBlock node) {
+ columns.clear();
+ renderChildren(node);
+ writer.block();
+ }
+
+ @Override
+ protected void renderHead(TableHead node) {
+ renderChildren(node);
+ // TODO: Not sure about this.. Should block() detect if a line was already written? Or should line() itself be lazy?
+ writer.line();
+ for (TableCell.Alignment columnAlignment : columns) {
+ writer.raw('|');
+ if (columnAlignment == TableCell.Alignment.LEFT) {
+ writer.raw(":---");
+ } else if (columnAlignment == TableCell.Alignment.RIGHT) {
+ writer.raw("---:");
+ } else if (columnAlignment == TableCell.Alignment.CENTER) {
+ writer.raw(":---:");
+ } else {
+ writer.raw("---");
+ }
+ }
+ writer.raw("|");
+ // TODO
+ if (node.getNext() != null) {
+ writer.line();
+ }
+ }
+
+ @Override
+ protected void renderBody(TableBody node) {
+ renderChildren(node);
+ }
+
+ @Override
+ protected void renderRow(TableRow node) {
+ renderChildren(node);
+ // Trailing | at the end of the line
+ writer.raw("|");
+ // TODO
+ if (node.getNext() != null) {
+ writer.line();
+ }
+ }
+
+ @Override
+ protected void renderCell(TableCell node) {
+ if (node.getParent() != null && node.getParent().getParent() instanceof TableHead) {
+ columns.add(node.getAlignment());
+ }
+ writer.raw("|");
+ writer.pushRawEscape(pipe);
+ renderChildren(node);
+ writer.popRawEscape();
+ }
+
+ private void renderChildren(Node parent) {
+ Node node = parent.getFirstChild();
+ while (node != null) {
+ Node next = node.getNext();
+ context.render(node);
+ node = next;
+ }
+ }
+}
diff --git a/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesMarkdownRenderingTest.java b/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesMarkdownRenderingTest.java
new file mode 100644
index 000000000..8689e4ac2
--- /dev/null
+++ b/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesMarkdownRenderingTest.java
@@ -0,0 +1,70 @@
+package org.commonmark.ext.gfm.tables;
+
+import org.commonmark.Extension;
+import org.commonmark.parser.Parser;
+import org.commonmark.renderer.markdown.MarkdownRenderer;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+
+public class TablesMarkdownRenderingTest {
+
+ private static final Set EXTENSIONS = Collections.singleton(TablesExtension.create());
+ private static final Parser PARSER = Parser.builder().extensions(EXTENSIONS).build();
+ private static final MarkdownRenderer RENDERER = MarkdownRenderer.builder().extensions(EXTENSIONS).build();
+
+ @Test
+ public void testHeadNoBody() {
+ assertRoundTrip("|Abc|\n|---|\n");
+ assertRoundTrip("|Abc|Def|\n|---|---|\n");
+ assertRoundTrip("|Abc||\n|---|---|\n");
+ }
+
+ @Test
+ public void testHeadAndBody() {
+ assertRoundTrip("|Abc|\n|---|\n|1|\n");
+ assertRoundTrip("|Abc|Def|\n|---|---|\n|1|2|\n");
+ }
+
+ @Test
+ public void testBodyHasFewerColumns() {
+ // Could try not to write empty trailing cells but this is fine too
+ assertRoundTrip("|Abc|Def|\n|---|---|\n|1||\n");
+ }
+
+ @Test
+ public void testAlignment() {
+ assertRoundTrip("|Abc|Def|\n|:---|---|\n|1|2|\n");
+ assertRoundTrip("|Abc|Def|\n|---|---:|\n|1|2|\n");
+ assertRoundTrip("|Abc|Def|\n|:---:|:---:|\n|1|2|\n");
+ }
+
+ @Test
+ public void testInsideBlockQuote() {
+ assertRoundTrip("> |Abc|Def|\n> |---|---|\n> |1|2|\n");
+ }
+
+ @Test
+ public void testMultipleTables() {
+ assertRoundTrip("|Abc|Def|\n|---|---|\n\n|One|\n|---|\n|Only|\n");
+ }
+
+ @Test
+ public void testEscaping() {
+ assertRoundTrip("|Abc|Def|\n|---|---|\n|Pipe in|text \\||\n");
+ assertRoundTrip("|Abc|Def|\n|---|---|\n|Pipe in|code `\\|`|\n");
+ assertRoundTrip("|Abc|Def|\n|---|---|\n|Inline HTML|Foo\\|bar|\n");
+ }
+
+ protected String render(String source) {
+ return RENDERER.render(PARSER.parse(source));
+ }
+
+ private void assertRoundTrip(String input) {
+ String rendered = render(input);
+ assertEquals(input, rendered);
+ }
+}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index f5a9377f6..714144e89 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -23,15 +23,15 @@
public class CoreMarkdownNodeRenderer extends AbstractVisitor implements NodeRenderer {
private final AsciiMatcher textEscape =
- AsciiMatcher.builder().anyOf("[]<>`*&").build();
+ AsciiMatcher.builder().anyOf("[]<>`*&\n\\").build();
private final CharMatcher textEscapeInHeading =
AsciiMatcher.builder(textEscape).anyOf("#").build();
private final CharMatcher linkDestinationNeedsAngleBrackets =
- AsciiMatcher.builder().c(' ').c('(').c(')').c('<').c('>').c('\\').build();
+ AsciiMatcher.builder().c(' ').c('(').c(')').c('<').c('>').c('\n').c('\\').build();
private final CharMatcher linkDestinationEscapeInAngleBrackets =
- AsciiMatcher.builder().c('<').c('>').build();
+ AsciiMatcher.builder().c('<').c('>').c('\n').c('\\').build();
private final CharMatcher linkTitleEscapeInQuotes =
- AsciiMatcher.builder().c('"').build();
+ AsciiMatcher.builder().c('"').c('\n').c('\\').build();
private final Pattern orderedListMarkerPattern = Pattern.compile("^([0-9]{1,9})([.)])");
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
index 3b2ae18b0..f648d9c4f 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
@@ -17,25 +17,26 @@ public class MarkdownWriter {
private char lastChar;
private boolean atLineStart = true;
private final LinkedList prefixes = new LinkedList<>();
+ private final LinkedList rawEscapes = new LinkedList<>();
public MarkdownWriter(Appendable out) {
buffer = out;
}
/**
- * Write the supplied string (raw/unescaped).
+ * Write the supplied string (raw/unescaped except if {@link #pushRawEscape} was used).
*/
public void raw(String s) {
flushBlockSeparator();
- append(s);
+ write(s, null);
}
/**
- * Write the supplied character (raw/unescaped).
+ * Write the supplied character (raw/unescaped except if {@link #pushRawEscape} was used).
*/
public void raw(char c) {
flushBlockSeparator();
- append(c);
+ write(c);
}
/**
@@ -49,22 +50,7 @@ public void text(String s, CharMatcher escape) {
return;
}
flushBlockSeparator();
- try {
- for (int i = 0; i < s.length(); i++) {
- char ch = s.charAt(i);
- if (ch == '\n') {
- // Can't escape this with \, use numeric character reference
- buffer.append("
");
- } else {
- if (ch == '\\' || escape.matches(ch)) {
- buffer.append('\\');
- }
- buffer.append(ch);
- }
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
+ write(s, escape);
lastChar = s.charAt(s.length() - 1);
atLineStart = false;
@@ -74,7 +60,7 @@ public void text(String s, CharMatcher escape) {
* Write a newline (line terminator).
*/
public void line() {
- append('\n');
+ write('\n');
writePrefixes();
atLineStart = true;
}
@@ -118,6 +104,24 @@ public void popPrefix() {
prefixes.removeLast();
}
+ /**
+ * Escape the characters matching the supplied matcher, in all text (text and raw). This might be useful to
+ * extensions that add another layer of syntax, e.g. the tables extension that uses `|` to separate cells and needs
+ * all `|` characters to be escaped (even in code spans).
+ *
+ * @param rawEscape the characters to escape in raw text
+ */
+ public void pushRawEscape(CharMatcher rawEscape) {
+ rawEscapes.add(rawEscape);
+ }
+
+ /**
+ * Remove the last raw escape from the top of the stack.
+ */
+ public void popRawEscape() {
+ rawEscapes.removeLast();
+ }
+
/**
* @return the last character that was written
*/
@@ -151,9 +155,16 @@ public void setTight(boolean tight) {
this.tight = tight;
}
- private void append(String s) {
+ private void write(String s, CharMatcher escape) {
try {
- buffer.append(s);
+ if (rawEscapes.isEmpty() && escape == null) {
+ // Normal fast path
+ buffer.append(s);
+ } else {
+ for (int i = 0; i < s.length(); i++) {
+ append(s.charAt(i), escape);
+ }
+ }
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -165,9 +176,9 @@ private void append(String s) {
atLineStart = false;
}
- private void append(char c) {
+ private void write(char c) {
try {
- buffer.append(c);
+ append(c, null);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -179,7 +190,7 @@ private void append(char c) {
private void writePrefixes() {
if (!prefixes.isEmpty()) {
for (String prefix : prefixes) {
- append(prefix);
+ write(prefix, null);
}
}
}
@@ -189,13 +200,40 @@ private void writePrefixes() {
*/
private void flushBlockSeparator() {
if (blockSeparator != 0) {
- append('\n');
+ write('\n');
writePrefixes();
if (blockSeparator > 1) {
- append('\n');
+ write('\n');
writePrefixes();
}
blockSeparator = 0;
}
}
+
+ private void append(char c, CharMatcher escape) throws IOException {
+ if (needsEscaping(c, escape)) {
+ if (c == '\n') {
+ // Can't escape this with \, use numeric character reference
+ buffer.append("
");
+ } else {
+ buffer.append('\\');
+ buffer.append(c);
+ }
+ } else {
+ buffer.append(c);
+ }
+ }
+
+ private boolean needsEscaping(char c, CharMatcher escape) {
+ return (escape != null && escape.matches(c)) || rawNeedsEscaping(c);
+ }
+
+ private boolean rawNeedsEscaping(char c) {
+ for (CharMatcher rawEscape : rawEscapes) {
+ if (rawEscape.matches(c)) {
+ return true;
+ }
+ }
+ return false;
+ }
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index bc6d9b696..20453eed7 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -194,6 +194,9 @@ public void testLinks() {
assertRoundTrip("[a](c>)\n");
assertRoundTrip("[a](c>)\n");
assertRoundTrip("[a](/uri \"foo \\\" bar\")\n");
+ assertRoundTrip("[link](/uri \"tes\\\\\")\n");
+ assertRoundTrip("[link](/url \"test
\")\n");
+ assertRoundTrip("[link]()\n");
}
@Test
From d726f72916bcdcdc3b5fb8d49d97beb6aaf784c2 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 6 Feb 2024 12:52:43 +1100
Subject: [PATCH 056/286] Change isTight/setTight to use push/pop pattern like
other settings
---
.../markdown/CoreMarkdownNodeRenderer.java | 10 ++--
.../renderer/markdown/MarkdownWriter.java | 49 +++++++++++--------
2 files changed, 32 insertions(+), 27 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 714144e89..c7fa9be7a 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -205,23 +205,21 @@ public void visit(BlockQuote blockQuote) {
@Override
public void visit(BulletList bulletList) {
- boolean oldTight = writer.getTight();
- writer.setTight(bulletList.isTight());
+ writer.pushTight(bulletList.isTight());
listHolder = new BulletListHolder(listHolder, bulletList);
visitChildren(bulletList);
listHolder = listHolder.parent;
- writer.setTight(oldTight);
+ writer.popTight();
writer.block();
}
@Override
public void visit(OrderedList orderedList) {
- boolean oldTight = writer.getTight();
- writer.setTight(orderedList.isTight());
+ writer.pushTight(orderedList.isTight());
listHolder = new OrderedListHolder(listHolder, orderedList);
visitChildren(orderedList);
listHolder = listHolder.parent;
- writer.setTight(oldTight);
+ writer.popTight();
writer.block();
}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
index f648d9c4f..1231a4a73 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
@@ -13,10 +13,13 @@ public class MarkdownWriter {
private final Appendable buffer;
private int blockSeparator = 0;
- private boolean tight;
private char lastChar;
private boolean atLineStart = true;
+
+ // Stacks of settings that affect various rendering behaviors. The common pattern here is that callers use "push" to
+ // change a setting, render some nodes, and then "pop" the setting off the stack again to restore previous state.
private final LinkedList prefixes = new LinkedList<>();
+ private final LinkedList tight = new LinkedList<>();
private final LinkedList rawEscapes = new LinkedList<>();
public MarkdownWriter(Appendable out) {
@@ -72,7 +75,7 @@ public void line() {
public void block() {
// Remember whether this should be a tight or loose separator now because tight could get changed in between
// this and the next flush.
- blockSeparator = tight ? 1 : 2;
+ blockSeparator = isTight() ? 1 : 2;
atLineStart = true;
}
@@ -104,6 +107,25 @@ public void popPrefix() {
prefixes.removeLast();
}
+ /**
+ * Change whether blocks are tight or loose. Loose is the default where blocks are separated by a blank line. Tight
+ * is where blocks are not separated by a blank line. Tight blocks are used in lists, if there are no blank lines
+ * within the list.
+ *
+ * Note that changing this does not affect block separators that have already been enqueued with {@link #block()},
+ * only future ones.
+ */
+ public void pushTight(boolean tight) {
+ this.tight.addLast(tight);
+ }
+
+ /**
+ * Remove the last "tight" setting from the top of the stack.
+ */
+ public void popTight() {
+ this.tight.removeLast();
+ }
+
/**
* Escape the characters matching the supplied matcher, in all text (text and raw). This might be useful to
* extensions that add another layer of syntax, e.g. the tables extension that uses `|` to separate cells and needs
@@ -136,25 +158,6 @@ public boolean isAtLineStart() {
return atLineStart;
}
- /**
- * @return whether blocks are currently set to tight or loose, see {@link #setTight(boolean)}
- */
- public boolean getTight() {
- return tight;
- }
-
- /**
- * Change whether blocks are tight or loose. Loose is the default where blocks are separated by a blank line. Tight
- * is where blocks are not separated by a blank line. Tight blocks are used in lists, if there are no blank lines
- * within the list.
- *
- * Note that changing this does not affect block separators that have already been enqueued (with {@link #block()},
- * only future ones.
- */
- public void setTight(boolean tight) {
- this.tight = tight;
- }
-
private void write(String s, CharMatcher escape) {
try {
if (rawEscapes.isEmpty() && escape == null) {
@@ -224,6 +227,10 @@ private void append(char c, CharMatcher escape) throws IOException {
}
}
+ private boolean isTight() {
+ return !tight.isEmpty() && tight.getLast();
+ }
+
private boolean needsEscaping(char c, CharMatcher escape) {
return (escape != null && escape.matches(c)) || rawNeedsEscaping(c);
}
From aaf18a2974d55bb9d0c83ce97e97e1439ff27dc2 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 6 Feb 2024 12:58:30 +1100
Subject: [PATCH 057/286] Use tight blocks to render tables
---
.../tables/internal/TableMarkdownNodeRenderer.java | 14 ++++----------
1 file changed, 4 insertions(+), 10 deletions(-)
diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableMarkdownNodeRenderer.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableMarkdownNodeRenderer.java
index bf94db5bb..487147164 100644
--- a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableMarkdownNodeRenderer.java
+++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableMarkdownNodeRenderer.java
@@ -30,15 +30,15 @@ public TableMarkdownNodeRenderer(MarkdownNodeRendererContext context) {
@Override
protected void renderBlock(TableBlock node) {
columns.clear();
+ writer.pushTight(true);
renderChildren(node);
+ writer.popTight();
writer.block();
}
@Override
protected void renderHead(TableHead node) {
renderChildren(node);
- // TODO: Not sure about this.. Should block() detect if a line was already written? Or should line() itself be lazy?
- writer.line();
for (TableCell.Alignment columnAlignment : columns) {
writer.raw('|');
if (columnAlignment == TableCell.Alignment.LEFT) {
@@ -52,10 +52,7 @@ protected void renderHead(TableHead node) {
}
}
writer.raw("|");
- // TODO
- if (node.getNext() != null) {
- writer.line();
- }
+ writer.block();
}
@Override
@@ -68,10 +65,7 @@ protected void renderRow(TableRow node) {
renderChildren(node);
// Trailing | at the end of the line
writer.raw("|");
- // TODO
- if (node.getNext() != null) {
- writer.line();
- }
+ writer.block();
}
@Override
From 01a52bce23dd7efb1a31a772d086b34fd9930b0d Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 6 Feb 2024 22:04:50 +1100
Subject: [PATCH 058/286] Rename file
---
...arkdownRenderingTest.java => TableMarkdownRendererTest.java} | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
rename commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/{TablesMarkdownRenderingTest.java => TableMarkdownRendererTest.java} (98%)
diff --git a/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesMarkdownRenderingTest.java b/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TableMarkdownRendererTest.java
similarity index 98%
rename from commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesMarkdownRenderingTest.java
rename to commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TableMarkdownRendererTest.java
index 8689e4ac2..cf63fb19c 100644
--- a/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TablesMarkdownRenderingTest.java
+++ b/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TableMarkdownRendererTest.java
@@ -10,7 +10,7 @@
import static org.junit.Assert.assertEquals;
-public class TablesMarkdownRenderingTest {
+public class TableMarkdownRendererTest {
private static final Set EXTENSIONS = Collections.singleton(TablesExtension.create());
private static final Parser PARSER = Parser.builder().extensions(EXTENSIONS).build();
From a490faee954641b6a8d8bacd754f832c4eb646ec Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 6 Feb 2024 22:31:13 +1100
Subject: [PATCH 059/286] Strikethrough markdown renderer
---
.../ext/gfm/strikethrough/Strikethrough.java | 12 ++++---
.../strikethrough/StrikethroughExtension.java | 30 +++++++++++-----
.../StrikethroughDelimiterProcessor.java | 3 +-
.../StrikethroughMarkdownNodeRenderer.java | 34 ++++++++++++++++++
.../StrikethroughMarkdownRendererTest.java | 36 +++++++++++++++++++
5 files changed, 102 insertions(+), 13 deletions(-)
create mode 100644 commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughMarkdownNodeRenderer.java
create mode 100644 commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughMarkdownRendererTest.java
diff --git a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/Strikethrough.java b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/Strikethrough.java
index 115ae9ea4..0c24642bc 100644
--- a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/Strikethrough.java
+++ b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/Strikethrough.java
@@ -4,19 +4,23 @@
import org.commonmark.node.Delimited;
/**
- * A strikethrough node containing text and other inline nodes nodes as children.
+ * A strikethrough node containing text and other inline nodes as children.
*/
public class Strikethrough extends CustomNode implements Delimited {
- private static final String DELIMITER = "~~";
+ private String delimiter;
+
+ public Strikethrough(String delimiter) {
+ this.delimiter = delimiter;
+ }
@Override
public String getOpeningDelimiter() {
- return DELIMITER;
+ return delimiter;
}
@Override
public String getClosingDelimiter() {
- return DELIMITER;
+ return delimiter;
}
}
diff --git a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/StrikethroughExtension.java b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/StrikethroughExtension.java
index 4f0228a1c..aa7dff716 100644
--- a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/StrikethroughExtension.java
+++ b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/StrikethroughExtension.java
@@ -1,17 +1,21 @@
package org.commonmark.ext.gfm.strikethrough;
import org.commonmark.Extension;
-import org.commonmark.renderer.text.TextContentRenderer;
-import org.commonmark.renderer.text.TextContentNodeRendererContext;
-import org.commonmark.renderer.text.TextContentNodeRendererFactory;
import org.commonmark.ext.gfm.strikethrough.internal.StrikethroughDelimiterProcessor;
import org.commonmark.ext.gfm.strikethrough.internal.StrikethroughHtmlNodeRenderer;
+import org.commonmark.ext.gfm.strikethrough.internal.StrikethroughMarkdownNodeRenderer;
import org.commonmark.ext.gfm.strikethrough.internal.StrikethroughTextContentNodeRenderer;
-import org.commonmark.renderer.html.HtmlRenderer;
-import org.commonmark.renderer.html.HtmlNodeRendererContext;
-import org.commonmark.renderer.html.HtmlNodeRendererFactory;
import org.commonmark.parser.Parser;
import org.commonmark.renderer.NodeRenderer;
+import org.commonmark.renderer.html.HtmlNodeRendererContext;
+import org.commonmark.renderer.html.HtmlNodeRendererFactory;
+import org.commonmark.renderer.html.HtmlRenderer;
+import org.commonmark.renderer.markdown.MarkdownNodeRendererContext;
+import org.commonmark.renderer.markdown.MarkdownNodeRendererFactory;
+import org.commonmark.renderer.markdown.MarkdownRenderer;
+import org.commonmark.renderer.text.TextContentNodeRendererContext;
+import org.commonmark.renderer.text.TextContentNodeRendererFactory;
+import org.commonmark.renderer.text.TextContentRenderer;
/**
* Extension for GFM strikethrough using {@code ~} or {@code ~~} (GitHub Flavored Markdown).
@@ -42,7 +46,7 @@
*
*/
public class StrikethroughExtension implements Parser.ParserExtension, HtmlRenderer.HtmlRendererExtension,
- TextContentRenderer.TextContentRendererExtension {
+ TextContentRenderer.TextContentRendererExtension, MarkdownRenderer.MarkdownRendererExtension {
private final boolean requireTwoTildes;
@@ -89,13 +93,23 @@ public NodeRenderer create(TextContentNodeRendererContext context) {
});
}
+ @Override
+ public void extend(MarkdownRenderer.Builder rendererBuilder) {
+ rendererBuilder.nodeRendererFactory(new MarkdownNodeRendererFactory() {
+ @Override
+ public NodeRenderer create(MarkdownNodeRendererContext context) {
+ return new StrikethroughMarkdownNodeRenderer(context);
+ }
+ });
+ }
+
public static class Builder {
private boolean requireTwoTildes = false;
/**
* @param requireTwoTildes Whether two tilde characters ({@code ~~}) are required for strikethrough or whether
- * one is also enough. Default is {@code false}; both a single tilde and two tildes can be used for strikethrough.
+ * one is also enough. Default is {@code false}; both a single tilde and two tildes can be used for strikethrough.
* @return {@code this}
*/
public Builder requireTwoTildes(boolean requireTwoTildes) {
diff --git a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java
index 3dedff1b9..4657106ab 100644
--- a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java
+++ b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughDelimiterProcessor.java
@@ -43,7 +43,8 @@ public int process(DelimiterRun openingRun, DelimiterRun closingRun) {
Text opener = openingRun.getOpener();
// Wrap nodes between delimiters in strikethrough.
- Node strikethrough = new Strikethrough();
+ String delimiter = openingRun.length() == 1 ? opener.getLiteral() : opener.getLiteral() + opener.getLiteral();
+ Node strikethrough = new Strikethrough(delimiter);
SourceSpans sourceSpans = new SourceSpans();
sourceSpans.addAllFrom(openingRun.getOpeners(openingRun.length()));
diff --git a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughMarkdownNodeRenderer.java b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughMarkdownNodeRenderer.java
new file mode 100644
index 000000000..1c91dd64f
--- /dev/null
+++ b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/internal/StrikethroughMarkdownNodeRenderer.java
@@ -0,0 +1,34 @@
+package org.commonmark.ext.gfm.strikethrough.internal;
+
+import org.commonmark.ext.gfm.strikethrough.Strikethrough;
+import org.commonmark.node.Node;
+import org.commonmark.renderer.markdown.MarkdownNodeRendererContext;
+import org.commonmark.renderer.markdown.MarkdownWriter;
+
+public class StrikethroughMarkdownNodeRenderer extends StrikethroughNodeRenderer {
+
+ private final MarkdownNodeRendererContext context;
+ private final MarkdownWriter writer;
+
+ public StrikethroughMarkdownNodeRenderer(MarkdownNodeRendererContext context) {
+ this.context = context;
+ this.writer = context.getWriter();
+ }
+
+ @Override
+ public void render(Node node) {
+ Strikethrough strikethrough = (Strikethrough) node;
+ writer.raw(strikethrough.getOpeningDelimiter());
+ renderChildren(node);
+ writer.raw(strikethrough.getClosingDelimiter());
+ }
+
+ private void renderChildren(Node parent) {
+ Node node = parent.getFirstChild();
+ while (node != null) {
+ Node next = node.getNext();
+ context.render(node);
+ node = next;
+ }
+ }
+}
diff --git a/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughMarkdownRendererTest.java b/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughMarkdownRendererTest.java
new file mode 100644
index 000000000..b722018b8
--- /dev/null
+++ b/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughMarkdownRendererTest.java
@@ -0,0 +1,36 @@
+package org.commonmark.ext.gfm.strikethrough;
+
+import org.commonmark.Extension;
+import org.commonmark.parser.Parser;
+import org.commonmark.renderer.markdown.MarkdownRenderer;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+
+public class StrikethroughMarkdownRendererTest {
+
+ private static final Set EXTENSIONS = Collections.singleton(StrikethroughExtension.create());
+ private static final Parser PARSER = Parser.builder().extensions(EXTENSIONS).build();
+ private static final MarkdownRenderer RENDERER = MarkdownRenderer.builder().extensions(EXTENSIONS).build();
+
+ @Test
+ public void testStrikethrough() {
+ assertRoundTrip("~foo~ ~bar~\n");
+ assertRoundTrip("~~f~oo~~ ~~bar~~\n");
+
+ // TODO this new special character needs to be escaped:
+// assertRoundTrip("\\~foo\\~\n");
+ }
+
+ protected String render(String source) {
+ return RENDERER.render(PARSER.parse(source));
+ }
+
+ private void assertRoundTrip(String input) {
+ String rendered = render(input);
+ assertEquals(input, rendered);
+ }
+}
From 981d2363851da3fa81bd7ec333d237604b823235 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 6 Feb 2024 23:04:37 +1100
Subject: [PATCH 060/286] Add getSpecialCharacters for escaping of extension
characters
---
.../strikethrough/StrikethroughExtension.java | 8 +++++++
.../StrikethroughMarkdownRendererTest.java | 6 +++---
.../ext/gfm/tables/TablesExtension.java | 8 +++++++
.../internal/util/AsciiMatcher.java | 16 ++++++++++----
.../markdown/CoreMarkdownNodeRenderer.java | 9 ++++----
.../markdown/MarkdownNodeRendererContext.java | 8 +++++++
.../markdown/MarkdownNodeRendererFactory.java | 8 +++++++
.../renderer/markdown/MarkdownRenderer.java | 21 +++++++++++++++++--
8 files changed, 71 insertions(+), 13 deletions(-)
diff --git a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/StrikethroughExtension.java b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/StrikethroughExtension.java
index aa7dff716..f87f3e9c8 100644
--- a/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/StrikethroughExtension.java
+++ b/commonmark-ext-gfm-strikethrough/src/main/java/org/commonmark/ext/gfm/strikethrough/StrikethroughExtension.java
@@ -17,6 +17,9 @@
import org.commonmark.renderer.text.TextContentNodeRendererFactory;
import org.commonmark.renderer.text.TextContentRenderer;
+import java.util.Collections;
+import java.util.Set;
+
/**
* Extension for GFM strikethrough using {@code ~} or {@code ~~} (GitHub Flavored Markdown).
*
Example input:
@@ -100,6 +103,11 @@ public void extend(MarkdownRenderer.Builder rendererBuilder) {
public NodeRenderer create(MarkdownNodeRendererContext context) {
return new StrikethroughMarkdownNodeRenderer(context);
}
+
+ @Override
+ public Set getSpecialCharacters() {
+ return Collections.singleton('~');
+ }
});
}
diff --git a/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughMarkdownRendererTest.java b/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughMarkdownRendererTest.java
index b722018b8..96df48cec 100644
--- a/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughMarkdownRendererTest.java
+++ b/commonmark-ext-gfm-strikethrough/src/test/java/org/commonmark/ext/gfm/strikethrough/StrikethroughMarkdownRendererTest.java
@@ -19,10 +19,10 @@ public class StrikethroughMarkdownRendererTest {
@Test
public void testStrikethrough() {
assertRoundTrip("~foo~ ~bar~\n");
- assertRoundTrip("~~f~oo~~ ~~bar~~\n");
+ assertRoundTrip("~~foo~~ ~~bar~~\n");
+ assertRoundTrip("~~f\\~oo~~ ~~bar~~\n");
- // TODO this new special character needs to be escaped:
-// assertRoundTrip("\\~foo\\~\n");
+ assertRoundTrip("\\~foo\\~\n");
}
protected String render(String source) {
diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TablesExtension.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TablesExtension.java
index d23f6f5fc..92e1f0ba4 100644
--- a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TablesExtension.java
+++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TablesExtension.java
@@ -17,6 +17,9 @@
import org.commonmark.renderer.text.TextContentNodeRendererFactory;
import org.commonmark.renderer.text.TextContentRenderer;
+import java.util.Collections;
+import java.util.Set;
+
/**
* Extension for GFM tables using "|" pipes (GitHub Flavored Markdown).
*
@@ -72,6 +75,11 @@ public void extend(MarkdownRenderer.Builder rendererBuilder) {
public NodeRenderer create(MarkdownNodeRendererContext context) {
return new TableMarkdownNodeRenderer(context);
}
+
+ @Override
+ public Set getSpecialCharacters() {
+ return Collections.emptySet();
+ }
});
}
}
diff --git a/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java b/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java
index d31020fa3..dd7e8d5eb 100644
--- a/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java
+++ b/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java
@@ -1,6 +1,7 @@
package org.commonmark.internal.util;
import java.util.BitSet;
+import java.util.Set;
public class AsciiMatcher implements CharMatcher {
private final BitSet set;
@@ -33,6 +34,14 @@ private Builder(BitSet set) {
this.set = set;
}
+ public Builder c(char c) {
+ if (c > 127) {
+ throw new IllegalArgumentException("Can only match ASCII characters");
+ }
+ set.set(c);
+ return this;
+ }
+
public Builder anyOf(String s) {
for (int i = 0; i < s.length(); i++) {
c(s.charAt(i));
@@ -40,11 +49,10 @@ public Builder anyOf(String s) {
return this;
}
- public Builder c(char c) {
- if (c > 127) {
- throw new IllegalArgumentException("Can only match ASCII characters");
+ public Builder anyOf(Set characters) {
+ for (Character c : characters) {
+ c(c);
}
- set.set(c);
return this;
}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index c7fa9be7a..2db7ef30c 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -22,10 +22,8 @@
*/
public class CoreMarkdownNodeRenderer extends AbstractVisitor implements NodeRenderer {
- private final AsciiMatcher textEscape =
- AsciiMatcher.builder().anyOf("[]<>`*&\n\\").build();
- private final CharMatcher textEscapeInHeading =
- AsciiMatcher.builder(textEscape).anyOf("#").build();
+ private final AsciiMatcher textEscape;
+ private final CharMatcher textEscapeInHeading;
private final CharMatcher linkDestinationNeedsAngleBrackets =
AsciiMatcher.builder().c(' ').c('(').c(')').c('<').c('>').c('\n').c('\\').build();
private final CharMatcher linkDestinationEscapeInAngleBrackets =
@@ -46,6 +44,9 @@ public class CoreMarkdownNodeRenderer extends AbstractVisitor implements NodeRen
public CoreMarkdownNodeRenderer(MarkdownNodeRendererContext context) {
this.context = context;
this.writer = context.getWriter();
+
+ textEscape = AsciiMatcher.builder().anyOf("[]<>`*&\n\\").anyOf(context.getSpecialCharacters()).build();
+ textEscapeInHeading = AsciiMatcher.builder(textEscape).anyOf("#").build();
}
@Override
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererContext.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererContext.java
index 8fe0f73d5..5805c458b 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererContext.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererContext.java
@@ -2,6 +2,8 @@
import org.commonmark.node.Node;
+import java.util.Set;
+
public interface MarkdownNodeRendererContext {
/**
@@ -16,4 +18,10 @@ public interface MarkdownNodeRendererContext {
* @param node the node to render
*/
void render(Node node);
+
+ /**
+ * @return additional special characters that need to be escaped if they occur in normal text; currently only ASCII
+ * characters are allowed
+ */
+ Set getSpecialCharacters();
}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererFactory.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererFactory.java
index 7b3134277..adfe8a07b 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererFactory.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererFactory.java
@@ -2,6 +2,8 @@
import org.commonmark.renderer.NodeRenderer;
+import java.util.Set;
+
/**
* Factory for instantiating new node renderers Æ’or rendering.
*/
@@ -14,4 +16,10 @@ public interface MarkdownNodeRendererFactory {
* @return a node renderer
*/
NodeRenderer create(MarkdownNodeRendererContext context);
+
+ /**
+ * @return the additional special characters that this factory would like to have escaped in normal text; currently
+ * only ASCII characters are allowed
+ */
+ Set getSpecialCharacters();
}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
index 4dee17ed6..926105202 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
@@ -6,8 +6,7 @@
import org.commonmark.renderer.NodeRenderer;
import org.commonmark.renderer.Renderer;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
/**
* Renders nodes to CommonMark Markdown.
@@ -32,6 +31,11 @@ private MarkdownRenderer(Builder builder) {
public NodeRenderer create(MarkdownNodeRendererContext context) {
return new CoreMarkdownNodeRenderer(context);
}
+
+ @Override
+ public Set getSpecialCharacters() {
+ return Collections.emptySet();
+ }
});
}
@@ -111,13 +115,21 @@ public interface MarkdownRendererExtension extends Extension {
private class RendererContext implements MarkdownNodeRendererContext {
private final MarkdownWriter writer;
private final NodeRendererMap nodeRendererMap = new NodeRendererMap();
+ private final Set additionalTextEscapes;
private RendererContext(MarkdownWriter writer) {
+ // Set fields that are used by interface
this.writer = writer;
+ Set escapes = new HashSet();
+ for (MarkdownNodeRendererFactory factory : nodeRendererFactories) {
+ escapes.addAll(factory.getSpecialCharacters());
+ }
+ additionalTextEscapes = Collections.unmodifiableSet(escapes);
// The first node renderer for a node type "wins".
for (int i = nodeRendererFactories.size() - 1; i >= 0; i--) {
MarkdownNodeRendererFactory nodeRendererFactory = nodeRendererFactories.get(i);
+ // Pass in this as context here, which uses the fields set above
NodeRenderer nodeRenderer = nodeRendererFactory.create(this);
nodeRendererMap.add(nodeRenderer);
}
@@ -132,5 +144,10 @@ public MarkdownWriter getWriter() {
public void render(Node node) {
nodeRendererMap.render(node);
}
+
+ @Override
+ public Set getSpecialCharacters() {
+ return additionalTextEscapes;
+ }
}
}
From 64eafc413dfc5bc936dd1ae7dbf189d46a8652fa Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Tue, 6 Feb 2024 23:06:42 +1100
Subject: [PATCH 061/286] Use getSpecialCharacters for table extension too
---
.../java/org/commonmark/ext/gfm/tables/TablesExtension.java | 2 +-
.../ext/gfm/tables/TableMarkdownRendererTest.java | 6 ++++++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TablesExtension.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TablesExtension.java
index 92e1f0ba4..d18c38283 100644
--- a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TablesExtension.java
+++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/TablesExtension.java
@@ -78,7 +78,7 @@ public NodeRenderer create(MarkdownNodeRendererContext context) {
@Override
public Set getSpecialCharacters() {
- return Collections.emptySet();
+ return Collections.singleton('|');
}
});
}
diff --git a/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TableMarkdownRendererTest.java b/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TableMarkdownRendererTest.java
index cf63fb19c..1db917d08 100644
--- a/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TableMarkdownRendererTest.java
+++ b/commonmark-ext-gfm-tables/src/test/java/org/commonmark/ext/gfm/tables/TableMarkdownRendererTest.java
@@ -59,6 +59,12 @@ public void testEscaping() {
assertRoundTrip("|Abc|Def|\n|---|---|\n|Inline HTML|Foo\\|bar|\n");
}
+ @Test
+ public void testEscaped() {
+ // `|` in Text nodes needs to be escaped, otherwise the generated Markdown does not get parsed back as a table
+ assertRoundTrip("\\|Abc\\|\n\\|---\\|\n");
+ }
+
protected String render(String source) {
return RENDERER.render(PARSER.parse(source));
}
From 2966df1113eb28c5a3940718e62a25a75a24fd69 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Wed, 7 Feb 2024 19:13:55 +1100
Subject: [PATCH 062/286] Ins extensions markdown renderer
---
.../org/commonmark/ext/ins/InsExtension.java | 28 +++++++++++++--
.../ins/internal/InsMarkdownNodeRenderer.java | 32 +++++++++++++++++
.../ext/ins/InsMarkdownRendererTest.java | 34 +++++++++++++++++++
3 files changed, 91 insertions(+), 3 deletions(-)
create mode 100644 commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/internal/InsMarkdownNodeRenderer.java
create mode 100644 commonmark-ext-ins/src/test/java/org/commonmark/ext/ins/InsMarkdownRendererTest.java
diff --git a/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/InsExtension.java b/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/InsExtension.java
index 2f980d93b..065719558 100644
--- a/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/InsExtension.java
+++ b/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/InsExtension.java
@@ -3,16 +3,23 @@
import org.commonmark.Extension;
import org.commonmark.ext.ins.internal.InsDelimiterProcessor;
import org.commonmark.ext.ins.internal.InsHtmlNodeRenderer;
+import org.commonmark.ext.ins.internal.InsMarkdownNodeRenderer;
import org.commonmark.ext.ins.internal.InsTextContentNodeRenderer;
import org.commonmark.parser.Parser;
import org.commonmark.renderer.NodeRenderer;
import org.commonmark.renderer.html.HtmlNodeRendererContext;
import org.commonmark.renderer.html.HtmlNodeRendererFactory;
import org.commonmark.renderer.html.HtmlRenderer;
+import org.commonmark.renderer.markdown.MarkdownNodeRendererContext;
+import org.commonmark.renderer.markdown.MarkdownNodeRendererFactory;
+import org.commonmark.renderer.markdown.MarkdownRenderer;
import org.commonmark.renderer.text.TextContentNodeRendererContext;
import org.commonmark.renderer.text.TextContentNodeRendererFactory;
import org.commonmark.renderer.text.TextContentRenderer;
+import java.util.Collections;
+import java.util.Set;
+
/**
* Extension for ins using ++
*
@@ -24,9 +31,7 @@
* The parsed ins text regions are turned into {@link Ins} nodes.
*
*/
-public class InsExtension implements Parser.ParserExtension,
- HtmlRenderer.HtmlRendererExtension,
- TextContentRenderer.TextContentRendererExtension {
+public class InsExtension implements Parser.ParserExtension, HtmlRenderer.HtmlRendererExtension, TextContentRenderer.TextContentRendererExtension, MarkdownRenderer.MarkdownRendererExtension {
private InsExtension() {
}
@@ -59,4 +64,21 @@ public NodeRenderer create(TextContentNodeRendererContext context) {
}
});
}
+
+ @Override
+ public void extend(MarkdownRenderer.Builder rendererBuilder) {
+ rendererBuilder.nodeRendererFactory(new MarkdownNodeRendererFactory() {
+ @Override
+ public NodeRenderer create(MarkdownNodeRendererContext context) {
+ return new InsMarkdownNodeRenderer(context);
+ }
+
+ @Override
+ public Set getSpecialCharacters() {
+ // We technically don't need to escape single occurrences of +, but that's all the extension API
+ // exposes currently.
+ return Collections.singleton('+');
+ }
+ });
+ }
}
diff --git a/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/internal/InsMarkdownNodeRenderer.java b/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/internal/InsMarkdownNodeRenderer.java
new file mode 100644
index 000000000..851d47282
--- /dev/null
+++ b/commonmark-ext-ins/src/main/java/org/commonmark/ext/ins/internal/InsMarkdownNodeRenderer.java
@@ -0,0 +1,32 @@
+package org.commonmark.ext.ins.internal;
+
+import org.commonmark.node.Node;
+import org.commonmark.renderer.markdown.MarkdownNodeRendererContext;
+import org.commonmark.renderer.markdown.MarkdownWriter;
+
+public class InsMarkdownNodeRenderer extends InsNodeRenderer {
+
+ private final MarkdownNodeRendererContext context;
+ private final MarkdownWriter writer;
+
+ public InsMarkdownNodeRenderer(MarkdownNodeRendererContext context) {
+ this.context = context;
+ this.writer = context.getWriter();
+ }
+
+ @Override
+ public void render(Node node) {
+ writer.raw("++");
+ renderChildren(node);
+ writer.raw("++");
+ }
+
+ private void renderChildren(Node parent) {
+ Node node = parent.getFirstChild();
+ while (node != null) {
+ Node next = node.getNext();
+ context.render(node);
+ node = next;
+ }
+ }
+}
diff --git a/commonmark-ext-ins/src/test/java/org/commonmark/ext/ins/InsMarkdownRendererTest.java b/commonmark-ext-ins/src/test/java/org/commonmark/ext/ins/InsMarkdownRendererTest.java
new file mode 100644
index 000000000..16cefc7f1
--- /dev/null
+++ b/commonmark-ext-ins/src/test/java/org/commonmark/ext/ins/InsMarkdownRendererTest.java
@@ -0,0 +1,34 @@
+package org.commonmark.ext.ins;
+
+import org.commonmark.Extension;
+import org.commonmark.parser.Parser;
+import org.commonmark.renderer.markdown.MarkdownRenderer;
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+
+public class InsMarkdownRendererTest {
+
+ private static final Set EXTENSIONS = Collections.singleton(InsExtension.create());
+ private static final Parser PARSER = Parser.builder().extensions(EXTENSIONS).build();
+ private static final MarkdownRenderer RENDERER = MarkdownRenderer.builder().extensions(EXTENSIONS).build();
+
+ @Test
+ public void testStrikethrough() {
+ assertRoundTrip("++foo++\n");
+
+ assertRoundTrip("\\+\\+foo\\+\\+\n");
+ }
+
+ protected String render(String source) {
+ return RENDERER.render(PARSER.parse(source));
+ }
+
+ private void assertRoundTrip(String input) {
+ String rendered = render(input);
+ assertEquals(input, rendered);
+ }
+}
From 0c5ad6e1f16ae2b655db93fd07c0af502fead16e Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 8 Feb 2024 23:26:03 +1100
Subject: [PATCH 063/286] Use parsed delimiter for emphasis if available
---
.../markdown/CoreMarkdownNodeRenderer.java | 10 ++++--
.../markdown/MarkdownRendererTest.java | 34 ++++++++++++++-----
2 files changed, 32 insertions(+), 12 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 2db7ef30c..748ff5dfb 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -45,7 +45,7 @@ public CoreMarkdownNodeRenderer(MarkdownNodeRendererContext context) {
this.context = context;
this.writer = context.getWriter();
- textEscape = AsciiMatcher.builder().anyOf("[]<>`*&\n\\").anyOf(context.getSpecialCharacters()).build();
+ textEscape = AsciiMatcher.builder().anyOf("[]<>`*_&\n\\").anyOf(context.getSpecialCharacters()).build();
textEscapeInHeading = AsciiMatcher.builder(textEscape).anyOf("#").build();
}
@@ -282,8 +282,12 @@ public void visit(Code code) {
@Override
public void visit(Emphasis emphasis) {
- // When emphasis is nested, a different delimiter needs to be used
- char delimiter = writer.getLastChar() == '*' ? '_' : '*';
+ String delimiter = emphasis.getOpeningDelimiter();
+ // Use delimiter that was parsed if available
+ if (delimiter == null) {
+ // When emphasis is nested, a different delimiter needs to be used
+ delimiter = writer.getLastChar() == '*' ? "_" : "*";
+ }
writer.raw(delimiter);
super.visit(emphasis);
writer.raw(delimiter);
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index 20453eed7..af6a3488a 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -1,6 +1,6 @@
package org.commonmark.renderer.markdown;
-import org.commonmark.node.Node;
+import org.commonmark.node.*;
import org.commonmark.parser.Parser;
import org.junit.Test;
@@ -173,9 +173,21 @@ public void testEmphasis() {
// When nesting, a different delimiter needs to be used
assertRoundTrip("*_foo_*\n");
assertRoundTrip("*_*foo*_*\n");
+ assertRoundTrip("_*foo*_\n");
// Not emphasis (needs * inside words)
- assertRoundTrip("foo_bar_\n");
+ assertRoundTrip("foo\\_bar\\_\n");
+
+ // Even when rendering a manually constructed tree, the emphasis delimiter needs to be chosen correctly.
+ Document doc = new Document();
+ Paragraph p = new Paragraph();
+ doc.appendChild(p);
+ Emphasis e1 = new Emphasis();
+ p.appendChild(e1);
+ Emphasis e2 = new Emphasis();
+ e1.appendChild(e2);
+ e2.appendChild(new Text("hi"));
+ assertEquals("*_hi_*\n", render(doc));
}
@Test
@@ -226,17 +238,21 @@ public void testSoftLineBreaks() {
assertRoundTrip("foo\nbar\n");
}
- private Node parse(String source) {
- return Parser.builder().build().parse(source);
+ private void assertRoundTrip(String input) {
+ String rendered = parseAndRender(input);
+ assertEquals(input, rendered);
}
- private String render(String source) {
+ private String parseAndRender(String source) {
Node parsed = parse(source);
- return MarkdownRenderer.builder().build().render(parsed);
+ return render(parsed);
}
- private void assertRoundTrip(String input) {
- String rendered = render(input);
- assertEquals(input, rendered);
+ private Node parse(String source) {
+ return Parser.builder().build().parse(source);
+ }
+
+ private String render(Node node) {
+ return MarkdownRenderer.builder().build().render(node);
}
}
From 5a0960f9903ffef5b3b488969fd922c6e2067b4d Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 8 Feb 2024 23:26:32 +1100
Subject: [PATCH 064/286] Add integration test for multiple extensions
---
.../MarkdownRendererIntegrationTest.java | 46 +++++++++++++++++++
1 file changed, 46 insertions(+)
create mode 100644 commonmark-integration-test/src/test/java/org/commonmark/integration/MarkdownRendererIntegrationTest.java
diff --git a/commonmark-integration-test/src/test/java/org/commonmark/integration/MarkdownRendererIntegrationTest.java b/commonmark-integration-test/src/test/java/org/commonmark/integration/MarkdownRendererIntegrationTest.java
new file mode 100644
index 000000000..51f761cfd
--- /dev/null
+++ b/commonmark-integration-test/src/test/java/org/commonmark/integration/MarkdownRendererIntegrationTest.java
@@ -0,0 +1,46 @@
+package org.commonmark.integration;
+
+import org.commonmark.Extension;
+import org.commonmark.ext.autolink.AutolinkExtension;
+import org.commonmark.ext.front.matter.YamlFrontMatterExtension;
+import org.commonmark.ext.gfm.strikethrough.StrikethroughExtension;
+import org.commonmark.ext.gfm.tables.TablesExtension;
+import org.commonmark.ext.image.attributes.ImageAttributesExtension;
+import org.commonmark.ext.ins.InsExtension;
+import org.commonmark.ext.task.list.items.TaskListItemsExtension;
+import org.commonmark.parser.Parser;
+import org.commonmark.renderer.markdown.MarkdownRenderer;
+import org.junit.Test;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+public class MarkdownRendererIntegrationTest {
+
+ private static final List EXTENSIONS = Arrays.asList(
+ AutolinkExtension.create(),
+ ImageAttributesExtension.create(),
+ InsExtension.create(),
+ StrikethroughExtension.create(),
+ TablesExtension.create(),
+ TaskListItemsExtension.create(),
+ YamlFrontMatterExtension.create());
+ private static final Parser PARSER = Parser.builder().extensions(EXTENSIONS).build();
+ private static final MarkdownRenderer RENDERER = MarkdownRenderer.builder().extensions(EXTENSIONS).build();
+
+ @Test
+ public void testStrikethroughInTable() {
+ assertRoundTrip("|Abc|\n|---|\n|~strikethrough~|\n|\\~escaped\\~|\n");
+ }
+
+ private String render(String source) {
+ return RENDERER.render(PARSER.parse(source));
+ }
+
+ private void assertRoundTrip(String input) {
+ String rendered = render(input);
+ assertEquals(input, rendered);
+ }
+}
From 114956af772cb82aa8f5c262e735aadcb0adc9cf Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 8 Feb 2024 23:31:37 +1100
Subject: [PATCH 065/286] Fix thematic break and list items
---
.../renderer/markdown/CoreMarkdownNodeRenderer.java | 3 ++-
.../renderer/markdown/MarkdownRendererTest.java | 9 +++++----
2 files changed, 7 insertions(+), 5 deletions(-)
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 748ff5dfb..30d6d1a5c 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -89,7 +89,8 @@ public void visit(Document document) {
@Override
public void visit(ThematicBreak thematicBreak) {
- writer.raw("***");
+ // Let's use ___ as it doesn't introduce ambiguity with * or - list item markers
+ writer.raw("___");
writer.block();
}
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index af6a3488a..eaabe837c 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -12,10 +12,11 @@ public class MarkdownRendererTest {
@Test
public void testThematicBreaks() {
- assertRoundTrip("***\n");
- // TODO: spec: If you want a thematic break in a list item, use a different bullet:
-
- assertRoundTrip("***\n\nfoo\n");
+ assertRoundTrip("___\n");
+ assertRoundTrip("___\n\nfoo\n");
+ // List item with hr -> hr needs to not use the same as the marker
+ assertRoundTrip("* ___\n");
+ assertRoundTrip("- ___\n");
}
@Test
From a88ab234e01ab4e1f39e1466490536f7d1f5d93b Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Sat, 10 Feb 2024 11:54:30 +1100
Subject: [PATCH 066/286] README: Bump versions
---
README.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/README.md b/README.md
index bcf552c5b..5e1342740 100644
--- a/README.md
+++ b/README.md
@@ -34,7 +34,7 @@ Coordinates for core library (see all on [Maven Central]):
org.commonmarkcommonmark
- 0.20.0
+ 0.21.0
```
@@ -234,7 +234,7 @@ First, add an additional dependency (see [Maven Central] for others):
org.commonmarkcommonmark-ext-gfm-tables
- 0.20.0
+ 0.21.0
```
From 2cfe732fc8c67ab04350c1b54326e46ce4039d60 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Mon, 2 Oct 2023 19:26:48 +1100
Subject: [PATCH 067/286] Test on Java 21
Bump `actions/setup-java` to `v2` as well, which requires specifying a
distribution and use 8 instead of 1.8.
---
.github/workflows/ci.yml | 15 +++++++++------
.github/workflows/release.yml | 5 +++--
2 files changed, 12 insertions(+), 8 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 381d7c73f..df0fa28fb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,15 +9,16 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- java: [1.8, 11, 17]
+ java: [8, 11, 17, 21]
steps:
- name: Checkout sources
uses: actions/checkout@v2
- name: Set up JDK
- uses: actions/setup-java@v1
+ uses: actions/setup-java@v2
with:
java-version: ${{ matrix.java }}
+ distribution: 'zulu'
- name: Build
run: mvn -B package javadoc:javadoc
@@ -29,9 +30,10 @@ jobs:
uses: actions/checkout@v2
- name: Set up JDK
- uses: actions/setup-java@v1
+ uses: actions/setup-java@v2
with:
- java-version: 1.8
+ java-version: 8
+ distribution: 'zulu'
- name: Build with coverage
run: mvn -B -Pcoverage clean test jacoco:report-aggregate
@@ -46,9 +48,10 @@ jobs:
uses: actions/checkout@v2
- name: Set up JDK
- uses: actions/setup-java@v1
+ uses: actions/setup-java@v2
with:
- java-version: 1.8
+ java-version: 8
+ distribution: 'zulu'
- name: Android Lint checks
run: cd commonmark-android-test && ./gradlew :app:lint
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 4edf451c0..2d269e640 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -17,9 +17,10 @@ jobs:
uses: actions/checkout@v2
- name: Set up Maven Central repository
- uses: actions/setup-java@v1
+ uses: actions/setup-java@v2
with:
- java-version: 1.8
+ java-version: 8
+ distribution: 'zulu'
server-id: ossrh
server-username: MAVEN_USERNAME # env variable to use for username in release
server-password: MAVEN_PASSWORD # env variable to use for password in release
From 5311b9bd6d09441946a736c06b2a298c2a916368 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Sat, 10 Feb 2024 11:56:38 +1100
Subject: [PATCH 068/286] Raise minimum version to Java 11
---
.github/workflows/ci.yml | 6 +++---
README.md | 6 +++---
pom.xml | 5 ++---
3 files changed, 8 insertions(+), 9 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index df0fa28fb..3fa93f210 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,7 +9,7 @@ jobs:
runs-on: ubuntu-latest
strategy:
matrix:
- java: [8, 11, 17, 21]
+ java: [11, 17, 21]
steps:
- name: Checkout sources
uses: actions/checkout@v2
@@ -32,7 +32,7 @@ jobs:
- name: Set up JDK
uses: actions/setup-java@v2
with:
- java-version: 8
+ java-version: 11
distribution: 'zulu'
- name: Build with coverage
@@ -50,7 +50,7 @@ jobs:
- name: Set up JDK
uses: actions/setup-java@v2
with:
- java-version: 8
+ java-version: 11
distribution: 'zulu'
- name: Android Lint checks
diff --git a/README.md b/README.md
index 5e1342740..dc27c0987 100644
--- a/README.md
+++ b/README.md
@@ -23,9 +23,9 @@ full library with a nice API and the following features:
* Flexible (manipulate the AST after parsing, customize HTML rendering)
* Extensible (tables, strikethrough, autolinking and more, see below)
-The library is supported on Java 8 or later. It should work on Java 7
-and Android too, but that is on a best-effort basis, please report
-problems. For Android the minimum API level is 19, see the
+The library is supported on Java 11 and later. It should work on Android too,
+but that is on a best-effort basis, please report problems. For Android the
+minimum API level is 19, see the
[commonmark-android-test](commonmark-android-test) directory.
Coordinates for core library (see all on [Maven Central]):
diff --git a/pom.xml b/pom.xml
index f5c7749ef..da06b2c2b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -38,10 +38,9 @@
org.apache.maven.pluginsmaven-compiler-plugin
- 3.10.1
+ 3.12.1
- 7
- 7
+ 11
From 01fd7732e55b3aaa98a18c2d54b2d5dbc5dac4fa Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Sat, 10 Feb 2024 12:38:49 +1100
Subject: [PATCH 069/286] Bump release workflow version as well
---
.github/workflows/release.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 2d269e640..2ecfa6d49 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -19,7 +19,7 @@ jobs:
- name: Set up Maven Central repository
uses: actions/setup-java@v2
with:
- java-version: 8
+ java-version: 11
distribution: 'zulu'
server-id: ossrh
server-username: MAVEN_USERNAME # env variable to use for username in release
From 979d2f6bbbc6ccaa174946e9dc65d539a37c1f1c Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Thu, 15 Feb 2024 18:19:04 +1100
Subject: [PATCH 070/286] Documentation
---
README.md | 35 ++++++++++++++-----
commonmark/pom.xml | 2 +-
.../java/org/commonmark/package-info.java | 1 +
.../markdown/MarkdownNodeRendererContext.java | 3 ++
.../markdown/MarkdownNodeRendererFactory.java | 2 +-
.../renderer/markdown/MarkdownRenderer.java | 17 ++++++---
.../renderer/markdown/package-info.java | 4 +++
.../renderer/text/package-info.java | 2 +-
.../org/commonmark/test/UsageExampleTest.java | 17 +++++++--
9 files changed, 65 insertions(+), 18 deletions(-)
create mode 100644 commonmark/src/main/java/org/commonmark/renderer/markdown/package-info.java
diff --git a/README.md b/README.md
index bcf552c5b..25df264ef 100644
--- a/README.md
+++ b/README.md
@@ -13,13 +13,14 @@ Java library for parsing and rendering [Markdown] text according to the
Introduction
------------
-Provides classes for parsing input to an abstract syntax tree of nodes
-(AST), visiting and manipulating nodes, and rendering to HTML. It
-started out as a port of [commonmark.js], but has since evolved into a
-full library with a nice API and the following features:
+Provides classes for parsing input to an abstract syntax tree (AST),
+visiting and manipulating nodes, and rendering to HTML or back to Markdown.
+It started out as a port of [commonmark.js], but has since evolved into an
+extensible library with the following features:
* Small (core has no dependencies, extensions in separate artifacts)
-* Fast (10-20 times faster than pegdown, see benchmarks in repo)
+* Fast (10-20 times faster than [pegdown] which used to be a popular Markdown
+ library, see benchmarks in repo)
* Flexible (manipulate the AST after parsing, customize HTML rendering)
* Extensible (tables, strikethrough, autolinking and more, see below)
@@ -63,9 +64,9 @@ import org.commonmark.parser.Parser;
import org.commonmark.renderer.html.HtmlRenderer;
Parser parser = Parser.builder().build();
-Node document = parser.parse("This is *Sparta*");
+Node document = parser.parse("This is *Markdown*");
HtmlRenderer renderer = HtmlRenderer.builder().build();
-renderer.render(document); // "
This is Sparta
\n"
+renderer.render(document); // "
This is Markdown
\n"
```
This uses the parser and renderer with default options. Both builders have
@@ -81,8 +82,23 @@ to which tags are allowed, etc. That is the responsibility of the caller, and
if you expose the resulting HTML, you probably want to run a sanitizer on it
after this.
-For rendering to plain text, there's also a `TextContentRenderer` with
-a very similar API.
+#### Render to Markdown
+
+```java
+import org.commonmark.node.*;
+import org.commonmark.renderer.markdown.MarkdownRenderer;
+
+MarkdownRenderer renderer = MarkdownRenderer.builder().build();
+Node document = new Document();
+Heading heading = new Heading();
+heading.setLevel(2);
+heading.appendChild(new Text("My title"));
+document.appendChild(heading);
+
+renderer.render(document); // "## My title\n"
+```
+
+For rendering to plain text with minimal markup, there's also `TextContentRenderer`.
#### Use a visitor to process parsed nodes
@@ -390,6 +406,7 @@ BSD (2-clause) licensed, see LICENSE.txt file.
[CommonMark]: https://commonmark.org/
[Markdown]: https://daringfireball.net/projects/markdown/
[commonmark.js]: https://github.com/commonmark/commonmark.js
+[pegdown]: https://github.com/sirthias/pegdown
[CommonMark Dingus]: https://spec.commonmark.org/dingus/
[Maven Central]: https://search.maven.org/#search|ga|1|g%3A%22org.commonmark%22
[Semantic Versioning]: https://semver.org/
diff --git a/commonmark/pom.xml b/commonmark/pom.xml
index 64c456192..9988370a8 100644
--- a/commonmark/pom.xml
+++ b/commonmark/pom.xml
@@ -9,7 +9,7 @@
commonmarkcommonmark-java core
- Core of commonmark-java (implementation of CommonMark for parsing markdown and rendering to HTML)
+ Core of commonmark-java (a library for parsing Markdown to an AST, modifying the AST and rendering it to HTML or Markdown)
diff --git a/commonmark/src/main/java/org/commonmark/package-info.java b/commonmark/src/main/java/org/commonmark/package-info.java
index e784703e9..b683017f6 100644
--- a/commonmark/src/main/java/org/commonmark/package-info.java
+++ b/commonmark/src/main/java/org/commonmark/package-info.java
@@ -4,6 +4,7 @@
*
{@link org.commonmark.parser} for parsing input text to AST nodes
*
{@link org.commonmark.node} for AST node types and visitors
*
{@link org.commonmark.renderer.html} for HTML rendering
+ *
{@link org.commonmark.renderer.markdown} for Markdown rendering
*
*/
package org.commonmark;
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererContext.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererContext.java
index 5805c458b..40640d1b4 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererContext.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererContext.java
@@ -4,6 +4,9 @@
import java.util.Set;
+/**
+ * Context that is passed to custom node renderers, see {@link MarkdownNodeRendererFactory#create}.
+ */
public interface MarkdownNodeRendererContext {
/**
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererFactory.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererFactory.java
index adfe8a07b..14221ea56 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererFactory.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownNodeRendererFactory.java
@@ -5,7 +5,7 @@
import java.util.Set;
/**
- * Factory for instantiating new node renderers Æ’or rendering.
+ * Factory for instantiating new node renderers for rendering custom nodes.
*/
public interface MarkdownNodeRendererFactory {
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
index 926105202..25fa9a142 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownRenderer.java
@@ -9,14 +9,17 @@
import java.util.*;
/**
- * Renders nodes to CommonMark Markdown.
+ * Renders nodes to Markdown (CommonMark syntax); use {@link #builder()} to create a renderer.
*
- * Note that it does not currently attempt to preserve the exact syntax of the original input Markdown (if any):
+ * Note that it doesn't currently preserve the exact syntax of the original input Markdown (if any):
*
*
Headings are output as ATX headings if possible (multi-line headings need Setext headings)
*
Escaping might be over-eager, e.g. a plain {@code *} might be escaped
* even though it doesn't need to be in that particular context
+ *
Leading whitespace in paragraphs is not preserved
*
+ * However, it should produce Markdown that is semantically equivalent to the input, i.e. if the Markdown was parsed
+ * again and compared against the original AST, it should be the same (minus bugs).
*/
public class MarkdownRenderer implements Renderer {
@@ -66,7 +69,7 @@ public String render(Node node) {
*/
public static class Builder {
- private List nodeRendererFactories = new ArrayList<>();
+ private final List nodeRendererFactories = new ArrayList<>();
/**
* @return the configured {@link MarkdownRenderer}
@@ -106,9 +109,15 @@ public Builder extensions(Iterable extends Extension> extensions) {
}
/**
- * Extension for {@link MarkdownRenderer}.
+ * Extension for {@link MarkdownRenderer} for rendering custom nodes.
*/
public interface MarkdownRendererExtension extends Extension {
+
+ /**
+ * Extend Markdown rendering, usually by registering custom node renderers using {@link Builder#nodeRendererFactory}.
+ *
+ * @param rendererBuilder the renderer builder to extend
+ */
void extend(Builder rendererBuilder);
}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/package-info.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/package-info.java
new file mode 100644
index 000000000..f707671d5
--- /dev/null
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/package-info.java
@@ -0,0 +1,4 @@
+/**
+ * Markdown rendering (see {@link org.commonmark.renderer.markdown.MarkdownRenderer})
+ */
+package org.commonmark.renderer.markdown;
diff --git a/commonmark/src/main/java/org/commonmark/renderer/text/package-info.java b/commonmark/src/main/java/org/commonmark/renderer/text/package-info.java
index 07a558091..8309f4bd6 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/text/package-info.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/text/package-info.java
@@ -1,4 +1,4 @@
/**
- * Text content rendering (see {@link org.commonmark.renderer.text.TextContentRenderer})
+ * Plain text rendering with minimal markup (see {@link org.commonmark.renderer.text.TextContentRenderer})
*/
package org.commonmark.renderer.text;
diff --git a/commonmark/src/test/java/org/commonmark/test/UsageExampleTest.java b/commonmark/src/test/java/org/commonmark/test/UsageExampleTest.java
index 9ff646630..08235965a 100644
--- a/commonmark/src/test/java/org/commonmark/test/UsageExampleTest.java
+++ b/commonmark/src/test/java/org/commonmark/test/UsageExampleTest.java
@@ -4,6 +4,7 @@
import org.commonmark.parser.Parser;
import org.commonmark.renderer.NodeRenderer;
import org.commonmark.renderer.html.*;
+import org.commonmark.renderer.markdown.MarkdownRenderer;
import org.junit.Ignore;
import org.junit.Test;
@@ -22,9 +23,21 @@ public class UsageExampleTest {
@Test
public void parseAndRender() {
Parser parser = Parser.builder().build();
- Node document = parser.parse("This is *Sparta*");
+ Node document = parser.parse("This is *Markdown*");
HtmlRenderer renderer = HtmlRenderer.builder().escapeHtml(true).build();
- assertEquals("
This is Sparta
\n", renderer.render(document));
+ assertEquals("
This is Markdown
\n", renderer.render(document));
+ }
+
+ @Test
+ public void renderToMarkdown() {
+ MarkdownRenderer renderer = MarkdownRenderer.builder().build();
+ Node document = new Document();
+ Heading heading = new Heading();
+ heading.setLevel(2);
+ heading.appendChild(new Text("My title"));
+ document.appendChild(heading);
+
+ assertEquals("## My title\n", renderer.render(document));
}
@Test
From 94eb5ff58a9cf589d28945e79a8a702abcbf6344 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Sat, 24 Feb 2024 11:23:27 +1100
Subject: [PATCH 071/286] Add test for hard line break in heading
---
.../org/commonmark/renderer/markdown/MarkdownRendererTest.java | 1 +
1 file changed, 1 insertion(+)
diff --git a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
index eaabe837c..9bab92bcc 100644
--- a/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
+++ b/commonmark/src/test/java/org/commonmark/renderer/markdown/MarkdownRendererTest.java
@@ -30,6 +30,7 @@ public void testHeadings() {
assertRoundTrip("###### foo\n");
assertRoundTrip("Foo\nbar\n===\n");
+ assertRoundTrip("Foo \nbar\n===\n");
assertRoundTrip("[foo\nbar](/url)\n===\n");
assertRoundTrip("# foo\n\nbar\n");
From ce9e056352ee9d8c1a415a0b4354e34209c51ba6 Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Sat, 24 Feb 2024 14:36:34 +1100
Subject: [PATCH 072/286] Modular JAR: Add module-info module descriptor
We've waited long enough for the ecosystem and tools such as Android to
catch up. Making it proper modular means people can run jlink on it.
Fixes #125.
---
CHANGELOG.md | 9 +
README.md | 13 +-
commonmark-ext-autolink/pom.xml | 18 +-
.../src/main/java/module-info.java | 6 +
commonmark-ext-gfm-strikethrough/pom.xml | 16 --
.../src/main/java/module-info.java | 5 +
commonmark-ext-gfm-tables/pom.xml | 16 --
.../src/main/java/module-info.java | 5 +
.../gfm/tables/internal/TableBlockParser.java | 16 +-
.../internal/TableMarkdownNodeRenderer.java | 5 +-
commonmark-ext-heading-anchor/pom.xml | 16 --
.../src/main/java/module-info.java | 5 +
commonmark-ext-image-attributes/pom.xml | 16 --
.../src/main/java/module-info.java | 5 +
commonmark-ext-ins/pom.xml | 16 --
.../src/main/java/module-info.java | 5 +
commonmark-ext-task-list-items/pom.xml | 16 --
.../src/main/java/module-info.java | 5 +
commonmark-ext-yaml-front-matter/pom.xml | 16 --
.../src/main/java/module-info.java | 5 +
.../internal/YamlFrontMatterBlockParser.java | 4 +-
commonmark-test-util/pom.xml | 16 --
.../src/main/java/module-info.java | 6 +
commonmark/pom.xml | 16 --
commonmark/src/main/java/module-info.java | 13 +
.../commonmark/internal/BlockQuoteParser.java | 5 +-
.../java/org/commonmark/internal/Bracket.java | 2 +-
.../commonmark/internal/DocumentParser.java | 36 ++-
.../internal/FencedCodeBlockParser.java | 7 +-
.../commonmark/internal/HeadingParser.java | 9 +-
.../commonmark/internal/HtmlBlockParser.java | 18 +-
.../internal/IndentedCodeBlockParser.java | 3 +-
.../commonmark/internal/InlineParserImpl.java | 17 +-
.../LinkReferenceDefinitionParser.java | 4 +-
.../internal/inline/AutolinkInlineParser.java | 2 +
.../inline/BackslashInlineParser.java | 2 +-
.../inline/BackticksInlineParser.java | 6 +-
.../internal/inline/EntityInlineParser.java | 4 +-
.../internal/inline/HtmlInlineParser.java | 4 +-
.../internal/inline/InlineParserState.java | 3 +
.../internal/inline/ParsedInline.java | 1 +
.../internal/inline/ParsedInlineImpl.java | 1 +
.../commonmark/internal/util/CharMatcher.java | 6 -
.../commonmark/internal/util/LinkScanner.java | 49 +++-
.../org/commonmark/internal/util/Parsing.java | 236 ------------------
.../inline => parser/beta}/Position.java | 2 +-
.../inline => parser/beta}/Scanner.java | 4 +-
.../markdown/CoreMarkdownNodeRenderer.java | 12 +-
.../renderer/markdown/MarkdownWriter.java | 2 +-
.../{internal/util => text}/AsciiMatcher.java | 5 +-
.../java/org/commonmark/text/CharMatcher.java | 13 +
.../java/org/commonmark/text/Characters.java | 159 ++++++++++++
.../inline => parser/beta}/ScannerTest.java | 4 +-
.../CharactersTest.java} | 6 +-
pom.xml | 2 +-
55 files changed, 420 insertions(+), 473 deletions(-)
create mode 100644 commonmark-ext-autolink/src/main/java/module-info.java
create mode 100644 commonmark-ext-gfm-strikethrough/src/main/java/module-info.java
create mode 100644 commonmark-ext-gfm-tables/src/main/java/module-info.java
create mode 100644 commonmark-ext-heading-anchor/src/main/java/module-info.java
create mode 100644 commonmark-ext-image-attributes/src/main/java/module-info.java
create mode 100644 commonmark-ext-ins/src/main/java/module-info.java
create mode 100644 commonmark-ext-task-list-items/src/main/java/module-info.java
create mode 100644 commonmark-ext-yaml-front-matter/src/main/java/module-info.java
create mode 100644 commonmark-test-util/src/main/java/module-info.java
create mode 100644 commonmark/src/main/java/module-info.java
delete mode 100644 commonmark/src/main/java/org/commonmark/internal/util/CharMatcher.java
rename commonmark/src/main/java/org/commonmark/{internal/inline => parser/beta}/Position.java (89%)
rename commonmark/src/main/java/org/commonmark/{internal/inline => parser/beta}/Scanner.java (99%)
rename commonmark/src/main/java/org/commonmark/{internal/util => text}/AsciiMatcher.java (94%)
create mode 100644 commonmark/src/main/java/org/commonmark/text/CharMatcher.java
create mode 100644 commonmark/src/main/java/org/commonmark/text/Characters.java
rename commonmark/src/test/java/org/commonmark/{internal/inline => parser/beta}/ScannerTest.java (97%)
rename commonmark/src/test/java/org/commonmark/{internal/util/ParsingTest.java => text/CharactersTest.java} (78%)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f558a3601..157c166b1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/).
This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html),
with the exception that 0.x versions can break between minor versions.
+## Unreleased
+### Changed
+- Modular JAR: Require at least Java 11 and add a module descriptor (module-info),
+ remove no longer necessary `Automatic-Module-Name` header
+- New package `org.commonmark.parser.beta` containing classes that are not part of
+ the stable API but are exported from the module (because they might be useful for
+ extension parsers).
+
## [0.21.0] - 2022-11-17
### Added
- GitHub strikethrough: With the previous version we adjusted the
@@ -379,6 +387,7 @@ API breaking changes (caused by changes in spec):
Initial release of commonmark-java, a port of commonmark.js with extensions
for autolinking URLs, GitHub flavored strikethrough and tables.
+[0.22.0]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.21.0...commonmark-parent-0.22.0
[0.21.0]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.20.0...commonmark-parent-0.21.0
[0.20.0]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.19.0...commonmark-parent-0.20.0
[0.19.0]: https://github.com/commonmark/commonmark-java/compare/commonmark-parent-0.18.2...commonmark-parent-0.19.0
diff --git a/README.md b/README.md
index da7a07dd2..5cce587d9 100644
--- a/README.md
+++ b/README.md
@@ -24,9 +24,11 @@ extensible library with the following features:
* Flexible (manipulate the AST after parsing, customize HTML rendering)
* Extensible (tables, strikethrough, autolinking and more, see below)
-The library is supported on Java 11 and later. It should work on Android too,
-but that is on a best-effort basis, please report problems. For Android the
-minimum API level is 19, see the
+The library is supported on Java 11 and later. It works on Android too, but
+that is currently not part of the builds and so only supported on a best-effort
+basis, please report any problems. For Android the minimum API level is 19, see
+the [commonmark-android-test](commonmark-android-test)
+directory.
[commonmark-android-test](commonmark-android-test) directory.
Coordinates for core library (see all on [Maven Central]):
@@ -44,7 +46,8 @@ The module names to use in Java 9 are `org.commonmark`,
Note that for 0.x releases of this library, the API is not considered stable
yet and may break between minor releases. After 1.0, [Semantic Versioning] will
-be followed.
+be followed. A package containing `beta` means it's not subject to stable API
+guarantees yet; but for normal usage it should not be necessary to use.
See the [spec.txt](commonmark-test-util/src/main/resources/spec.txt)
file if you're wondering which version of the spec is currently
@@ -399,7 +402,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) file.
License
-------
-Copyright (c) 2015-2019 Atlassian and others.
+Copyright (c) Atlassian and others.
BSD (2-clause) licensed, see LICENSE.txt file.
diff --git a/commonmark-ext-autolink/pom.xml b/commonmark-ext-autolink/pom.xml
index c82ecdae8..2f70f7a0e 100644
--- a/commonmark-ext-autolink/pom.xml
+++ b/commonmark-ext-autolink/pom.xml
@@ -12,7 +12,7 @@
commonmark-java extension for turning plain URLs and email addresses into links
- 0.10.0
+ 0.11.0
@@ -33,20 +33,4 @@
-
-
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
-
- org.commonmark.ext.autolink
-
-
-
-
-
-
-
diff --git a/commonmark-ext-autolink/src/main/java/module-info.java b/commonmark-ext-autolink/src/main/java/module-info.java
new file mode 100644
index 000000000..6fe98c3b1
--- /dev/null
+++ b/commonmark-ext-autolink/src/main/java/module-info.java
@@ -0,0 +1,6 @@
+module org.commonmark.ext.autolink {
+ exports org.commonmark.ext.autolink;
+
+ requires org.commonmark;
+ requires org.nibor.autolink;
+}
diff --git a/commonmark-ext-gfm-strikethrough/pom.xml b/commonmark-ext-gfm-strikethrough/pom.xml
index 878a1e586..b452073ab 100644
--- a/commonmark-ext-gfm-strikethrough/pom.xml
+++ b/commonmark-ext-gfm-strikethrough/pom.xml
@@ -24,20 +24,4 @@
-
-
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
-
- org.commonmark.ext.gfm.strikethrough
-
-
-
-
-
-
-
diff --git a/commonmark-ext-gfm-strikethrough/src/main/java/module-info.java b/commonmark-ext-gfm-strikethrough/src/main/java/module-info.java
new file mode 100644
index 000000000..772710f00
--- /dev/null
+++ b/commonmark-ext-gfm-strikethrough/src/main/java/module-info.java
@@ -0,0 +1,5 @@
+module org.commonmark.ext.gfm.strikethrough {
+ exports org.commonmark.ext.gfm.strikethrough;
+
+ requires org.commonmark;
+}
diff --git a/commonmark-ext-gfm-tables/pom.xml b/commonmark-ext-gfm-tables/pom.xml
index c6448889f..01fe2b23c 100644
--- a/commonmark-ext-gfm-tables/pom.xml
+++ b/commonmark-ext-gfm-tables/pom.xml
@@ -24,20 +24,4 @@
-
-
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
-
- org.commonmark.ext.gfm.tables
-
-
-
-
-
-
-
diff --git a/commonmark-ext-gfm-tables/src/main/java/module-info.java b/commonmark-ext-gfm-tables/src/main/java/module-info.java
new file mode 100644
index 000000000..472c84c3d
--- /dev/null
+++ b/commonmark-ext-gfm-tables/src/main/java/module-info.java
@@ -0,0 +1,5 @@
+module org.commonmark.ext.gfm.tables {
+ exports org.commonmark.ext.gfm.tables;
+
+ requires org.commonmark;
+}
diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java
index b7cea14db..2518a3eee 100644
--- a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java
+++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableBlockParser.java
@@ -1,7 +1,6 @@
package org.commonmark.ext.gfm.tables.internal;
import org.commonmark.ext.gfm.tables.*;
-import org.commonmark.internal.util.Parsing;
import org.commonmark.node.Block;
import org.commonmark.node.Node;
import org.commonmark.node.SourceSpan;
@@ -9,6 +8,7 @@
import org.commonmark.parser.SourceLine;
import org.commonmark.parser.SourceLines;
import org.commonmark.parser.block.*;
+import org.commonmark.text.Characters;
import java.util.ArrayList;
import java.util.List;
@@ -39,11 +39,11 @@ public Block getBlock() {
@Override
public BlockContinue tryContinue(ParserState state) {
CharSequence content = state.getLine().getContent();
- int pipe = Parsing.find('|', content, state.getNextNonSpaceIndex());
+ int pipe = Characters.find('|', content, state.getNextNonSpaceIndex());
if (pipe != -1) {
if (pipe == state.getNextNonSpaceIndex()) {
// If we *only* have a pipe character (and whitespace), that is not a valid table row and ends the table.
- if (Parsing.skipSpaceTab(content, pipe + 1, content.length()) == content.length()) {
+ if (Characters.skipSpaceTab(content, pipe + 1, content.length()) == content.length()) {
// We also don't want the pipe to be added via lazy continuation.
canHaveLazyContinuationLines = false;
return BlockContinue.none();
@@ -124,8 +124,8 @@ private TableCell parseCell(SourceLine cell, int column, InlineParser inlinePars
}
CharSequence content = cell.getContent();
- int start = Parsing.skipSpaceTab(content, 0, content.length());
- int end = Parsing.skipSpaceTabBackwards(content, content.length() - 1, start);
+ int start = Characters.skipSpaceTab(content, 0, content.length());
+ int end = Characters.skipSpaceTabBackwards(content, content.length() - 1, start);
inlineParser.parse(SourceLines.of(cell.substring(start, end + 1)), tableCell);
return tableCell;
@@ -133,14 +133,14 @@ private TableCell parseCell(SourceLine cell, int column, InlineParser inlinePars
private static List split(SourceLine line) {
CharSequence row = line.getContent();
- int nonSpace = Parsing.skipSpaceTab(row, 0, row.length());
+ int nonSpace = Characters.skipSpaceTab(row, 0, row.length());
int cellStart = nonSpace;
int cellEnd = row.length();
if (row.charAt(nonSpace) == '|') {
// This row has leading/trailing pipes - skip the leading pipe
cellStart = nonSpace + 1;
// Strip whitespace from the end but not the pipe or we could miss an empty ("||") cell
- int nonSpaceEnd = Parsing.skipSpaceTabBackwards(row, row.length() - 1, cellStart);
+ int nonSpaceEnd = Characters.skipSpaceTabBackwards(row, row.length() - 1, cellStart);
cellEnd = nonSpaceEnd + 1;
}
List cells = new ArrayList<>();
@@ -267,7 +267,7 @@ public static class Factory extends AbstractBlockParserFactory {
@Override
public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockParser) {
List paragraphLines = matchedBlockParser.getParagraphLines().getLines();
- if (paragraphLines.size() == 1 && Parsing.find('|', paragraphLines.get(0).getContent(), 0) != -1) {
+ if (paragraphLines.size() == 1 && Characters.find('|', paragraphLines.get(0).getContent(), 0) != -1) {
SourceLine line = state.getLine();
SourceLine separatorLine = line.substring(state.getIndex(), line.getContent().length());
List columns = parseSeparator(separatorLine.getContent());
diff --git a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableMarkdownNodeRenderer.java b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableMarkdownNodeRenderer.java
index 487147164..2fe60f80d 100644
--- a/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableMarkdownNodeRenderer.java
+++ b/commonmark-ext-gfm-tables/src/main/java/org/commonmark/ext/gfm/tables/internal/TableMarkdownNodeRenderer.java
@@ -1,12 +1,11 @@
package org.commonmark.ext.gfm.tables.internal;
import org.commonmark.ext.gfm.tables.*;
-import org.commonmark.internal.util.AsciiMatcher;
-import org.commonmark.internal.util.CharMatcher;
import org.commonmark.node.Node;
import org.commonmark.renderer.NodeRenderer;
import org.commonmark.renderer.markdown.MarkdownNodeRendererContext;
import org.commonmark.renderer.markdown.MarkdownWriter;
+import org.commonmark.text.AsciiMatcher;
import java.util.ArrayList;
import java.util.List;
@@ -18,7 +17,7 @@ public class TableMarkdownNodeRenderer extends TableNodeRenderer implements Node
private final MarkdownWriter writer;
private final MarkdownNodeRendererContext context;
- private final CharMatcher pipe = AsciiMatcher.builder().c('|').build();
+ private final AsciiMatcher pipe = AsciiMatcher.builder().c('|').build();
private final List columns = new ArrayList<>();
diff --git a/commonmark-ext-heading-anchor/pom.xml b/commonmark-ext-heading-anchor/pom.xml
index 49bc4a032..330490ea6 100644
--- a/commonmark-ext-heading-anchor/pom.xml
+++ b/commonmark-ext-heading-anchor/pom.xml
@@ -24,20 +24,4 @@
-
-
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
-
- org.commonmark.ext.heading.anchor
-
-
-
-
-
-
-
diff --git a/commonmark-ext-heading-anchor/src/main/java/module-info.java b/commonmark-ext-heading-anchor/src/main/java/module-info.java
new file mode 100644
index 000000000..3b94c75ec
--- /dev/null
+++ b/commonmark-ext-heading-anchor/src/main/java/module-info.java
@@ -0,0 +1,5 @@
+module org.commonmark.ext.heading.anchor {
+ exports org.commonmark.ext.heading.anchor;
+
+ requires org.commonmark;
+}
diff --git a/commonmark-ext-image-attributes/pom.xml b/commonmark-ext-image-attributes/pom.xml
index 959b1406c..6fbec9007 100644
--- a/commonmark-ext-image-attributes/pom.xml
+++ b/commonmark-ext-image-attributes/pom.xml
@@ -24,20 +24,4 @@
-
-
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
-
- org.commonmark.ext.image.attributes
-
-
-
-
-
-
-
diff --git a/commonmark-ext-image-attributes/src/main/java/module-info.java b/commonmark-ext-image-attributes/src/main/java/module-info.java
new file mode 100644
index 000000000..171281091
--- /dev/null
+++ b/commonmark-ext-image-attributes/src/main/java/module-info.java
@@ -0,0 +1,5 @@
+module org.commonmark.ext.image.attributes {
+ exports org.commonmark.ext.image.attributes;
+
+ requires org.commonmark;
+}
diff --git a/commonmark-ext-ins/pom.xml b/commonmark-ext-ins/pom.xml
index 708532472..68e5f627b 100644
--- a/commonmark-ext-ins/pom.xml
+++ b/commonmark-ext-ins/pom.xml
@@ -24,20 +24,4 @@
-
-
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
-
- org.commonmark.ext.ins
-
-
-
-
-
-
-
diff --git a/commonmark-ext-ins/src/main/java/module-info.java b/commonmark-ext-ins/src/main/java/module-info.java
new file mode 100644
index 000000000..aa5c5e84c
--- /dev/null
+++ b/commonmark-ext-ins/src/main/java/module-info.java
@@ -0,0 +1,5 @@
+module org.commonmark.ext.ins {
+ exports org.commonmark.ext.ins;
+
+ requires org.commonmark;
+}
diff --git a/commonmark-ext-task-list-items/pom.xml b/commonmark-ext-task-list-items/pom.xml
index ec3ac1a1f..3e3bebf32 100644
--- a/commonmark-ext-task-list-items/pom.xml
+++ b/commonmark-ext-task-list-items/pom.xml
@@ -24,20 +24,4 @@
-
-
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
-
- org.commonmark.ext.task.list.items
-
-
-
-
-
-
-
diff --git a/commonmark-ext-task-list-items/src/main/java/module-info.java b/commonmark-ext-task-list-items/src/main/java/module-info.java
new file mode 100644
index 000000000..30134c51b
--- /dev/null
+++ b/commonmark-ext-task-list-items/src/main/java/module-info.java
@@ -0,0 +1,5 @@
+module org.commonmark.ext.task.list.items {
+ exports org.commonmark.ext.task.list.items;
+
+ requires org.commonmark;
+}
diff --git a/commonmark-ext-yaml-front-matter/pom.xml b/commonmark-ext-yaml-front-matter/pom.xml
index e5e120caa..97bdc5c48 100644
--- a/commonmark-ext-yaml-front-matter/pom.xml
+++ b/commonmark-ext-yaml-front-matter/pom.xml
@@ -24,20 +24,4 @@
-
-
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
-
- org.commonmark.ext.front.matter
-
-
-
-
-
-
-
diff --git a/commonmark-ext-yaml-front-matter/src/main/java/module-info.java b/commonmark-ext-yaml-front-matter/src/main/java/module-info.java
new file mode 100644
index 000000000..20d38fe0a
--- /dev/null
+++ b/commonmark-ext-yaml-front-matter/src/main/java/module-info.java
@@ -0,0 +1,5 @@
+module org.commonmark.ext.front.matter {
+ exports org.commonmark.ext.front.matter;
+
+ requires org.commonmark;
+}
diff --git a/commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/internal/YamlFrontMatterBlockParser.java b/commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/internal/YamlFrontMatterBlockParser.java
index 2010b4f71..469cf4e2f 100644
--- a/commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/internal/YamlFrontMatterBlockParser.java
+++ b/commonmark-ext-yaml-front-matter/src/main/java/org/commonmark/ext/front/matter/internal/YamlFrontMatterBlockParser.java
@@ -2,8 +2,8 @@
import org.commonmark.ext.front.matter.YamlFrontMatterBlock;
import org.commonmark.ext.front.matter.YamlFrontMatterNode;
-import org.commonmark.internal.DocumentBlockParser;
import org.commonmark.node.Block;
+import org.commonmark.node.Document;
import org.commonmark.parser.InlineParser;
import org.commonmark.parser.SourceLine;
import org.commonmark.parser.block.*;
@@ -119,7 +119,7 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar
CharSequence line = state.getLine().getContent();
BlockParser parentParser = matchedBlockParser.getMatchedBlockParser();
// check whether this line is the first line of whole document or not
- if (parentParser instanceof DocumentBlockParser && parentParser.getBlock().getFirstChild() == null &&
+ if (parentParser.getBlock() instanceof Document && parentParser.getBlock().getFirstChild() == null &&
REGEX_BEGIN.matcher(line).matches()) {
return BlockStart.of(new YamlFrontMatterBlockParser()).atIndex(state.getNextNonSpaceIndex());
}
diff --git a/commonmark-test-util/pom.xml b/commonmark-test-util/pom.xml
index c61000412..ce332e6c2 100644
--- a/commonmark-test-util/pom.xml
+++ b/commonmark-test-util/pom.xml
@@ -18,20 +18,4 @@
-
-
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
-
- org.commonmark.testutil
-
-
-
-
-
-
-
diff --git a/commonmark-test-util/src/main/java/module-info.java b/commonmark-test-util/src/main/java/module-info.java
new file mode 100644
index 000000000..ef983a513
--- /dev/null
+++ b/commonmark-test-util/src/main/java/module-info.java
@@ -0,0 +1,6 @@
+module org.commonmark.testutil {
+ exports org.commonmark.testutil;
+ exports org.commonmark.testutil.example;
+
+ requires junit;
+}
diff --git a/commonmark/pom.xml b/commonmark/pom.xml
index 9988370a8..524ba9c42 100644
--- a/commonmark/pom.xml
+++ b/commonmark/pom.xml
@@ -29,22 +29,6 @@
-
-
-
- org.apache.maven.plugins
- maven-jar-plugin
-
-
-
- org.commonmark
-
-
-
-
-
-
-
benchmark
diff --git a/commonmark/src/main/java/module-info.java b/commonmark/src/main/java/module-info.java
new file mode 100644
index 000000000..009fc7d18
--- /dev/null
+++ b/commonmark/src/main/java/module-info.java
@@ -0,0 +1,13 @@
+module org.commonmark {
+ exports org.commonmark;
+ exports org.commonmark.node;
+ exports org.commonmark.parser;
+ exports org.commonmark.parser.beta;
+ exports org.commonmark.parser.block;
+ exports org.commonmark.parser.delimiter;
+ exports org.commonmark.renderer;
+ exports org.commonmark.renderer.html;
+ exports org.commonmark.renderer.markdown;
+ exports org.commonmark.renderer.text;
+ exports org.commonmark.text;
+}
diff --git a/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java b/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java
index 00cdbc11e..87c923e06 100644
--- a/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/BlockQuoteParser.java
@@ -4,6 +4,7 @@
import org.commonmark.node.Block;
import org.commonmark.node.BlockQuote;
import org.commonmark.parser.block.*;
+import org.commonmark.text.Characters;
public class BlockQuoteParser extends AbstractBlockParser {
@@ -30,7 +31,7 @@ public BlockContinue tryContinue(ParserState state) {
if (isMarker(state, nextNonSpace)) {
int newColumn = state.getColumn() + state.getIndent() + 1;
// optional following space or tab
- if (Parsing.isSpaceOrTab(state.getLine().getContent(), nextNonSpace + 1)) {
+ if (Characters.isSpaceOrTab(state.getLine().getContent(), nextNonSpace + 1)) {
newColumn++;
}
return BlockContinue.atColumn(newColumn);
@@ -50,7 +51,7 @@ public BlockStart tryStart(ParserState state, MatchedBlockParser matchedBlockPar
if (isMarker(state, nextNonSpace)) {
int newColumn = state.getColumn() + state.getIndent() + 1;
// optional following space or tab
- if (Parsing.isSpaceOrTab(state.getLine().getContent(), nextNonSpace + 1)) {
+ if (Characters.isSpaceOrTab(state.getLine().getContent(), nextNonSpace + 1)) {
newColumn++;
}
return BlockStart.of(new BlockQuoteParser()).atColumn(newColumn);
diff --git a/commonmark/src/main/java/org/commonmark/internal/Bracket.java b/commonmark/src/main/java/org/commonmark/internal/Bracket.java
index 46296262f..9c73a1327 100644
--- a/commonmark/src/main/java/org/commonmark/internal/Bracket.java
+++ b/commonmark/src/main/java/org/commonmark/internal/Bracket.java
@@ -1,7 +1,7 @@
package org.commonmark.internal;
-import org.commonmark.internal.inline.Position;
import org.commonmark.node.Text;
+import org.commonmark.parser.beta.Position;
/**
* Opening bracket for links ([) or images (![).
diff --git a/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java b/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java
index 086c3dbc0..2cc37e306 100644
--- a/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/DocumentParser.java
@@ -5,6 +5,7 @@
import org.commonmark.parser.*;
import org.commonmark.parser.block.*;
import org.commonmark.parser.delimiter.DelimiterProcessor;
+import org.commonmark.text.Characters;
import java.io.BufferedReader;
import java.io.IOException;
@@ -112,7 +113,7 @@ public static void checkEnabledBlockTypes(Set> enabledBlo
public Document parse(String input) {
int lineStart = 0;
int lineBreak;
- while ((lineBreak = Parsing.findLineBreak(input, lineStart)) != -1) {
+ while ((lineBreak = Characters.findLineBreak(input, lineStart)) != -1) {
String line = input.substring(lineStart, lineBreak);
parseLine(line);
if (lineBreak + 1 < input.length() && input.charAt(lineBreak) == '\r' && input.charAt(lineBreak + 1) == '\n') {
@@ -230,7 +231,7 @@ private void parseLine(CharSequence ln) {
findNextNonSpace();
// this is a little performance optimization:
- if (isBlank() || (indent < Parsing.CODE_BLOCK_INDENT && Parsing.isLetter(this.line.getContent(), nextNonSpace))) {
+ if (isBlank() || (indent < Parsing.CODE_BLOCK_INDENT && Characters.isLetter(this.line.getContent(), nextNonSpace))) {
setNewIndex(nextNonSpace);
break;
}
@@ -315,7 +316,7 @@ private void setLine(CharSequence ln) {
column = 0;
columnIsInTab = false;
- CharSequence lineContent = Parsing.prepareLine(ln);
+ CharSequence lineContent = prepareLine(ln);
SourceSpan sourceSpan = null;
if (includeSourceSpans != IncludeSourceSpans.NONE) {
sourceSpan = SourceSpan.of(lineIndex, 0, lineContent.length());
@@ -542,6 +543,35 @@ private void closeBlockParsers(int count) {
}
}
+ /**
+ * Prepares the input line replacing {@code \0}
+ */
+ private static CharSequence prepareLine(CharSequence line) {
+ // Avoid building a new string in the majority of cases (no \0)
+ StringBuilder sb = null;
+ int length = line.length();
+ for (int i = 0; i < length; i++) {
+ char c = line.charAt(i);
+ if (c == '\0') {
+ if (sb == null) {
+ sb = new StringBuilder(length);
+ sb.append(line, 0, i);
+ }
+ sb.append('\uFFFD');
+ } else {
+ if (sb != null) {
+ sb.append(c);
+ }
+ }
+ }
+
+ if (sb != null) {
+ return sb.toString();
+ } else {
+ return line;
+ }
+ }
+
private static class MatchedBlockParserImpl implements MatchedBlockParser {
private final BlockParser matchedBlockParser;
diff --git a/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java
index 2d7d2c0c9..a16758dd4 100644
--- a/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/FencedCodeBlockParser.java
@@ -5,6 +5,7 @@
import org.commonmark.node.FencedCodeBlock;
import org.commonmark.parser.SourceLine;
import org.commonmark.parser.block.*;
+import org.commonmark.text.Characters;
import static org.commonmark.internal.util.Escaping.unescapeString;
@@ -103,7 +104,7 @@ private static FencedCodeBlockParser checkOpener(CharSequence line, int index, i
}
if (backticks >= 3 && tildes == 0) {
// spec: If the info string comes after a backtick fence, it may not contain any backtick characters.
- if (Parsing.find('`', line, index + backticks) != -1) {
+ if (Characters.find('`', line, index + backticks) != -1) {
return null;
}
return new FencedCodeBlockParser('`', backticks, indent);
@@ -121,12 +122,12 @@ private static FencedCodeBlockParser checkOpener(CharSequence line, int index, i
private boolean isClosing(CharSequence line, int index) {
char fenceChar = block.getFenceChar();
int fenceLength = block.getFenceLength();
- int fences = Parsing.skip(fenceChar, line, index, line.length()) - index;
+ int fences = Characters.skip(fenceChar, line, index, line.length()) - index;
if (fences < fenceLength) {
return false;
}
// spec: The closing code fence [...] may be followed only by spaces, which are ignored.
- int after = Parsing.skipSpaceTab(line, index + fences, line.length());
+ int after = Characters.skipSpaceTab(line, index + fences, line.length());
return after == line.length();
}
}
diff --git a/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java b/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java
index 81c60d0d1..d422c1241 100644
--- a/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/HeadingParser.java
@@ -1,14 +1,15 @@
package org.commonmark.internal;
-import org.commonmark.internal.inline.Position;
-import org.commonmark.internal.inline.Scanner;
import org.commonmark.internal.util.Parsing;
import org.commonmark.node.Block;
import org.commonmark.node.Heading;
import org.commonmark.parser.InlineParser;
import org.commonmark.parser.SourceLine;
import org.commonmark.parser.SourceLines;
+import org.commonmark.parser.beta.Position;
+import org.commonmark.parser.beta.Scanner;
import org.commonmark.parser.block.*;
+import org.commonmark.text.Characters;
public class HeadingParser extends AbstractBlockParser {
@@ -148,8 +149,8 @@ private static int getSetextHeadingLevel(CharSequence line, int index) {
}
private static boolean isSetextHeadingRest(CharSequence line, int index, char marker) {
- int afterMarker = Parsing.skip(marker, line, index, line.length());
- int afterSpace = Parsing.skipSpaceTab(line, afterMarker, line.length());
+ int afterMarker = Characters.skip(marker, line, index, line.length());
+ int afterSpace = Characters.skipSpaceTab(line, afterMarker, line.length());
return afterSpace >= line.length();
}
}
diff --git a/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java
index 0e2050567..ce66c20da 100644
--- a/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/HtmlBlockParser.java
@@ -1,6 +1,5 @@
package org.commonmark.internal;
-import org.commonmark.internal.util.Parsing;
import org.commonmark.node.Block;
import org.commonmark.node.HtmlBlock;
import org.commonmark.node.Paragraph;
@@ -11,6 +10,21 @@
public class HtmlBlockParser extends AbstractBlockParser {
+ private static final String TAGNAME = "[A-Za-z][A-Za-z0-9-]*";
+ private static final String ATTRIBUTENAME = "[a-zA-Z_:][a-zA-Z0-9:._-]*";
+ private static final String UNQUOTEDVALUE = "[^\"'=<>`\\x00-\\x20]+";
+ private static final String SINGLEQUOTEDVALUE = "'[^']*'";
+ private static final String DOUBLEQUOTEDVALUE = "\"[^\"]*\"";
+ private static final String ATTRIBUTEVALUE = "(?:" + UNQUOTEDVALUE + "|" + SINGLEQUOTEDVALUE
+ + "|" + DOUBLEQUOTEDVALUE + ")";
+ private static final String ATTRIBUTEVALUESPEC = "(?:" + "\\s*=" + "\\s*" + ATTRIBUTEVALUE
+ + ")";
+ private static final String ATTRIBUTE = "(?:" + "\\s+" + ATTRIBUTENAME + ATTRIBUTEVALUESPEC
+ + "?)";
+
+ private static final String OPENTAG = "<" + TAGNAME + ATTRIBUTE + "*" + "\\s*/?>";
+ private static final String CLOSETAG = "" + TAGNAME + "\\s*[>]";
+
private static final Pattern[][] BLOCK_PATTERNS = new Pattern[][]{
{null, null}, // not used (no type 0)
{
@@ -54,7 +68,7 @@ public class HtmlBlockParser extends AbstractBlockParser {
null // terminated by blank line
},
{
- Pattern.compile("^(?:" + Parsing.OPENTAG + '|' + Parsing.CLOSETAG + ")\\s*$", Pattern.CASE_INSENSITIVE),
+ Pattern.compile("^(?:" + OPENTAG + '|' + CLOSETAG + ")\\s*$", Pattern.CASE_INSENSITIVE),
null // terminated by blank line
}
};
diff --git a/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java b/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java
index af74a587c..3598f5615 100644
--- a/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/IndentedCodeBlockParser.java
@@ -6,6 +6,7 @@
import org.commonmark.node.Paragraph;
import org.commonmark.parser.SourceLine;
import org.commonmark.parser.block.*;
+import org.commonmark.text.Characters;
import java.util.ArrayList;
import java.util.List;
@@ -40,7 +41,7 @@ public void addLine(SourceLine line) {
public void closeBlock() {
int lastNonBlank = lines.size() - 1;
while (lastNonBlank >= 0) {
- if (!Parsing.isBlank(lines.get(lastNonBlank))) {
+ if (!Characters.isBlank(lines.get(lastNonBlank))) {
break;
}
lastNonBlank--;
diff --git a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java
index c14b9e885..113e80db9 100644
--- a/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java
+++ b/commonmark/src/main/java/org/commonmark/internal/InlineParserImpl.java
@@ -1,15 +1,16 @@
package org.commonmark.internal;
-import org.commonmark.internal.inline.Scanner;
import org.commonmark.internal.inline.*;
import org.commonmark.internal.util.Escaping;
import org.commonmark.internal.util.LinkScanner;
-import org.commonmark.internal.util.Parsing;
import org.commonmark.node.*;
import org.commonmark.parser.InlineParser;
import org.commonmark.parser.InlineParserContext;
import org.commonmark.parser.SourceLines;
+import org.commonmark.parser.beta.Position;
+import org.commonmark.parser.beta.Scanner;
import org.commonmark.parser.delimiter.DelimiterProcessor;
+import org.commonmark.text.Characters;
import java.util.*;
@@ -482,12 +483,12 @@ private Node parseText() {
if (c == '\n') {
// We parsed until the end of the line. Trim any trailing spaces and remember them (for hard line breaks).
- int end = Parsing.skipBackwards(' ', content, content.length() - 1, 0) + 1;
+ int end = Characters.skipBackwards(' ', content, content.length() - 1, 0) + 1;
trailingSpaces = content.length() - end;
content = content.substring(0, end);
} else if (c == Scanner.END) {
// For the last line, both tabs and spaces are trimmed for some reason (checked with commonmark.js).
- int end = Parsing.skipSpaceTabBackwards(content, content.length() - 1, 0) + 1;
+ int end = Characters.skipSpaceTabBackwards(content, content.length() - 1, 0) + 1;
content = content.substring(0, end);
}
@@ -525,10 +526,10 @@ private DelimiterData scanDelimiters(DelimiterProcessor delimiterProcessor, char
int after = scanner.peekCodePoint();
// We could be more lazy here, in most cases we don't need to do every match case.
- boolean beforeIsPunctuation = before == Scanner.END || Parsing.isPunctuationCodePoint(before);
- boolean beforeIsWhitespace = before == Scanner.END || Parsing.isWhitespaceCodePoint(before);
- boolean afterIsPunctuation = after == Scanner.END || Parsing.isPunctuationCodePoint(after);
- boolean afterIsWhitespace = after == Scanner.END || Parsing.isWhitespaceCodePoint(after);
+ boolean beforeIsPunctuation = before == Scanner.END || Characters.isPunctuationCodePoint(before);
+ boolean beforeIsWhitespace = before == Scanner.END || Characters.isWhitespaceCodePoint(before);
+ boolean afterIsPunctuation = after == Scanner.END || Characters.isPunctuationCodePoint(after);
+ boolean afterIsWhitespace = after == Scanner.END || Characters.isWhitespaceCodePoint(after);
boolean leftFlanking = !afterIsWhitespace &&
(!afterIsPunctuation || beforeIsWhitespace || beforeIsPunctuation);
diff --git a/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java b/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java
index 2bceb7549..d11a6f228 100644
--- a/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/LinkReferenceDefinitionParser.java
@@ -1,13 +1,13 @@
package org.commonmark.internal;
-import org.commonmark.internal.inline.Position;
-import org.commonmark.internal.inline.Scanner;
import org.commonmark.internal.util.Escaping;
import org.commonmark.internal.util.LinkScanner;
import org.commonmark.node.LinkReferenceDefinition;
import org.commonmark.node.SourceSpan;
import org.commonmark.parser.SourceLine;
import org.commonmark.parser.SourceLines;
+import org.commonmark.parser.beta.Position;
+import org.commonmark.parser.beta.Scanner;
import java.util.ArrayList;
import java.util.List;
diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java
index ecfd2d972..36c43e196 100644
--- a/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/inline/AutolinkInlineParser.java
@@ -3,6 +3,8 @@
import org.commonmark.node.Link;
import org.commonmark.node.Text;
import org.commonmark.parser.SourceLines;
+import org.commonmark.parser.beta.Position;
+import org.commonmark.parser.beta.Scanner;
import java.util.regex.Pattern;
diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java
index f57a67a74..02c136951 100644
--- a/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/inline/BackslashInlineParser.java
@@ -2,8 +2,8 @@
import org.commonmark.internal.util.Escaping;
import org.commonmark.node.HardLineBreak;
-import org.commonmark.node.Node;
import org.commonmark.node.Text;
+import org.commonmark.parser.beta.Scanner;
import java.util.regex.Pattern;
diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java
index ad079444a..bef8e1f99 100644
--- a/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/inline/BackticksInlineParser.java
@@ -1,9 +1,11 @@
package org.commonmark.internal.inline;
-import org.commonmark.internal.util.Parsing;
import org.commonmark.node.Code;
import org.commonmark.node.Text;
import org.commonmark.parser.SourceLines;
+import org.commonmark.parser.beta.Position;
+import org.commonmark.parser.beta.Scanner;
+import org.commonmark.text.Characters;
/**
* Attempt to parse backticks, returning either a backtick code span or a literal sequence of backticks.
@@ -31,7 +33,7 @@ public ParsedInline tryParse(InlineParserState inlineParserState) {
if (content.length() >= 3 &&
content.charAt(0) == ' ' &&
content.charAt(content.length() - 1) == ' ' &&
- Parsing.hasNonSpace(content)) {
+ Characters.hasNonSpace(content)) {
content = content.substring(1, content.length() - 1);
}
diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java
index c29b8694f..2b7d296fb 100644
--- a/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/inline/EntityInlineParser.java
@@ -1,8 +1,10 @@
package org.commonmark.internal.inline;
-import org.commonmark.internal.util.AsciiMatcher;
+import org.commonmark.text.AsciiMatcher;
import org.commonmark.internal.util.Html5Entities;
import org.commonmark.node.Text;
+import org.commonmark.parser.beta.Position;
+import org.commonmark.parser.beta.Scanner;
/**
* Attempts to parse a HTML entity or numeric character reference.
diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java b/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java
index 605901c22..c85ae9d71 100644
--- a/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java
+++ b/commonmark/src/main/java/org/commonmark/internal/inline/HtmlInlineParser.java
@@ -1,7 +1,9 @@
package org.commonmark.internal.inline;
-import org.commonmark.internal.util.AsciiMatcher;
+import org.commonmark.text.AsciiMatcher;
import org.commonmark.node.HtmlInline;
+import org.commonmark.parser.beta.Position;
+import org.commonmark.parser.beta.Scanner;
/**
* Attempt to parse inline HTML.
diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/InlineParserState.java b/commonmark/src/main/java/org/commonmark/internal/inline/InlineParserState.java
index f6cb6bf49..ea8689be5 100644
--- a/commonmark/src/main/java/org/commonmark/internal/inline/InlineParserState.java
+++ b/commonmark/src/main/java/org/commonmark/internal/inline/InlineParserState.java
@@ -1,5 +1,8 @@
package org.commonmark.internal.inline;
+import org.commonmark.parser.beta.Position;
+import org.commonmark.parser.beta.Scanner;
+
public interface InlineParserState {
/**
diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInline.java b/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInline.java
index 7e6ece88e..7223c1687 100644
--- a/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInline.java
+++ b/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInline.java
@@ -1,6 +1,7 @@
package org.commonmark.internal.inline;
import org.commonmark.node.Node;
+import org.commonmark.parser.beta.Position;
public abstract class ParsedInline {
diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInlineImpl.java b/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInlineImpl.java
index aea325f27..55f9cc4da 100644
--- a/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInlineImpl.java
+++ b/commonmark/src/main/java/org/commonmark/internal/inline/ParsedInlineImpl.java
@@ -1,6 +1,7 @@
package org.commonmark.internal.inline;
import org.commonmark.node.Node;
+import org.commonmark.parser.beta.Position;
public class ParsedInlineImpl extends ParsedInline {
private final Node node;
diff --git a/commonmark/src/main/java/org/commonmark/internal/util/CharMatcher.java b/commonmark/src/main/java/org/commonmark/internal/util/CharMatcher.java
deleted file mode 100644
index de730e90d..000000000
--- a/commonmark/src/main/java/org/commonmark/internal/util/CharMatcher.java
+++ /dev/null
@@ -1,6 +0,0 @@
-package org.commonmark.internal.util;
-
-public interface CharMatcher {
-
- boolean matches(char c);
-}
diff --git a/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java b/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java
index 3ca34c5f0..ffed047e5 100644
--- a/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java
+++ b/commonmark/src/main/java/org/commonmark/internal/util/LinkScanner.java
@@ -1,6 +1,6 @@
package org.commonmark.internal.util;
-import org.commonmark.internal.inline.Scanner;
+import org.commonmark.parser.beta.Scanner;
public class LinkScanner {
@@ -14,7 +14,7 @@ public static boolean scanLinkLabelContent(Scanner scanner) {
switch (scanner.peek()) {
case '\\':
scanner.next();
- if (Parsing.isEscapable(scanner.peek())) {
+ if (isEscapable(scanner.peek())) {
scanner.next();
}
break;
@@ -44,7 +44,7 @@ public static boolean scanLinkDestination(Scanner scanner) {
switch (scanner.peek()) {
case '\\':
scanner.next();
- if (Parsing.isEscapable(scanner.peek())) {
+ if (isEscapable(scanner.peek())) {
scanner.next();
}
break;
@@ -100,7 +100,7 @@ public static boolean scanLinkTitleContent(Scanner scanner, char endDelimiter) {
char c = scanner.peek();
if (c == '\\') {
scanner.next();
- if (Parsing.isEscapable(scanner.peek())) {
+ if (isEscapable(scanner.peek())) {
scanner.next();
}
} else if (c == endDelimiter) {
@@ -128,7 +128,7 @@ private static boolean scanLinkDestinationWithBalancedParens(Scanner scanner) {
return !empty;
case '\\':
scanner.next();
- if (Parsing.isEscapable(scanner.peek())) {
+ if (isEscapable(scanner.peek())) {
scanner.next();
}
break;
@@ -160,4 +160,43 @@ private static boolean scanLinkDestinationWithBalancedParens(Scanner scanner) {
}
return true;
}
+
+ private static boolean isEscapable(char c) {
+ switch (c) {
+ case '!':
+ case '"':
+ case '#':
+ case '$':
+ case '%':
+ case '&':
+ case '\'':
+ case '(':
+ case ')':
+ case '*':
+ case '+':
+ case ',':
+ case '-':
+ case '.':
+ case '/':
+ case ':':
+ case ';':
+ case '<':
+ case '=':
+ case '>':
+ case '?':
+ case '@':
+ case '[':
+ case '\\':
+ case ']':
+ case '^':
+ case '_':
+ case '`':
+ case '{':
+ case '|':
+ case '}':
+ case '~':
+ return true;
+ }
+ return false;
+ }
}
diff --git a/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java b/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java
index 8b02e99b1..972fdef62 100644
--- a/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java
+++ b/commonmark/src/main/java/org/commonmark/internal/util/Parsing.java
@@ -1,246 +1,10 @@
package org.commonmark.internal.util;
public class Parsing {
-
- private static final String TAGNAME = "[A-Za-z][A-Za-z0-9-]*";
- private static final String ATTRIBUTENAME = "[a-zA-Z_:][a-zA-Z0-9:._-]*";
- private static final String UNQUOTEDVALUE = "[^\"'=<>`\\x00-\\x20]+";
- private static final String SINGLEQUOTEDVALUE = "'[^']*'";
- private static final String DOUBLEQUOTEDVALUE = "\"[^\"]*\"";
- private static final String ATTRIBUTEVALUE = "(?:" + UNQUOTEDVALUE + "|" + SINGLEQUOTEDVALUE
- + "|" + DOUBLEQUOTEDVALUE + ")";
- private static final String ATTRIBUTEVALUESPEC = "(?:" + "\\s*=" + "\\s*" + ATTRIBUTEVALUE
- + ")";
- private static final String ATTRIBUTE = "(?:" + "\\s+" + ATTRIBUTENAME + ATTRIBUTEVALUESPEC
- + "?)";
-
- public static final String OPENTAG = "<" + TAGNAME + ATTRIBUTE + "*" + "\\s*/?>";
- public static final String CLOSETAG = "" + TAGNAME + "\\s*[>]";
-
public static int CODE_BLOCK_INDENT = 4;
public static int columnsToNextTabStop(int column) {
// Tab stop is 4
return 4 - (column % 4);
}
-
- public static int find(char c, CharSequence s, int startIndex) {
- int length = s.length();
- for (int i = startIndex; i < length; i++) {
- if (s.charAt(i) == c) {
- return i;
- }
- }
- return -1;
- }
-
- public static int findLineBreak(CharSequence s, int startIndex) {
- int length = s.length();
- for (int i = startIndex; i < length; i++) {
- switch (s.charAt(i)) {
- case '\n':
- case '\r':
- return i;
- }
- }
- return -1;
- }
-
- public static boolean isBlank(CharSequence s) {
- return findNonSpace(s, 0) == -1;
- }
-
- public static boolean hasNonSpace(CharSequence s) {
- int length = s.length();
- int skipped = skip(' ', s, 0, length);
- return skipped != length;
- }
-
- public static boolean isLetter(CharSequence s, int index) {
- int codePoint = Character.codePointAt(s, index);
- return Character.isLetter(codePoint);
- }
-
- public static boolean isSpaceOrTab(CharSequence s, int index) {
- if (index < s.length()) {
- switch (s.charAt(index)) {
- case ' ':
- case '\t':
- return true;
- }
- }
- return false;
- }
-
- public static boolean isEscapable(char c) {
- switch (c) {
- case '!':
- case '"':
- case '#':
- case '$':
- case '%':
- case '&':
- case '\'':
- case '(':
- case ')':
- case '*':
- case '+':
- case ',':
- case '-':
- case '.':
- case '/':
- case ':':
- case ';':
- case '<':
- case '=':
- case '>':
- case '?':
- case '@':
- case '[':
- case '\\':
- case ']':
- case '^':
- case '_':
- case '`':
- case '{':
- case '|':
- case '}':
- case '~':
- return true;
- }
- return false;
- }
-
- // See https://spec.commonmark.org/0.29/#punctuation-character
- public static boolean isPunctuationCodePoint(int codePoint) {
- switch (Character.getType(codePoint)) {
- case Character.CONNECTOR_PUNCTUATION:
- case Character.DASH_PUNCTUATION:
- case Character.END_PUNCTUATION:
- case Character.FINAL_QUOTE_PUNCTUATION:
- case Character.INITIAL_QUOTE_PUNCTUATION:
- case Character.OTHER_PUNCTUATION:
- case Character.START_PUNCTUATION:
- return true;
- default:
- switch (codePoint) {
- case '$':
- case '+':
- case '<':
- case '=':
- case '>':
- case '^':
- case '`':
- case '|':
- case '~':
- return true;
- default:
- return false;
- }
- }
- }
-
- public static boolean isWhitespaceCodePoint(int codePoint) {
- switch (codePoint) {
- case ' ':
- case '\t':
- case '\r':
- case '\n':
- case '\f':
- return true;
- default:
- return Character.getType(codePoint) == Character.SPACE_SEPARATOR;
- }
- }
-
- /**
- * Prepares the input line replacing {@code \0}
- */
- public static CharSequence prepareLine(CharSequence line) {
- // Avoid building a new string in the majority of cases (no \0)
- StringBuilder sb = null;
- int length = line.length();
- for (int i = 0; i < length; i++) {
- char c = line.charAt(i);
- if (c == '\0') {
- if (sb == null) {
- sb = new StringBuilder(length);
- sb.append(line, 0, i);
- }
- sb.append('\uFFFD');
- } else {
- if (sb != null) {
- sb.append(c);
- }
- }
- }
-
- if (sb != null) {
- return sb.toString();
- } else {
- return line;
- }
- }
-
- public static int skip(char skip, CharSequence s, int startIndex, int endIndex) {
- for (int i = startIndex; i < endIndex; i++) {
- if (s.charAt(i) != skip) {
- return i;
- }
- }
- return endIndex;
- }
-
- public static int skipBackwards(char skip, CharSequence s, int startIndex, int lastIndex) {
- for (int i = startIndex; i >= lastIndex; i--) {
- if (s.charAt(i) != skip) {
- return i;
- }
- }
- return lastIndex - 1;
- }
-
- public static int skipSpaceTab(CharSequence s, int startIndex, int endIndex) {
- for (int i = startIndex; i < endIndex; i++) {
- switch (s.charAt(i)) {
- case ' ':
- case '\t':
- break;
- default:
- return i;
- }
- }
- return endIndex;
- }
-
- public static int skipSpaceTabBackwards(CharSequence s, int startIndex, int lastIndex) {
- for (int i = startIndex; i >= lastIndex; i--) {
- switch (s.charAt(i)) {
- case ' ':
- case '\t':
- break;
- default:
- return i;
- }
- }
- return lastIndex - 1;
- }
-
- private static int findNonSpace(CharSequence s, int startIndex) {
- int length = s.length();
- for (int i = startIndex; i < length; i++) {
- switch (s.charAt(i)) {
- case ' ':
- case '\t':
- case '\n':
- case '\u000B':
- case '\f':
- case '\r':
- break;
- default:
- return i;
- }
- }
- return -1;
- }
}
diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/Position.java b/commonmark/src/main/java/org/commonmark/parser/beta/Position.java
similarity index 89%
rename from commonmark/src/main/java/org/commonmark/internal/inline/Position.java
rename to commonmark/src/main/java/org/commonmark/parser/beta/Position.java
index 5f06a063a..3dbb4870f 100644
--- a/commonmark/src/main/java/org/commonmark/internal/inline/Position.java
+++ b/commonmark/src/main/java/org/commonmark/parser/beta/Position.java
@@ -1,4 +1,4 @@
-package org.commonmark.internal.inline;
+package org.commonmark.parser.beta;
/**
* Position within a {@link Scanner}. This is intentionally kept opaque so as not to expose the internal structure of
diff --git a/commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java b/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java
similarity index 99%
rename from commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java
rename to commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java
index 9de96a587..482f8eb2a 100644
--- a/commonmark/src/main/java/org/commonmark/internal/inline/Scanner.java
+++ b/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java
@@ -1,9 +1,9 @@
-package org.commonmark.internal.inline;
+package org.commonmark.parser.beta;
-import org.commonmark.internal.util.CharMatcher;
import org.commonmark.node.SourceSpan;
import org.commonmark.parser.SourceLine;
import org.commonmark.parser.SourceLines;
+import org.commonmark.text.CharMatcher;
import java.util.List;
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 30d6d1a5c..03fc7d759 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -1,10 +1,10 @@
package org.commonmark.renderer.markdown;
-import org.commonmark.internal.util.AsciiMatcher;
-import org.commonmark.internal.util.CharMatcher;
-import org.commonmark.internal.util.Parsing;
+import org.commonmark.text.AsciiMatcher;
import org.commonmark.node.*;
+import org.commonmark.text.CharMatcher;
import org.commonmark.renderer.NodeRenderer;
+import org.commonmark.text.Characters;
import java.util.Arrays;
import java.util.HashSet;
@@ -268,7 +268,7 @@ public void visit(Code code) {
// If it starts and ends with a space (but is not only spaces), add an additional space (otherwise they would
// get removed on parsing).
boolean addSpace = literal.startsWith("`") || literal.endsWith("`") ||
- (literal.startsWith(" ") && literal.endsWith(" ") && Parsing.hasNonSpace(literal));
+ (literal.startsWith(" ") && literal.endsWith(" ") && Characters.hasNonSpace(literal));
if (addSpace) {
writer.raw(' ');
}
@@ -416,9 +416,9 @@ private static int findMaxRunLength(char c, CharSequence s) {
int backticks = 0;
int start = 0;
while (start < s.length()) {
- int index = Parsing.find(c, s, start);
+ int index = Characters.find(c, s, start);
if (index != -1) {
- start = Parsing.skip(c, s, index + 1, s.length());
+ start = Characters.skip(c, s, index + 1, s.length());
backticks = Math.max(backticks, start - index);
} else {
break;
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
index 1231a4a73..c9f427021 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/MarkdownWriter.java
@@ -1,6 +1,6 @@
package org.commonmark.renderer.markdown;
-import org.commonmark.internal.util.CharMatcher;
+import org.commonmark.text.CharMatcher;
import java.io.IOException;
import java.util.LinkedList;
diff --git a/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java b/commonmark/src/main/java/org/commonmark/text/AsciiMatcher.java
similarity index 94%
rename from commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java
rename to commonmark/src/main/java/org/commonmark/text/AsciiMatcher.java
index dd7e8d5eb..0d9cea458 100644
--- a/commonmark/src/main/java/org/commonmark/internal/util/AsciiMatcher.java
+++ b/commonmark/src/main/java/org/commonmark/text/AsciiMatcher.java
@@ -1,8 +1,11 @@
-package org.commonmark.internal.util;
+package org.commonmark.text;
import java.util.BitSet;
import java.util.Set;
+/**
+ * Char matcher that can match ASCII characters efficiently.
+ */
public class AsciiMatcher implements CharMatcher {
private final BitSet set;
diff --git a/commonmark/src/main/java/org/commonmark/text/CharMatcher.java b/commonmark/src/main/java/org/commonmark/text/CharMatcher.java
new file mode 100644
index 000000000..2833e65c3
--- /dev/null
+++ b/commonmark/src/main/java/org/commonmark/text/CharMatcher.java
@@ -0,0 +1,13 @@
+package org.commonmark.text;
+
+/**
+ * Matcher interface for {@code char} values.
+ *
+ * Note that because this matches on {@code char} values only (as opposed to {@code int} code points),
+ * this only operates on the level of code units and doesn't support supplementary characters
+ * (see {@link Character#isSupplementaryCodePoint(int)}).
+ */
+public interface CharMatcher {
+
+ boolean matches(char c);
+}
diff --git a/commonmark/src/main/java/org/commonmark/text/Characters.java b/commonmark/src/main/java/org/commonmark/text/Characters.java
new file mode 100644
index 000000000..6c2333c86
--- /dev/null
+++ b/commonmark/src/main/java/org/commonmark/text/Characters.java
@@ -0,0 +1,159 @@
+package org.commonmark.text;
+
+/**
+ * Functions for finding characters in strings or checking characters.
+ */
+public class Characters {
+
+ public static int find(char c, CharSequence s, int startIndex) {
+ int length = s.length();
+ for (int i = startIndex; i < length; i++) {
+ if (s.charAt(i) == c) {
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ public static int findLineBreak(CharSequence s, int startIndex) {
+ int length = s.length();
+ for (int i = startIndex; i < length; i++) {
+ switch (s.charAt(i)) {
+ case '\n':
+ case '\r':
+ return i;
+ }
+ }
+ return -1;
+ }
+
+ public static boolean isBlank(CharSequence s) {
+ return findNonSpace(s, 0) == -1;
+ }
+
+ public static boolean hasNonSpace(CharSequence s) {
+ int length = s.length();
+ int skipped = skip(' ', s, 0, length);
+ return skipped != length;
+ }
+
+ public static boolean isLetter(CharSequence s, int index) {
+ int codePoint = Character.codePointAt(s, index);
+ return Character.isLetter(codePoint);
+ }
+
+ public static boolean isSpaceOrTab(CharSequence s, int index) {
+ if (index < s.length()) {
+ switch (s.charAt(index)) {
+ case ' ':
+ case '\t':
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // See https://spec.commonmark.org/0.29/#punctuation-character
+ public static boolean isPunctuationCodePoint(int codePoint) {
+ switch (Character.getType(codePoint)) {
+ case Character.CONNECTOR_PUNCTUATION:
+ case Character.DASH_PUNCTUATION:
+ case Character.END_PUNCTUATION:
+ case Character.FINAL_QUOTE_PUNCTUATION:
+ case Character.INITIAL_QUOTE_PUNCTUATION:
+ case Character.OTHER_PUNCTUATION:
+ case Character.START_PUNCTUATION:
+ return true;
+ default:
+ switch (codePoint) {
+ case '$':
+ case '+':
+ case '<':
+ case '=':
+ case '>':
+ case '^':
+ case '`':
+ case '|':
+ case '~':
+ return true;
+ default:
+ return false;
+ }
+ }
+ }
+
+ public static boolean isWhitespaceCodePoint(int codePoint) {
+ switch (codePoint) {
+ case ' ':
+ case '\t':
+ case '\r':
+ case '\n':
+ case '\f':
+ return true;
+ default:
+ return Character.getType(codePoint) == Character.SPACE_SEPARATOR;
+ }
+ }
+
+ public static int skip(char skip, CharSequence s, int startIndex, int endIndex) {
+ for (int i = startIndex; i < endIndex; i++) {
+ if (s.charAt(i) != skip) {
+ return i;
+ }
+ }
+ return endIndex;
+ }
+
+ public static int skipBackwards(char skip, CharSequence s, int startIndex, int lastIndex) {
+ for (int i = startIndex; i >= lastIndex; i--) {
+ if (s.charAt(i) != skip) {
+ return i;
+ }
+ }
+ return lastIndex - 1;
+ }
+
+ public static int skipSpaceTab(CharSequence s, int startIndex, int endIndex) {
+ for (int i = startIndex; i < endIndex; i++) {
+ switch (s.charAt(i)) {
+ case ' ':
+ case '\t':
+ break;
+ default:
+ return i;
+ }
+ }
+ return endIndex;
+ }
+
+ public static int skipSpaceTabBackwards(CharSequence s, int startIndex, int lastIndex) {
+ for (int i = startIndex; i >= lastIndex; i--) {
+ switch (s.charAt(i)) {
+ case ' ':
+ case '\t':
+ break;
+ default:
+ return i;
+ }
+ }
+ return lastIndex - 1;
+ }
+
+ private static int findNonSpace(CharSequence s, int startIndex) {
+ int length = s.length();
+ for (int i = startIndex; i < length; i++) {
+ switch (s.charAt(i)) {
+ case ' ':
+ case '\t':
+ case '\n':
+ case '\u000B':
+ case '\f':
+ case '\r':
+ break;
+ default:
+ return i;
+ }
+ }
+ return -1;
+ }
+}
diff --git a/commonmark/src/test/java/org/commonmark/internal/inline/ScannerTest.java b/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java
similarity index 97%
rename from commonmark/src/test/java/org/commonmark/internal/inline/ScannerTest.java
rename to commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java
index 030a765af..668f852c4 100644
--- a/commonmark/src/test/java/org/commonmark/internal/inline/ScannerTest.java
+++ b/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java
@@ -1,8 +1,10 @@
-package org.commonmark.internal.inline;
+package org.commonmark.parser.beta;
import org.commonmark.node.SourceSpan;
import org.commonmark.parser.SourceLine;
import org.commonmark.parser.SourceLines;
+import org.commonmark.parser.beta.Position;
+import org.commonmark.parser.beta.Scanner;
import org.junit.Test;
import java.util.Arrays;
diff --git a/commonmark/src/test/java/org/commonmark/internal/util/ParsingTest.java b/commonmark/src/test/java/org/commonmark/text/CharactersTest.java
similarity index 78%
rename from commonmark/src/test/java/org/commonmark/internal/util/ParsingTest.java
rename to commonmark/src/test/java/org/commonmark/text/CharactersTest.java
index f51c8647b..23bf07355 100644
--- a/commonmark/src/test/java/org/commonmark/internal/util/ParsingTest.java
+++ b/commonmark/src/test/java/org/commonmark/text/CharactersTest.java
@@ -1,10 +1,10 @@
-package org.commonmark.internal.util;
+package org.commonmark.text;
import org.junit.Test;
import static org.junit.Assert.assertTrue;
-public class ParsingTest {
+public class CharactersTest {
@Test
public void isPunctuation() {
@@ -17,7 +17,7 @@ public void isPunctuation() {
};
for (char c : chars) {
- assertTrue("Expected to be punctuation: " + c, Parsing.isPunctuationCodePoint(c));
+ assertTrue("Expected to be punctuation: " + c, Characters.isPunctuationCodePoint(c));
}
}
}
diff --git a/pom.xml b/pom.xml
index da06b2c2b..4eac3b7b2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -56,7 +56,7 @@
org.apache.maven.pluginsmaven-javadoc-plugin
- 3.4.1
+ 3.5.0*.internal,*.internal.*
From a3d31d99fff86be3e6561e23ee66fd0310ed2adb Mon Sep 17 00:00:00 2001
From: Robin Stocker
Date: Sat, 2 Mar 2024 11:37:13 +1100
Subject: [PATCH 073/286] Clean up Characters, Strings test util
---
.../java/org/commonmark/testutil/Strings.java | 12 -----
.../markdown/CoreMarkdownNodeRenderer.java | 27 ++++------
.../java/org/commonmark/text/Characters.java | 34 +++++-------
.../org/commonmark/test/PathologicalTest.java | 52 +++++++++----------
.../org/commonmark/test/SpecialInputTest.java | 5 +-
.../org/commonmark/text/CharactersTest.java | 11 ++++
6 files changed, 62 insertions(+), 79 deletions(-)
delete mode 100644 commonmark-test-util/src/main/java/org/commonmark/testutil/Strings.java
diff --git a/commonmark-test-util/src/main/java/org/commonmark/testutil/Strings.java b/commonmark-test-util/src/main/java/org/commonmark/testutil/Strings.java
deleted file mode 100644
index ed709ed81..000000000
--- a/commonmark-test-util/src/main/java/org/commonmark/testutil/Strings.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package org.commonmark.testutil;
-
-public class Strings {
-
- public static String repeat(String s, int count) {
- StringBuilder sb = new StringBuilder(s.length() * count);
- for (int i = 0; i < count; i++) {
- sb.append(s);
- }
- return sb.toString();
- }
-}
diff --git a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
index 03fc7d759..31a7ceb50 100644
--- a/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
+++ b/commonmark/src/main/java/org/commonmark/renderer/markdown/CoreMarkdownNodeRenderer.java
@@ -149,11 +149,12 @@ public void visit(IndentedCodeBlock indentedCodeBlock) {
@Override
public void visit(FencedCodeBlock fencedCodeBlock) {
String literal = fencedCodeBlock.getLiteral();
- String fence = repeat(String.valueOf(fencedCodeBlock.getFenceChar()), fencedCodeBlock.getFenceLength());
+ int count = fencedCodeBlock.getFenceLength();
+ String fence = String.valueOf(fencedCodeBlock.getFenceChar()).repeat(count);
int indent = fencedCodeBlock.getFenceIndent();
if (indent > 0) {
- String indentPrefix = repeat(" ", indent);
+ String indentPrefix = " ".repeat(indent);
writer.writePrefix(indentPrefix);
writer.pushPrefix(indentPrefix);
}
@@ -231,18 +232,20 @@ public void visit(ListItem listItem) {
boolean pushedPrefix = false;
if (listHolder instanceof BulletListHolder) {
BulletListHolder bulletListHolder = (BulletListHolder) listHolder;
- String marker = repeat(" ", listItem.getMarkerIndent()) + bulletListHolder.bulletMarker;
+ int count = listItem.getMarkerIndent();
+ String marker = " ".repeat(count) + bulletListHolder.bulletMarker;
writer.writePrefix(marker);
- writer.writePrefix(repeat(" ", contentIndent - marker.length()));
- writer.pushPrefix(repeat(" ", contentIndent));
+ writer.writePrefix(" ".repeat(contentIndent - marker.length()));
+ writer.pushPrefix(" ".repeat(contentIndent));
pushedPrefix = true;
} else if (listHolder instanceof OrderedListHolder) {
OrderedListHolder orderedListHolder = (OrderedListHolder) listHolder;
- String marker = repeat(" ", listItem.getMarkerIndent()) + orderedListHolder.number + orderedListHolder.delimiter;
+ int count = listItem.getMarkerIndent();
+ String marker = " ".repeat(count) + orderedListHolder.number + orderedListHolder.delimiter;
orderedListHolder.number++;
writer.writePrefix(marker);
- writer.writePrefix(repeat(" ", contentIndent - marker.length()));
- writer.pushPrefix(repeat(" ", contentIndent));
+ writer.writePrefix(" ".repeat(contentIndent - marker.length()));
+ writer.pushPrefix(" ".repeat(contentIndent));
pushedPrefix = true;
}
if (listItem.getFirstChild() == null) {
@@ -436,14 +439,6 @@ private static boolean contains(String s, CharMatcher charMatcher) {
return false;
}
- private static String repeat(String s, int count) {
- StringBuilder sb = new StringBuilder(s.length() * count);
- for (int i = 0; i < count; i++) {
- sb.append(s);
- }
- return sb.toString();
- }
-
private static List getLines(String literal) {
// Without -1, split would discard all trailing empty strings, which is not what we want, e.g. it would
// return the same result for "abc", "abc\n" and "abc\n\n".
diff --git a/commonmark/src/main/java/org/commonmark/text/Characters.java b/commonmark/src/main/java/org/commonmark/text/Characters.java
index 6c2333c86..4d9532329 100644
--- a/commonmark/src/main/java/org/commonmark/text/Characters.java
+++ b/commonmark/src/main/java/org/commonmark/text/Characters.java
@@ -27,8 +27,11 @@ public static int findLineBreak(CharSequence s, int startIndex) {
return -1;
}
+ /**
+ * @see blank line
+ */
public static boolean isBlank(CharSequence s) {
- return findNonSpace(s, 0) == -1;
+ return skipSpaceTab(s, 0, s.length()) == s.length();
}
public static boolean hasNonSpace(CharSequence s) {
@@ -53,7 +56,9 @@ public static boolean isSpaceOrTab(CharSequence s, int index) {
return false;
}
- // See https://spec.commonmark.org/0.29/#punctuation-character
+ /**
+ * @see punctuation character
+ */
public static boolean isPunctuationCodePoint(int codePoint) {
switch (Character.getType(codePoint)) {
case Character.CONNECTOR_PUNCTUATION:
@@ -82,13 +87,18 @@ public static boolean isPunctuationCodePoint(int codePoint) {
}
}
+ /**
+ * Check whether the provided code point is a Unicode whitespace character as defined in the spec.
+ *
+ * @see Unicode whitespace character
+ */
public static boolean isWhitespaceCodePoint(int codePoint) {
switch (codePoint) {
case ' ':
case '\t':
- case '\r':
case '\n':
case '\f':
+ case '\r':
return true;
default:
return Character.getType(codePoint) == Character.SPACE_SEPARATOR;
@@ -138,22 +148,4 @@ public static int skipSpaceTabBackwards(CharSequence s, int startIndex, int last
}
return lastIndex - 1;
}
-
- private static int findNonSpace(CharSequence s, int startIndex) {
- int length = s.length();
- for (int i = startIndex; i < length; i++) {
- switch (s.charAt(i)) {
- case ' ':
- case '\t':
- case '\n':
- case '\u000B':
- case '\f':
- case '\r':
- break;
- default:
- return i;
- }
- }
- return -1;
- }
}
diff --git a/commonmark/src/test/java/org/commonmark/test/PathologicalTest.java b/commonmark/src/test/java/org/commonmark/test/PathologicalTest.java
index a853b1b11..ae1310ed2 100644
--- a/commonmark/src/test/java/org/commonmark/test/PathologicalTest.java
+++ b/commonmark/src/test/java/org/commonmark/test/PathologicalTest.java
@@ -10,8 +10,6 @@
import java.util.concurrent.TimeUnit;
-import static org.commonmark.testutil.Strings.repeat;
-
/**
* Pathological input cases (from commonmark.js).
*/
@@ -36,58 +34,58 @@ public void nestedStrongEmphasis() {
// this is limited by the stack size because visitor is recursive
x = 500;
assertRendering(
- repeat("*a **a ", x) + "b" + repeat(" a** a*", x),
- "
" + repeat("a a ", x) + "b" +
- repeat(" a a", x) + "
\n");
}
@Test
public void nestedBrackets() {
assertRendering(
- repeat("[", x) + "a" + repeat("]", x),
- "
" + repeat("[", x) + "a" + repeat("]", x) + "
\n");
+ "[".repeat(x) + "a" + "]".repeat(x),
+ "
" + "[".repeat(x) + "a" + "]".repeat(x) + "
\n");
}
@Test
@@ -95,29 +93,29 @@ public void nestedBlockQuotes() {
// this is limited by the stack size because visitor is recursive
x = 1000;
assertRendering(
- repeat("> ", x) + "a\n",
- repeat("