Support include of code snippets in a document #7

.gitignore

@@ -6,3 +6,4 @@ jaxb
 .idea
 *.iml
 *.versionsBackup
+/src/main/antlr4/.antlr

README.md

@@ -1,6 +1,6 @@
 # md-grammar
 
-Parser and writer for markdown documents. 
+Parser and writer for Markdown documents. 
 
 The markdown grammar follows [GitHub Flavored Markdown Spec](https://github.github.com/gfm/). 
 

pom.xml

@@ -54,7 +54,7 @@
 		<antlr.version>4.9.2</antlr.version>
 		<log4j.version>2.14.1</log4j.version>
 		<java.version>11</java.version>
-		<junit.version>5.7.0</junit.version>
+		<junit.version>5.7.2</junit.version>
 	</properties>
 
 	<dependencies>

src/main/antlr4/io/fixprotocol/md/antlr/MarkdownLexer.g4

@@ -166,9 +166,14 @@ LINECHAR
 
 mode FENCED;
 
+CLOSE_FENCE
+:
+	'```' (FENCED_NEWLINE | EOF) -> mode(DEFAULT_MODE)
+;
+
 TEXTLINE
 :
-	INITIALTEXTCHAR TEXTCHAR* NEWLINE
+	INITIALTEXTCHAR TEXTCHAR* FENCED_NEWLINE
 ;
 
 
@@ -178,11 +183,6 @@ INITIALTEXTCHAR
 	~[\n\r]
 ;
 
-CLOSE_FENCE
-:
-	'```' -> mode(DEFAULT_MODE)
-;
-
 LINENUMBER
 :
 	DIGIT+
@@ -215,7 +215,7 @@ WORD
 
 FENCED_NEWLINE
 :
-	'\r'? '\n' -> type(NEWLINE)
+	'\r'? '\n' 
 ;
 
 FENCED_IGNORE_WS

src/main/antlr4/io/fixprotocol/md/antlr/MarkdownParser.g4

@@ -68,7 +68,7 @@ quoteline
 
 fencedcodeblock
 :
-	OPEN_FENCE infostring? FENCED_IGNORE_WS? importspec? NEWLINE
+	OPEN_FENCE infostring? FENCED_IGNORE_WS? importspec? FENCED_NEWLINE
 	textline*
 	CLOSE_FENCE
 ;

src/main/java/io/fixprotocol/md/antlr/MarkdownEventSource.java

@@ -24,6 +24,7 @@
 import java.util.function.Consumer;
 import java.util.stream.Collectors;
 import org.antlr.v4.runtime.ParserRuleContext;
+import org.antlr.v4.runtime.RuleContext;
 import org.antlr.v4.runtime.tree.ErrorNode;
 import org.antlr.v4.runtime.tree.TerminalNode;
 import org.apache.logging.log4j.LogManager;
@@ -341,7 +342,7 @@ public void exitFencedcodeblock(FencedcodeblockContext ctx) {
     }
     if (text.isEmpty()) {
       final List<TextlineContext> lines = ctx.textline();
-      text = lines.stream().map(l -> l.getText()).collect(Collectors.joining("\n"));
+      text = lines.stream().map(RuleContext::getText).collect(Collectors.joining("\n"));
     }
 
     final MutableDocumentation documentation = contextFactory.createDocumentation(text, format);

src/main/java/io/fixprotocol/md/event/Context.java

@@ -33,7 +33,7 @@ public interface Context extends GraphContext, DocumentContext {
   /**
    * An array of keywords. Position may be significant.
    *
-   * @return an array of key words or {@link #EMPTY_CONTEXT} if no keys are known
+   * @return an array of keywords or {@link #EMPTY_CONTEXT} if no keys are known
    */
   String[] getKeys();
 

src/main/java/io/fixprotocol/md/event/DatatypeInference.java

@@ -19,11 +19,7 @@ class DatatypeInference {
     final ParsePosition parsePosition = new ParsePosition(0);
     numberFormat.parse(t, parsePosition);
     final int index = parsePosition.getIndex();
-    if (index == 0 || index < t.length()) {
-      return false;
-    } else {
-      return true;
-    }
+      return index != 0 && index >= t.length();
   };
 
   private final String falseValue;

src/main/java/io/fixprotocol/md/event/DetailProperties.java

@@ -27,7 +27,7 @@
 public interface DetailProperties {
 
   /**
-   * Access a integer property by its key
+   * Access an integer property by its key
    *
    * @param key key to the property
    * @return value of the property, or {@code null} if the property does not exist or is non-numeric

src/main/java/io/fixprotocol/md/event/DetailTable.java

@@ -30,14 +30,14 @@ interface TableRow extends DetailProperties, DocumentContext {
 
   /**
    *
-   * @return an List of TableColumn that describes this table
+   * @return a List of TableColumn that describes this table
    */
   List<? extends TableColumn> getTableColumns();
 
   /**
-   * Supplies a Iterable of row values
+   * Supplies an Iterable of row values
    *
-   * @return a Iterable of TableRow
+   * @return an Iterable of TableRow
    */
   Iterable<? extends TableRow> rows();
 }

src/main/java/io/fixprotocol/md/event/DocumentContext.java

@@ -22,7 +22,7 @@ public interface DocumentContext {
   int getCharPositionInLine();
 
   /**
-   * Line number in a document
+   * Returns ine number in a document
    *
    * @return line number or {@link #UNKNOWN_POSITION}
    */

src/main/java/io/fixprotocol/md/event/DocumentParser.java

@@ -73,7 +73,7 @@ void logError(int line, int charPositionInLine, String msg) {
   }
 
   /**
-   * Parse a markdown document
+   * Parse a Markdown document
    *
    * @param inputStream input as markdown
    * @param contextConsumer consumer of document events
@@ -86,7 +86,7 @@ public boolean parse(InputStream inputStream, Consumer<? super GraphContext> con
   }
 
   /**
-   * Parse a markdown document
+   * Parse a Markdown document
    *
    * @param inputStream input as markdown. Text is assumed to encoded as UTF-8.
    * @param contextConsumer consumer of document events

src/main/java/io/fixprotocol/md/event/DocumentWriter.java

@@ -101,7 +101,7 @@ public void write(DetailTable detailTable, final List<? extends TableColumn> tab
       for (final Entry<String, String> p : row.getProperties()) {
         final TableColumn tc = columnsByKey.get(p.getKey());
         if (tc != null) {
-          // Set spacing to longest value in each column
+          // Set spacing to the longest value in each column
           tc.updateWidth(p.getValue().length());
           // Infer common datatype
           final Class<?> datatype = datatypes.inferDatatype(p.getValue());

src/main/java/io/fixprotocol/md/event/MarkdownUtil.java

@@ -44,7 +44,7 @@ public static String markdownLiteralToPlainText(String literal) {
   /**
    * Translates plaintext to markdown
    *
-   * A standard markdown paragraph break is used.
+   * A standard Markdown paragraph break is used.
    *
    * @param text plaintext
    * @return a markdown string
@@ -121,7 +121,7 @@ public static String plainTextToMarkdownLiteral(String text) {
 
   /**
    * Trims leading and trailing whitespace or pipe characters, leaving just the text within a
-   * markdown table cell.
+   * Markdown table cell.
    *
    * @param str string to strip
    * @return a string without leading or trailing whitespace, or {@code null} if the parameter is

src/main/java/io/fixprotocol/md/event/TableColumn.java

@@ -18,7 +18,7 @@
 /**
  * Mutable attributes of a table column
  *
- * Attributes are mutable because a function that composes a table needs to be adjust spacing, etc.
+ * Attributes are mutable because a function that composes a table needs to adjust spacing, etc.
  * for best presentation.
  *
  * @author Don Mendelson
@@ -58,7 +58,7 @@ enum Alignment {
   int getWidth();
 
   /**
-   * Set the heading to display, may be different than key
+   * Set the heading to display, may be different from its key
    *
    * @param display text to display
    */

src/main/java/io/fixprotocol/md/util/AssociativeSet.java

@@ -39,7 +39,7 @@ public boolean add(final String v1, final String v2) {
   /**
    * Add values to this AssociativeSet
    *
-   * @param values a array of two-element arrays
+   * @param values an array of two-element arrays
    * @return {@code true} if this set did not already contain the specified elements
    */
   public boolean addAll(String[][] values) {

src/main/java/io/fixprotocol/md/util/FileSpec.java

@@ -12,7 +12,7 @@
  * start. This start and end must both be line numbers or both must be searches.
  *
  * A begin search stops at the first instance of the text specified. It is recommended to search for
- * unique values. An end search matches the first instance of text after the the start search's
+ * unique values. An end search matches the first instance of text after the start search's
  * target. The end search need not be unique in the file.
  *
  * If a range is not specified, then the whole file is assumed.

src/main/java/io/fixprotocol/md/util/ListUtil.java

@@ -13,8 +13,6 @@ public static <T> void append(List<T> dest, List<T> source, int from, int toExcl
   public static <T> void insert(List<T> dest, int insertPos, List<T> source, int from,
       int toExclusive) {
     dest.addAll(insertPos, source.subList(from, toExclusive));
-    // System.out.format("Insert %s at %d => %s%n", source.subList(from, toExclusive).toString(),
-    // insertPos, dest.toString());
   }
 
   /**
@@ -43,8 +41,6 @@ public static <T> List<T> merge(List<T> first, List<T> second) {
     for (int pos2 = 0; pos2 < second.size(); pos2++) {
       final int matchPos = merged.indexOf(second.get(pos2));
       if (matchPos != -1) {
-        // System.out.format("matched at pos2=%d pos1=%d val=[%s]%n", pos2, matchPos,
-        // second.get(pos2));
         insert(merged, matchPos, second, lastMatchPos + 1, pos2);
         lastMatchPos = pos2;
       } /*

src/test/java/io/fixprotocol/md/antlr/InfostringTest.java

@@ -165,14 +165,18 @@ void textRange() throws IOException {
     fail("Fenced code block not found");
   }
 
-
   @ParameterizedTest
   @ValueSource(strings = {"src/test/resources/fileimport.md"})
   void testRig(String fileName) throws Exception {
-    String[] args =
-        new String[] {"io.fixprotocol.md.antlr.Markdown", "document", "-tree", "-tokens", fileName};
-    TestRig testRig = new TestRig(args);
-    testRig.process();
+    try {
+      System.setOut(new PrintStream(new File("target/test/MdGrammarTest-fileimport.txt")));
+      String[] args =
+          new String[] {"io.fixprotocol.md.antlr.Markdown", "document", "-tree", "-tokens", fileName};
+      TestRig testRig = new TestRig(args);
+      testRig.process();
+    } finally {
+      System.setOut(System.out);
+    }
   }
 
 }

src/test/java/io/fixprotocol/md/antlr/MdGrammarTest.java

@@ -69,14 +69,18 @@ void invalid(String fileName) throws IOException {
     }
   }
 
-  @Disabled
   @ParameterizedTest
   @ValueSource(strings = {"src/test/resources/md2orchestra-proto.md"})
   void testRig(String fileName) throws Exception {
-    String[] args = new String[] {"io.fixprotocol.md.antlr.Markdown", "document", "-tree",
-        "-tokens", fileName};
-    TestRig testRig = new TestRig(args);
-    testRig.process();
+    try {
+      System.setOut(new PrintStream(new File("target/test/MdGrammarTest.txt")));
+      String[] args = new String[]{"io.fixprotocol.md.antlr.Markdown", "document", "-tree",
+              "-tokens", fileName};
+      TestRig testRig = new TestRig(args);
+      testRig.process();
+    } finally {
+      System.setOut(System.out);
+    }
   }
 
   @ParameterizedTest

src/test/java/io/fixprotocol/md/event/ConsumerTest.java

@@ -14,15 +14,29 @@
  */
 package io.fixprotocol.md.event;
 
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.*;
 import java.util.function.Consumer;
+
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
 
 public class ConsumerTest {
 
+  private static PrintStream out;
+
+  @BeforeAll
+  static void setUpOnce() throws FileNotFoundException {
+    new File("target/test").mkdirs();
+    out = new PrintStream(new FileOutputStream("target/test/ConsumerTest.txt"));
+  }
+
+  @AfterAll
+  static void cleanUpOnce() throws FileNotFoundException {
+    out.close();
+  }
+
   @ParameterizedTest
   @ValueSource(strings = {"src/test/resources/md2orchestra-proto.md"})
   void consume(String fileName) throws IOException {
@@ -33,21 +47,21 @@ public void accept(GraphContext contextual) {
         Context parent = contextual.getParent();
         if (parent != null) {
           final String[] parentKeys = parent.getKeys();
-          System.out.format("Parent context=%s level=%d%n",
+          out.format("Parent context=%s level=%d%n",
               parentKeys.length > 0 ? parentKeys[0] : "None", parent.getLevel());
         }
         if (contextual instanceof Detail) {
           Detail detail = (Detail) contextual;
-          detail.getProperties().forEach(property -> System.out.format("Property key=%s value=%s%n",
+          detail.getProperties().forEach(property -> out.format("Property key=%s value=%s%n",
               property.getKey(), property.getValue()));
         } else if (contextual instanceof Documentation) {
           Documentation documentation = (Documentation) contextual;       
-          System.out.format("Documentation %s format %s%n", documentation.getDocumentation(), 
+          out.format("Documentation %s format %s%n", documentation.getDocumentation(),
               documentation.getFormat());
         } else if (contextual instanceof Context) {
           Context context = (Context) contextual;
           final String[] keys = context.getKeys();
-          System.out.format("Context=%s level=%d%n", keys.length > 0 ? keys[0] : "None",
+          out.format("Context=%s level=%d%n", keys.length > 0 ? keys[0] : "None",
               context.getLevel());
         }