Fix implicit sentence detection

Natural language depends on either sentence terminal markers, or
implicit empty lines, for sentence disambiguation.  Markdown does not.

Previously, when list-items did not end in a typical terminal marker,
e.g., `!`, `?`, or `.`, these in markdown clear sentences were not
detected reliably.  This update ensures that, between markdown
block level nodes, sentences can occur in NLCST.

Author: wooorm

component.json

@@ -13,6 +13,7 @@
   ],
   "dependencies": {
     "wooorm/remark-range": "^2.0.0",
+    "jonschlinkert/repeat-string": "^1.5.2",
     "wooorm/nlcst-to-string": "^1.0.0"
   },
   "repository": "wooorm/mdast-util-to-nlcst",

index.js

@@ -17,6 +17,7 @@
 
 var range = require('remark-range');
 var toString = require('nlcst-to-string');
+var repeat = require('repeat-string');
 
 /*
  * Map of ignored mdast nodes: nodes which have no (simple)
@@ -34,7 +35,7 @@ var IGNORE = {
  * Constants.
  */
 
-var NON_NEWLINE = /[^\n]/;
+var C_NEWLINE = '\n';
 
 /**
  * Create an position object for `offset` in `file`.
@@ -129,13 +130,39 @@ all = function (parent, file, parser) {
     var index = -1;
     var result = [];
     var child;
+    var node;
+    var pos;
+    var prevEndLine;
+    var prevOffset;
+    var endLine;
 
     while (++index < length) {
-        child = one(children[index], index, parent, file, parser);
+        node = children[index];
+        pos = node.position;
+        endLine = pos.start.line;
+
+        if (prevEndLine && endLine !== prevEndLine) {
+            child = parser.tokenizeWhiteSpace(
+                repeat(C_NEWLINE, endLine - prevEndLine)
+            );
+
+            patch([child], file, prevOffset);
+
+            if (child.value.length < 2) {
+                child.value = repeat(C_NEWLINE, 2);
+            }
+
+            result.push(child);
+        }
+
+        child = one(node, index, parent, file, parser);
 
         if (child) {
             result = result.concat(child);
         }
+
+        prevEndLine = pos.end.line;
+        prevOffset = pos.end.offset;
     }
 
     return result;
@@ -154,17 +181,10 @@ all = function (parent, file, parser) {
  */
 one = function (node, index, parent, file, parser) {
     var type = node.type;
-    var siblings = parent && parent.children;
-    var prev = siblings && siblings[index - 1];
     var pos = node.position;
     var start = pos.start;
     var end = pos.end;
-    var final = prev && prev.position.end.offset;
     var replacement;
-    var result;
-    var space;
-
-    space = final && file.toString().slice(final, start.offset);
 
     if (type in IGNORE) {
         return null;
@@ -194,20 +214,6 @@ one = function (node, index, parent, file, parser) {
         )], file, start.offset);
     }
 
-    /**
-     * There’s a difference between block-nodes with
-     * lines between them. NLCST parsers need them to
-     * differentiate between paragraphs.
-     */
-
-    if (replacement && space && !NON_NEWLINE.test(space)) {
-        result = parser.tokenizeWhiteSpace(space);
-
-        patch([result], file, final);
-
-        replacement.unshift(result);
-    }
-
     return replacement || null;
 };
 

package.json

@@ -12,8 +12,9 @@
     "language"
   ],
   "dependencies": {
+    "nlcst-to-string": "^1.0.0",
     "remark-range": "^2.0.0",
-    "nlcst-to-string": "^1.0.0"
+    "repeat-string": "^1.5.2"
   },
   "repository": {
     "type": "git",

test/fixtures/blocks/output.json

@@ -345,6 +345,22 @@
         }
       }
     },
+    {
+      "type": "WhiteSpaceNode",
+      "value": "\n\n\n",
+      "position": {
+        "start": {
+          "line": 7,
+          "column": 24,
+          "offset": 53
+        },
+        "end": {
+          "line": 10,
+          "column": 1,
+          "offset": 56
+        }
+      }
+    },
     {
       "type": "WhiteSpaceNode",
       "value": "\n\n",
@@ -591,6 +607,22 @@
         }
       }
     },
+    {
+      "type": "WhiteSpaceNode",
+      "value": "\n\n",
+      "position": {
+        "start": {
+          "line": 12,
+          "column": 20,
+          "offset": 100
+        },
+        "end": {
+          "line": 14,
+          "column": 1,
+          "offset": 102
+        }
+      }
+    },
     {
       "type": "WhiteSpaceNode",
       "value": "\n\n",

test/fixtures/ignore/input.md

@@ -8,8 +8,8 @@ Rules:
 
 Tables:
 
-| Foo |
-| --- |
-| Bar |
+| Foo | Bar |
+| --- | --- |
+| Baz | Qux |
 
 ...that’s all, for now 😄.