Merge pull request #4 from digitalmoksha/sync-with-9.0.0

Sync from 8.4.1 to 9.0.1

Author: digitalmoksha

CHANGELOG.md

@@ -1,3 +1,8 @@
+9.0.1
+-------
+
+Synced with markdown-it 9.0.1, see the [CHANGELOG](https://github.com/markdown-it/markdown-it/blob/master/CHANGELOG.md)
+
 8.4.1
 -------
 

Gemfile

@@ -7,4 +7,8 @@ gem 'kramdown', require: 'kramdown'
 gem 'commonmarker'
 gem 'redcarpet'
 
+group :development, :test do
+  gem 'pry-byebug'
+end
+
 gemspec

README.md

@@ -5,7 +5,9 @@
 
 Ruby/RubyMotion version of Markdown-it (CommonMark compliant and extendable)
 
-This gem is a port of the [markdown-it Javascript package](https://github.com/markdown-it/markdown-it) by Vitaly Puzrin and Alex Kocharin. Currently synced with markdown-it 8.4.1
+This gem is a port of the [markdown-it Javascript package](https://github.com/markdown-it/markdown-it) by Vitaly Puzrin and Alex Kocharin. 
+
+_Currently synced with markdown-it 9.0.1_
 
 ---
 

lib/motion-markdown-it/common/entities.rb

@@ -3,6 +3,8 @@ module MarkdownIt
 
   # TODO this list needs to be brought up to same level as the WC3 document
   # http://www.w3.org/TR/xml-entity-names/byalpha.html
+  # Consider pulling in https://github.com/fb55/entities/blob/master/maps/entities.json,
+  # is kept up to date
   #------------------------------------------------------------------------------
   class HTMLEntities
 
@@ -626,6 +628,7 @@ class HTMLEntities
       'nearr'          => 0x2197,   # ↗                  NORTH EAST ARROW
       'nequiv'         => 0x2262,   # ≢                  NOT IDENTICAL TO
       'nexist'         => 0x2204,   # ∄                  THERE DOES NOT EXIST
+      'ngE'            => '≧̸',      # ≧̸                  from https://github.com/fb55/entities/blob/master/maps/entities.json
       'nge'            => 0x2271,   # ≱   dup            NEITHER GREATER-THAN NOR EQUAL TO
       'nges'           => 0x2271,   # ≱   dup       skip NEITHER GREATER-THAN NOR EQUAL TO
       'Ngr'            => 0x039d,   # Ν   dup       skip GREEK CAPITAL LETTER NU

lib/motion-markdown-it/common/utils.rb

@@ -48,14 +48,17 @@ def isValidEntityCode(c)
 
       #------------------------------------------------------------------------------
       def fromCodePoint(c)
-        c.chr(Encoding::UTF_8)
+        # Some html entities are mapped directly as characters rather than code points.
+        # So if we're passed an Integer, convert to character, otherwise just return
+        # the character.  For example `≧̸`
+        c.is_a?(Integer) ? c.chr(Encoding::UTF_8) : c
       end
 
       #------------------------------------------------------------------------------
       def fromCharCode(c)
         c.chr
       end
-      
+
       #------------------------------------------------------------------------------
       def charCodeAt(str, ch)
         str[ch].ord unless str[ch].nil?
@@ -216,11 +219,44 @@ def isMdAsciiPunct(ch)
       # Hepler to unify [reference labels].
       #------------------------------------------------------------------------------
       def normalizeReference(str)
-        # use .toUpperCase() instead of .toLowerCase()
-        # here to avoid a conflict with Object.prototype
-        # members (most notably, `__proto__`)
-        return str.strip.gsub(/\s+/, ' ').upcase
+        # Trim and collapse whitespace
+        #
+        str = str.strip.gsub(/\s+/, ' ')
+
+        # .toLowerCase().toUpperCase() should get rid of all differences
+        # between letter variants.
+        #
+        # Simple .toLowerCase() doesn't normalize 125 code points correctly,
+        # and .toUpperCase doesn't normalize 6 of them (list of exceptions:
+        # İ, ϴ, ẞ, Ω, K, Å - those are already uppercased, but have differently
+        # uppercased versions).
+        #
+        # Here's an example showing how it happens. Lets take greek letter omega:
+        # uppercase U+0398 (Θ), U+03f4 (ϴ) and lowercase U+03b8 (θ), U+03d1 (ϑ)
+        #
+        # Unicode entries:
+        # 0398;GREEK CAPITAL LETTER THETA;Lu;0;L;;;;;N;;;;03B8;
+        # 03B8;GREEK SMALL LETTER THETA;Ll;0;L;;;;;N;;;0398;;0398
+        # 03D1;GREEK THETA SYMBOL;Ll;0;L;<compat> 03B8;;;;N;GREEK SMALL LETTER SCRIPT THETA;;0398;;0398
+        # 03F4;GREEK CAPITAL THETA SYMBOL;Lu;0;L;<compat> 0398;;;;N;;;;03B8;
+        #
+        # Case-insensitive comparison should treat all of them as equivalent.
+        #
+        # But .toLowerCase() doesn't change ϑ (it's already lowercase),
+        # and .toUpperCase() doesn't change ϴ (already uppercase).
+        #
+        # Applying first lower then upper case normalizes any character:
+        # '\u0398\u03f4\u03b8\u03d1'.toLowerCase().toUpperCase() === '\u0398\u0398\u0398\u0398'
+        #
+        # Note: this is equivalent to unicode case folding; unicode normalization
+        # is a different step that is not required here.
+        #
+        # Final result should be uppercased, because it's later stored in an object
+        # (this avoid a conflict with Object.prototype members,
+        # most notably, `__proto__`)
+        #
+        return str.downcase.upcase
       end
     end
   end
-end
+end

lib/motion-markdown-it/helpers/parse_link_destination.rb

@@ -15,7 +15,7 @@ def parseLinkDestination(str, pos, max)
           pos += 1
           while (pos < max)
             code = charCodeAt(str, pos)
-            return result if (code == 0x0A || isSpace(code)) # \n
+            return result if (code == 0x0A) # \n
             if (code == 0x3E) #  >
               result[:pos] = pos + 1
               result[:str] = unescapeAll(str.slice((start + 1)...pos))
@@ -73,4 +73,4 @@ def parseLinkDestination(str, pos, max)
       end
     end
   end
-end
+end

lib/motion-markdown-it/parser_core.rb

@@ -8,9 +8,9 @@ module MarkdownIt
   class ParserCore
 
     attr_accessor   :ruler
-    
+
     RULES = [
-      [ 'normalize',      lambda { |state| RulesCore::Normalize.inline(state) }         ],
+      [ 'normalize',      lambda { |state| RulesCore::Normalize.normalize(state) }         ],
       [ 'block',          lambda { |state| RulesCore::Block.block(state) }              ],
       [ 'inline',         lambda { |state| RulesCore::Inline.inline(state) }            ],
       [ 'linkify',        lambda { |state| RulesCore::Linkify.linkify(state) }          ],
@@ -43,4 +43,4 @@ def process(state)
       end
     end
   end
-end
+end

lib/motion-markdown-it/ruler.rb

@@ -16,7 +16,7 @@
 #------------------------------------------------------------------------------
 
 module MarkdownIt
-  class Ruler 
+  class Ruler
 
     def initialize
       # // List of added rules. Each element is:
@@ -94,7 +94,7 @@ def __compile__
     # *
     # * ##### Example
     # *
-    # * Replace existing typorgapher replacement rule with new one:
+    # * Replace existing typographer replacement rule with new one:
     # *
     # * ```javascript
     # * var md = require('markdown-it')();
@@ -108,7 +108,7 @@ def at(name, fn, opt = {})
       index = __find__(name)
 
       raise(StandardError, "Parser rule not found: #{name}") if index == -1
-      
+
       @__rules__[index][:fn] = fn
       @__rules__[index][:alt] = opt[:alt] || ['']
       @__cache__ = nil

lib/motion-markdown-it/rules_block/fence.rb

@@ -32,7 +32,9 @@ def self.fence(state, startLine, endLine, silent)
         markup = state.src.slice(mem...pos)
         params = state.src.slice(pos...max)
 
-        return false if params.include?(fromCharCode(marker))
+        if (marker == 0x60) # `
+          return false if params.include?(fromCharCode(marker))
+        end
 
         # Since start is found, we can report success here in validation mode
         return true if silent

lib/motion-markdown-it/rules_block/list.rb

@@ -109,6 +109,18 @@ def self.list(state, startLine, endLine, silent)
         # if it's indented more than 3 spaces, it should be a code block
         return false if (state.sCount[startLine] - state.blkIndent >= 4)
 
+        # Special case:
+        #  - item 1
+        #   - item 2
+        #    - item 3
+        #     - item 4
+        #      - this one is a paragraph continuation
+        if (state.listIndent >= 0 &&
+            state.sCount[startLine] - state.listIndent >= 4 &&
+            state.sCount[startLine] < state.blkIndent)
+          return false
+        end
+
         # limit conditions when list can interrupt
         # a paragraph (validation mode only)
         if silent && state.parentType == 'paragraph'
@@ -220,11 +232,19 @@ def self.list(state, startLine, endLine, silent)
           token.markup = markerCharCode.chr
           token.map    = itemLines = [ startLine, 0 ]
 
-          oldIndent               = state.blkIndent
+          # change current state, then restore it after parser subcall
           oldTight                = state.tight
           oldTShift               = state.tShift[startLine]
-          oldLIndent              = state.sCount[startLine]
+          oldSCount               = state.sCount[startLine]
+
+          #  - example list
+          # ^ listIndent position will be here
+          #   ^ blkIndent position will be here
+          #
+          oldListIndent           = state.listIndent
+          state.listIndent        = state.blkIndent
           state.blkIndent         = indent
+
           state.tight             = true
           state.tShift[startLine] = contentStart - state.bMarks[startLine]
           state.sCount[startLine] = offset
@@ -250,9 +270,10 @@ def self.list(state, startLine, endLine, silent)
           # but we should filter last element, because it means list finish
           prevEmptyEnd = (state.line - startLine) > 1 && state.isEmpty(state.line - 1)
 
-          state.blkIndent         = oldIndent
+          state.blkIndent         = state.listIndent
+          state.listIndent        = oldListIndent
           state.tShift[startLine] = oldTShift
-          state.sCount[startLine] = oldLIndent
+          state.sCount[startLine] = oldSCount
           state.tight             = oldTight
 
           token                   = state.push('list_item_close', 'li', -1)
@@ -269,6 +290,9 @@ def self.list(state, startLine, endLine, silent)
           #
           break if (state.sCount[nextLine] < state.blkIndent)
 
+          # if it's indented more than 3 spaces, it should be a code block
+          break if (state.sCount[startLine] - state.blkIndent >= 4)
+
           # fail if terminating block found
           terminate = false
           (0...terminatorRules.length).each do |i|