Skip to content

Commit

Permalink
Markdown: Improved URL tokenization (#2678)
Browse files Browse the repository at this point in the history 
It now tokenizes all parts of a URL except for braces.
  • Loading branch information
@RunDevelopment
RunDevelopment authored Dec 22, 2020
1 parent df0738e commit 2af3e2c
Show file tree
Hide file tree
Showing 6 changed files with 123 additions and 183 deletions.
20 changes: 13 additions & 7 deletions components/prism-markdown.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -221,21 +221,27 @@
// [example](http://example.com "Optional title")
// [example][id]
// [example] [id]
pattern: createInline(/!?\[(?:(?!\])<inner>)+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)| ?\[(?:(?!\])<inner>)+\])/.source),
pattern: createInline(/!?\[(?:(?!\])<inner>)+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)|[ \t]?\[(?:(?!\])<inner>)+\])/.source),
lookbehind: true,
greedy: true,
inside: {
'variable': {
pattern: /(\[)[^\]]+(?=\]$)/,
lookbehind: true
},
'operator': /^!/,
'content': {
pattern: /(^!?\[)[^\]]+(?=\])/,
pattern: /(^\[)[^\]]+(?=\])/,
lookbehind: true,
inside: {} // see below
},
'variable': {
pattern: /(^\][ \t]?\[)[^\]]+(?=\]$)/,
lookbehind: true
},
'url': {
pattern: /(^\]\()[^\s)]+/,
lookbehind: true
},
'string': {
pattern: /"(?:\\.|[^"\\])*"(?=\)$)/
pattern: /(^[ \t]+)"(?:\\.|[^"\\])*"(?=\)$)/,
lookbehind: true
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion components/prism-markdown.min.js
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

67 changes: 23 additions & 44 deletions tests/languages/markdown/bold_feature.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,41 +24,32 @@ not__bold__ __this__either
[
["bold", [
["punctuation", "**"],
["content", [
"foobar"
]],
["content", ["foobar"]],
["punctuation", "**"]
]],
["bold", [
["punctuation", "**"],
["content", [
"foo\r\nbar"
]],
["content", ["foo\r\nbar"]],
["punctuation", "**"]
]],
["bold", [
["punctuation", "__"],
["content", [
"foobar"
]],
["content", ["foobar"]],
["punctuation", "__"]
]],
["bold", [
["punctuation", "__"],
["content", [
"foo\r\nbar"
]],
["content", ["foo\r\nbar"]],
["punctuation", "__"]
]],

["bold", [
["punctuation", "__"],
["content", [
"foo ",
["italic", [
["punctuation", "*"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "*"]
]],
" baz"
Expand All @@ -71,9 +62,7 @@ not__bold__ __this__either
"foo ",
["italic", [
["punctuation", "_"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "_"]
]],
" baz"
Expand All @@ -86,9 +75,7 @@ not__bold__ __this__either
"foo ",
["strike", [
["punctuation", "~"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "~"]
]],
" baz"
Expand All @@ -101,9 +88,7 @@ not__bold__ __this__either
"foo ",
["strike", [
["punctuation", "~~"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "~~"]
]],
" baz"
Expand All @@ -116,23 +101,22 @@ not__bold__ __this__either
"foo",
["url", [
"[",
["content", [
"bar"
]],
"](baz)"
["content", ["bar"]],
"](",
["url", "baz"],
")"
]]
]],
["punctuation", "__"]
]],

["bold", [
["punctuation", "**"],
["content", [
"foo ",
["italic", [
["punctuation", "*"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "*"]
]],
" baz"
Expand All @@ -145,9 +129,7 @@ not__bold__ __this__either
"foo ",
["italic", [
["punctuation", "_"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "_"]
]],
" baz"
Expand All @@ -160,9 +142,7 @@ not__bold__ __this__either
"foo ",
["strike", [
["punctuation", "~"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "~"]
]],
" baz"
Expand All @@ -175,9 +155,7 @@ not__bold__ __this__either
"foo ",
["strike", [
["punctuation", "~~"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "~~"]
]],
" baz"
Expand All @@ -190,14 +168,15 @@ not__bold__ __this__either
"foo",
["url", [
"[",
["content", [
"bar"
]],
"](baz)"
["content", ["bar"]],
"](",
["url", "baz"],
")"
]]
]],
["punctuation", "**"]
]],

"\r\n\r\nnot__bold__ __this__either"
]

Expand Down
67 changes: 23 additions & 44 deletions tests/languages/markdown/italic_feature.test
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,41 +24,32 @@ not_italic_ _this_either
[
["italic", [
["punctuation", "*"],
["content", [
"foobar"
]],
["content", ["foobar"]],
["punctuation", "*"]
]],
["italic", [
["punctuation", "*"],
["content", [
"foo\r\nbar"
]],
["content", ["foo\r\nbar"]],
["punctuation", "*"]
]],
["italic", [
["punctuation", "_"],
["content", [
"foobar"
]],
["content", ["foobar"]],
["punctuation", "_"]
]],
["italic", [
["punctuation", "_"],
["content", [
"foo\r\nbar"
]],
["content", ["foo\r\nbar"]],
["punctuation", "_"]
]],

["italic", [
["punctuation", "_"],
["content", [
"foo ",
["bold", [
["punctuation", "__"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "__"]
]],
" baz"
Expand All @@ -71,9 +62,7 @@ not_italic_ _this_either
"foo ",
["bold", [
["punctuation", "**"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "**"]
]],
" baz"
Expand All @@ -86,9 +75,7 @@ not_italic_ _this_either
"foo ",
["strike", [
["punctuation", "~"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "~"]
]],
" baz"
Expand All @@ -101,9 +88,7 @@ not_italic_ _this_either
"foo ",
["strike", [
["punctuation", "~~"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "~~"]
]],
" baz"
Expand All @@ -116,23 +101,22 @@ not_italic_ _this_either
"foo",
["url", [
"[",
["content", [
"bar"
]],
"](baz)"
["content", ["bar"]],
"](",
["url", "baz"],
")"
]]
]],
["punctuation", "_"]
]],

["italic", [
["punctuation", "*"],
["content", [
"foo ",
["bold", [
["punctuation", "__"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "__"]
]],
" baz"
Expand All @@ -145,9 +129,7 @@ not_italic_ _this_either
"foo ",
["bold", [
["punctuation", "**"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "**"]
]],
" baz"
Expand All @@ -160,9 +142,7 @@ not_italic_ _this_either
"foo ",
["strike", [
["punctuation", "~"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "~"]
]],
" baz"
Expand All @@ -175,9 +155,7 @@ not_italic_ _this_either
"foo ",
["strike", [
["punctuation", "~~"],
["content", [
"bar"
]],
["content", ["bar"]],
["punctuation", "~~"]
]],
" baz"
Expand All @@ -190,14 +168,15 @@ not_italic_ _this_either
"foo",
["url", [
"[",
["content", [
"bar"
]],
"](baz)"
["content", ["bar"]],
"](",
["url", "baz"],
")"
]]
]],
["punctuation", "*"]
]],

"\r\n\r\nnot_italic_ _this_either"
]

Expand Down
Loading

0 comments on commit 2af3e2c

Please sign in to comment.