remark: add support for meta in code

Previously, the info string of a code block, was not parsed at all, and was stored in `lang`
on `code` nodes.
Now, the first “word” of the info string is stored in `lang`, whereas the rest is stored in
`meta`.

Related to syntax-tree/mdast#22 (PR)
Closes GH-342 (issue)
Closes GH-345 (PR)

Reviewed-by: Titus Wormer <tituswormer@gmail.com> 
Reviewed-by: Christian Murphy <christian.murphy.42@gmail.com> 

Co-authored-by: Titus Wormer <tituswormer@gmail.com>

Author: arobase-che

packages/remark-parse/lib/tokenize/code-fenced.js

@@ -23,6 +23,8 @@ function fencedCode(eat, value, silent) {
   var marker;
   var character;
   var flag;
+  var lang;
+  var meta;
   var queue;
   var content;
   var exdentedContent;
@@ -228,9 +230,26 @@ function fencedCode(eat, value, silent) {
 
   subvalue += content + closing;
 
+  index = -1;
+  length = flag.length;
+
+  while (++index < length) {
+    character = flag.charAt(index);
+
+    if (character === C_SPACE || character === C_TAB) {
+      if (!lang) {
+        lang = flag.slice(0, index);
+      }
+    } else if (lang) {
+      meta = flag.slice(index);
+      break;
+    }
+  }
+
   return eat(subvalue)({
     type: 'code',
-    lang: flag || null,
+    lang: lang || flag || null,
+    meta: meta || null,
     value: trim(exdentedContent)
   });
 }

packages/remark-parse/lib/tokenize/code-indented.js

@@ -92,6 +92,7 @@ function indentedCode(eat, value, silent) {
     return eat(subvalue)({
       type: 'code',
       lang: null,
+      meta: null,
       value: trim(content)
     });
   }

packages/remark-stringify/lib/visitors/code.js

@@ -38,11 +38,17 @@ function code(node, parent) {
   var value = node.value;
   var options = self.options;
   var marker = options.fence;
-  var language = self.encode(node.lang || '', node);
+  var info = node.lang || '';
   var fence;
 
+  if (info && node.meta) {
+    info += ' ' + node.meta;
+  }
+
+  info = self.encode(info, node);
+
   /* Without (needed) fences. */
-  if (!language && !options.fences && value) {
+  if (!info && !options.fences && value) {
     /* Throw when pedantic, in a list item which
      * isn’t compiled using a tab. */
     if (
@@ -59,5 +65,5 @@ function code(node, parent) {
 
   fence = repeat(marker, Math.max(streak(value, marker) + 1, 3));
 
-  return fence + language + '\n' + value + '\n' + fence;
+  return fence + info + '\n' + value + '\n' + fence;
 }

test/fixtures/input/fenced-code-info-string.text

@@ -0,0 +1,34 @@
+Normal with language tag:
+
+```js this is an info string
+```
+
+With white space:
+
+``` bash info
+```
+
+With curly braces:
+
+``````lang{info}
+``````
+
+With nothing:
+
+```
+```
+
+With several streaks of white space:
+
+``` bash info string
+```
+
+With tabs as white space:
+
+``` bash		info
+```
+
+With spaces as white space:
+
+``` bash  info
+```

test/fixtures/tree/auto-link.json

@@ -694,6 +694,7 @@
     {
       "type": "code",
       "lang": null,
+      "meta": null,
       "value": "or here: <http://example.com/>",
       "position": {
         "start": {

test/fixtures/tree/backslash-escapes.commonmark.json

@@ -1916,6 +1916,7 @@
     {
       "type": "code",
       "lang": null,
+      "meta": null,
       "value": "Backslash: \\\\\n\nBacktick: \\`\n\nAsterisk: \\*\n\nUnderscore: \\_\n\nLeft brace: \\{\n\nRight brace: \\}\n\nLeft bracket: \\[\n\nRight bracket: \\]\n\nLeft paren: \\(\n\nRight paren: \\)\n\nGreater-than: \\>\n\nHash: \\#\n\nPeriod: \\.\n\nBang: \\!\n\nPlus: \\+\n\nMinus: \\-",
       "position": {
         "start": {
@@ -2018,6 +2019,7 @@
     {
       "type": "code",
       "lang": null,
+      "meta": null,
       "value": "Pipe: \\|\n\nTilde: \\~",
       "position": {
         "start": {
@@ -2092,6 +2094,7 @@
     {
       "type": "code",
       "lang": null,
+      "meta": null,
       "value": "Quote: \\\"\n\nDollar: \\$\n\nPercentage: \\%\n\nAmpersand: \\&\n\nSingle quote: \\'\n\nComma: \\,\n\nForward slash: \\/\n\nColon: \\:\n\nSemicolon: \\;\n\nLess-than: \\<\n\nEquals: \\=\n\nQuestion mark: \\?\n\nAt-sign: \\@\n\nCaret: \\^\n\nNew line: \\\nonly works in paragraphs.",
       "position": {
         "start": {

test/fixtures/tree/backslash-escapes.json

@@ -1645,6 +1645,7 @@
     {
       "type": "code",
       "lang": null,
+      "meta": null,
       "value": "Backslash: \\\\\n\nBacktick: \\`\n\nAsterisk: \\*\n\nUnderscore: \\_\n\nLeft brace: \\{\n\nRight brace: \\}\n\nLeft bracket: \\[\n\nRight bracket: \\]\n\nLeft paren: \\(\n\nRight paren: \\)\n\nGreater-than: \\>\n\nHash: \\#\n\nPeriod: \\.\n\nBang: \\!\n\nPlus: \\+\n\nMinus: \\-",
       "position": {
         "start": {
@@ -1747,6 +1748,7 @@
     {
       "type": "code",
       "lang": null,
+      "meta": null,
       "value": "Pipe: \\|\n\nTilde: \\~",
       "position": {
         "start": {
@@ -1821,6 +1823,7 @@
     {
       "type": "code",
       "lang": null,
+      "meta": null,
       "value": "Quote: \\\"\n\nDollar: \\$\n\nPercentage: \\%\n\nAmpersand: \\&\n\nSingle quote: \\'\n\nComma: \\,\n\nForward slash: \\/\n\nColon: \\:\n\nSemicolon: \\;\n\nLess-than: \\<\n\nEquals: \\=\n\nQuestion mark: \\?\n\nAt-sign: \\@\n\nCaret: \\^\n\nNew line: \\\nonly works in paragraphs.",
       "position": {
         "start": {

test/fixtures/tree/backslash-escapes.nogfm.json

@@ -1611,6 +1611,7 @@
     {
       "type": "code",
       "lang": null,
+      "meta": null,
       "value": "Backslash: \\\\\n\nBacktick: \\`\n\nAsterisk: \\*\n\nUnderscore: \\_\n\nLeft brace: \\{\n\nRight brace: \\}\n\nLeft bracket: \\[\n\nRight bracket: \\]\n\nLeft paren: \\(\n\nRight paren: \\)\n\nGreater-than: \\>\n\nHash: \\#\n\nPeriod: \\.\n\nBang: \\!\n\nPlus: \\+\n\nMinus: \\-",
       "position": {
         "start": {
@@ -1713,6 +1714,7 @@
     {
       "type": "code",
       "lang": null,
+      "meta": null,
       "value": "Pipe: \\|\n\nTilde: \\~",
       "position": {
         "start": {
@@ -1787,6 +1789,7 @@
     {
       "type": "code",
       "lang": null,
+      "meta": null,
       "value": "Quote: \\\"\n\nDollar: \\$\n\nPercentage: \\%\n\nAmpersand: \\&\n\nSingle quote: \\'\n\nComma: \\,\n\nForward slash: \\/\n\nColon: \\:\n\nSemicolon: \\;\n\nLess-than: \\<\n\nEquals: \\=\n\nQuestion mark: \\?\n\nAt-sign: \\@\n\nCaret: \\^\n\nNew line: \\\nonly works in paragraphs.",
       "position": {
         "start": {

test/fixtures/tree/block-elements.json

@@ -496,6 +496,7 @@
     {
       "type": "code",
       "lang": null,
+      "meta": null,
       "value": "1.   Second sublist.",
       "position": {
         "start": {
@@ -625,6 +626,7 @@
     {
       "type": "code",
       "lang": null,
+      "meta": null,
       "value": "And this is code();",
       "position": {
         "start": {

test/fixtures/tree/blockquote-lazy-code.commonmark.json

@@ -7,6 +7,7 @@
         {
           "type": "code",
           "lang": null,
+          "meta": null,
           "value": "foo",
           "position": {
             "start": {
@@ -40,6 +41,7 @@
     {
       "type": "code",
       "lang": null,
+      "meta": null,
       "value": "bar",
       "position": {
         "start": {