Support for single language code tabs (#75)

* Improved custom code renderer to support default markdown fenced code block with explicitly specified language.

* Bump version to 0.4.6

* Fix tests and address comments

* Added exhausting comment commenting apifyMarked

Author: mhamas

CHANGELOG.md

@@ -1,4 +1,9 @@
+0.4.6 / 2020/09/29
+==================
+- Improved custom code renderer to support default markdown fenced code block with explicitly specified language.
+
 0.4.5 / 2020/09/09
+==================
 - Fixed custom code renderer to support the use case of rendering code tabs.
 
 0.4.4 / 2020/09/01

package.json

@@ -1,6 +1,6 @@
 {
   "name": "apify-shared",
-  "version": "0.4.5",
+  "version": "0.4.6",
   "description": "Tools and constants shared across Apify projects.",
   "main": "build/index.js",
   "keywords": [
@@ -27,6 +27,7 @@
   "scripts": {
     "build": "rm -rf ./build && babel src --out-dir build && cp src/*.json build",
     "build-doc": "npm run clean && npm run build && node ./node_modules/jsdoc/jsdoc.js --package ./package.json -c ./jsdoc/conf.json -d docs",
+    "build-local-dev": "npm run build && cp package.json build && pushd build/ && npm i && popd",
     "test": "npm run build && mocha --timeout 5000 --require @babel/register --recursive",
     "test-cov": "npm run build && babel-node node_modules/isparta/bin/isparta cover --report text --report html node_modules/mocha/bin/_mocha -- --reporter dot",
     "prepublishOnly": "test $RUNNING_FROM_SCRIPT || (echo \"You must use publish.sh instead of 'npm publish' directly!\"; exit 1)",

src/marked.js

@@ -3,6 +3,31 @@ import matchAll from 'match-all';
 import { customHeadingRenderer } from './markdown_renderers';
 
 
+/**
+ * Map from the language of a fenced code block to the title of corresponding tab.
+ * The language is a string provided by the default marked tokenizer.
+ * Note that not all of the languages (such as python2) might be possible at the moment
+ * in the default marked tokenizer. We anyway include them here for
+ * robustness to potential future improvements of marked.
+ * In case tab title can't be resolved from language using this mapping, the language itself is used as a tab title.
+ */
+const LANGUAGE_TO_TAB_TITLE = {
+    js: 'Node.JS',
+    javascript: 'Node.js',
+    nodejs: 'Node.js',
+    bash: 'Bash',
+    curl: 'cURL',
+    dockerfile: 'Dockerfile',
+    php: 'PHP',
+    json: 'JSON',
+    xml: 'XML',
+    python: 'Python',
+    python2: 'Python 2',
+    python3: 'Python 3',
+    yml: 'YAML',
+    yaml: 'YAML',
+};
+
 const APIFY_CODE_TABS = 'apify-code-tabs';
 const DEFAULT_MARKED_RENDERER = new marked.Renderer();
 
@@ -29,39 +54,106 @@ const codeTabObjectFromCodeTabMarkdown = (markdown) => {
 
 
 /**
+ * This custom function is used in the same context as default `marked` function.
+ *
+ * It parses the given markdown and treats some headings and code blocks in a custom way
+ * -----------------------------------------------------------------------------------------------
+ * 0. Heading with {custom-id} in text will have id="custom-id" property on reasulting <h...> tag.
+ * E.g.
+ * # Welcome to Apify {welcome-title-id}
+ * is turned to
+ * <h1 id="welcome-title-id">Welcome to Apify</h1>
+ * -----------------------------------------------------------------------------------------------
+ * 1. Fenced code block with explicit language which is in the mapping LANGUAGE_TO_TAB_TITLE
+ * ```my-lang
+ * my-code
+ * ```
+ *  This block is turned into [apify-code-tabs]$INDEX[/apify-code-tabs] in returned HTML
+ *  and returned codeTabsObjectPerIndex contains key $INDEX with value
+ *  {
+ *      LANGUAGE_TO_TAB_TITLE[my-lang]: { lang: 'my-lang', code: 'my-code' }
+ *  }
+ * -----------------------------------------------------------------------------------------------
+ * 2. Fenced code block with explicit language which is NOT in the mapping LANGUAGE_TO_TAB_TITLE
+ * ```my-lang-not-in-mapping
+ * my-code
+ * ```
+ *  This block is turned into [apify-code-tabs]$INDEX[/apify-code-tabs] in returned HTML
+ *  and returned codeTabsObjectPerIndex contains key $INDEX with value
+ *  {
+ *      my-lang-not-in-mapping: { lang: 'my-lang-not-in-mapping', code: 'my-code' }
+ *  }
+ * -----------------------------------------------------------------------------------------------
+ * 3. Fenced code block with no language
+ * ```
+ * my-code
+ * ```
+ *
+ * is handled by default marked package and returned in HTML already parsed to <code> block.
+ * -----------------------------------------------------------------------------------------------
+ * 4. Indented code block
+ *      my-code
+ *
+ * is handled by default marked package and returned in HTML already parsed to <code> block.
+ * -----------------------------------------------------------------------------------------------
+ * 5. Special marked-tabs code fence
+ * Each code block of following form
+ * ```marked-tabs
+ * <marked-tab header="Node.js" lang="javascript">
+ * js-code
+ * </marked-tab>
+ *
+ * <marked-tab header="Python" lang="python">
+ * python-code
+ * </marked-tab>
+ * ```
+ * is replaced by [apify-code-tabs]$INDEX[/apify-code-tabs] in the returned HTML where $INDEX is
+ * an unique integer, to allow multiple marked-tabs components on the same page.
+ *
+ * For the example above codeTabsObjectPerIndex would contain key $INDEX with the following value
+ * {
+ *      'Node.js': {lang: 'javascript', code: 'js-code'},
+ *      'Python': {lang: 'python', code: 'python-code'}
+ * }
+ *
+ * i.e. each <marked-tab header="HEADER" lang="LANG">CODE</marked-tab> is turned into
+ * HEADER: {lang: LANG, code: CODE} entry.
+ *
+ * Note that you have to use double quotation marks around HEADER and LANG, otherwise, the expression will not be matched
+ * which results in unexpected and hard to debug errors.
+ *
+ * Each [apify-code-tabs]$INDEX[/apify-code-tabs] is meant to be later replaced be a react component
+ * rendering the appropriate codeTabBlockObject returned by this function.
  * @param {string} markdown
  * @return {{ html: string, codeTabsObjectPerIndex: Object.<number, Object.<string, {language: string, code: string}>> }}
  */
 export const apifyMarked = (markdown) => {
     const renderer = new marked.Renderer();
     renderer.heading = customHeadingRenderer;
     renderer.code = (code, language) => {
-        if (language === 'marked-tabs') {
+        if (language) {
             return code;
         }
         return DEFAULT_MARKED_RENDERER.code(code, language);
     };
     const tokens = marked.lexer(markdown);
 
-    /**
-     * Each code block of following form
-     * ```marked-tabs
-     * ... some code
-     * ```
-     * is replaced by [apify-code-tabs]INDEX[/apify-code-tabs] where index is
-     * an increasing integer starting at 0, to allow multiple marked-tabs components
-     * on the same page.
-     *
-     * [apify-code-tabs]INDEX[/apify-code-tabs] is meant to be later replaced be a react component
-     * rendering the appropriate codeTabBlockObject returned by this function.
-     */
     let markedTabTokenIndex = 0;
     const codeTabsObjectPerIndex = {};
     tokens.forEach((token) => {
-        if (token.type === 'code' && token.lang === 'marked-tabs') {
-            codeTabsObjectPerIndex[markedTabTokenIndex] = codeTabObjectFromCodeTabMarkdown(token.text);
+        if (token.type === 'code' && token.lang) {
+            if (token.lang === 'marked-tabs') {
+                codeTabsObjectPerIndex[markedTabTokenIndex] = codeTabObjectFromCodeTabMarkdown(token.text);
+            } else {
+                const tabTitle = LANGUAGE_TO_TAB_TITLE[token.lang] || token.lang;
+                codeTabsObjectPerIndex[markedTabTokenIndex] = {
+                    [tabTitle]: {
+                        language: token.lang,
+                        code: token.text,
+                    },
+                };
+            }
             token.text = `[${APIFY_CODE_TABS}]${markedTabTokenIndex}[/${APIFY_CODE_TABS}]`;
-
             markedTabTokenIndex++;
         }
     });

test/marked.js

@@ -10,7 +10,7 @@ const MARKDOWN_UNDER_TEST = `
 
 ## Code block with tabs
 \`\`\`marked-tabs
-<marked-tab header="NodeJS" lang="javascript">
+<marked-tab header="Node.js" lang="javascript">
 console.log('Some JS code');
 </marked-tab>
 
@@ -22,7 +22,7 @@ if count >= 1:
 print('Some python code on next line');
 </marked-tab>
 
-<marked-tab header="Curl" lang="bash">
+<marked-tab header="Bash" lang="bash">
 echo "Some bash code"
 </marked-tab>
 \`\`\`
@@ -32,12 +32,18 @@ echo "Some bash code"
 console.log('Your standard javascript code block')
 \`\`\`
 
+\`\`\`
+console.log('Fenced block with no language')
+\`\`\`
+
+    console.log('Tab indented block')
+
 ## Second block with tabs
 \`\`\`marked-tabs
-<marked-tab header="NodeJS" lang="javascript">
+<marked-tab header="Custom title" lang="javascript">
 console.log('Some JS code 2');
 </marked-tab>
-<marked-tab header="Curl" lang="bash">
+<marked-tab header="Bash" lang="bash">
 echo "Some bash code 2"
 </marked-tab>
 \`\`\`
@@ -49,9 +55,10 @@ This is footer text.
 const EXPECTED_HTML =   '\n' +
 '            <h1 id="title">Title</h1>\n' +
 '            <h2 id="code-block-with-tabs">Code block with tabs</h2>[apify-code-tabs]0[/apify-code-tabs]\n' +
-'            <h2 id="code-block-without-tabs">Code block without tabs</h2><pre><code class="language-javascript">console.log(&#39;Your standard javascript code block&#39;)</code></pre>\n' +
+'            <h2 id="code-block-without-tabs">Code block without tabs</h2>[apify-code-tabs]1[/apify-code-tabs]<pre><code>console.log(&#39;Fenced block with no language&#39;)</code></pre>\n' +
+'<pre><code>console.log(&#39;Tab indented block&#39;)</code></pre>\n' +
 '\n' +
-'            <h2 id="second-block-with-tabs">Second block with tabs</h2>[apify-code-tabs]1[/apify-code-tabs]\n' +
+'            <h2 id="second-block-with-tabs">Second block with tabs</h2>[apify-code-tabs]2[/apify-code-tabs]\n' +
 '            <h2 id="footer">Footer</h2><p>This is footer text.</p>\n';
 
 describe('apifyMarked custom renderer works', () => {
@@ -62,20 +69,23 @@ describe('apifyMarked custom renderer works', () => {
         expect(codeTabsObjectPerIndex).to.eql(
             {
                 '0': {
-                  NodeJS: { language: 'javascript', code: "console.log('Some JS code');" },
-                  Python: {
-                    language: 'python',
-                    code: "print('Some python code');\n" +
-                      'count = 1\n' +
-                      'if count >= 1:\n' +
-                      "    print('Some intended python code');\n" +
-                      "print('Some python code on next line');"
-                  },
-                  Curl: { language: 'bash', code: 'echo "Some bash code"' }
+                    'Node.js': { language: 'javascript', code: "console.log('Some JS code');" },
+                    Python: {
+                        language: 'python',
+                        code: "print('Some python code');\n" +
+                        'count = 1\n' +
+                        'if count >= 1:\n' +
+                        "    print('Some intended python code');\n" +
+                        "print('Some python code on next line');"
+                    },
+                    'Bash': { language: 'bash', code: 'echo "Some bash code"' }
                 },
                 '1': {
-                  NodeJS: { language: 'javascript', code: "console.log('Some JS code 2');" },
-                  Curl: { language: 'bash', code: 'echo "Some bash code 2"' }
+                    'Node.js': { language: 'javascript', code: "console.log('Your standard javascript code block')" },
+                },
+                '2': {
+                    'Custom title': { language: 'javascript', code: "console.log('Some JS code 2');" },
+                    Bash: { language: 'bash', code: 'echo "Some bash code 2"' }
                 }
             }
         );