Upgrade pipeline with env COMMONMARK=1 npm start to see new, otherw…

…ise parse current (github#20508)

* Update the trim nightmare

* Update create-processor.js

* Update other packages in the rendering pipeline

* A few more updates

* Fix tables

* Update lint-files.js

* Fix copy code blocks

* Update render-content.js

* remove whitespace from liquid conditionals

* We no longer need require eslint rules

* Neat, it worked

* Revert test change

* Update create-processor.js

* Without aliases

Co-authored-by: Chiedo John <2156688+chiedo@users.noreply.github.com>
Co-authored-by: Rachael Sewell <rachmari@github.com>

.eslintrc.js

@@ -15,8 +15,6 @@ module.exports = {
   },
   rules: {
     'import/no-extraneous-dependencies': ['error', { packageDir: '.' }],
-    'node/global-require': ['error'],
-    'import/no-dynamic-require': ['error'],
   },
   overrides: [
     {

includes/rest_operation.html

@@ -7,9 +7,9 @@ <h3 id="{{operation.slug}}" class="pt-3">
   </div>
   <pre><code><span class="color-bg-info-inverse color-text-inverse rounded-1 px-2 py-1" style="text-transform: uppercase">{{operation.verb}}</span> {{operation.requestPath}}</code></pre>
   <div>
-    {% if operation.parameters.size > 0 or operation.bodyParameters.size > 0 %}
+    {%- if operation.parameters.size > 0 or operation.bodyParameters.size > 0 -%}
       <h4 id="{{operation.slug}}--parameters">
-        <a href="#{{operation.slug}}--parameters">{% data ui.products.rest.reference.parameters %}</a>
+        <a href="#{{operation.slug}}--parameters">{%- data ui.products.rest.reference.parameters -%}</a>
       </h4>
       <table>
         <thead>
@@ -31,121 +31,121 @@ <h4 id="{{operation.slug}}--parameters">
             <td class="opacity-70">header</td>
             <td class="opacity-70">
               <p>
-                {% if operation.hasRequiredPreviews %}
+                {%- if operation.hasRequiredPreviews -%}
                   This API is under preview and subject to change.
-                {% else %}
+                {%- else -%}
                   Setting to <code>application/vnd.github.v3+json</code> is recommended.
-                {% endif %}
-                {% if operation.x-github.previews.size > 0 %}
+                {%- endif -%}
+                {%- if operation.x-github.previews.size > 0 -%}
                   <a href="#{{operation.slug}}-preview-notices">
-                    {% if operation.x-github.previews.size > 1 %}
-                      {% data ui.products.rest.reference.see_preview_notices %}
-                    {% else %}
-                      {% data ui.products.rest.reference.see_preview_notice %}
-                    {% endif %}
+                    {%- if operation.x-github.previews.size > 1 -%}
+                      {%- data ui.products.rest.reference.see_preview_notices -%}
+                    {%- else -%}
+                      {%- data ui.products.rest.reference.see_preview_notice -%}
+                    {%- endif -%}
                   </a>
-                {% endif %}
+                {%- endif -%}
               </p>
             </td>
           </tr>
-          {% for param in operation.parameters %}
+          {%- for param in operation.parameters -%}
             <tr>
               <td><code>{{ param.name }}</code></td>
               <td class="opacity-70">{{ param.schema.type }}</td>
               <td class="opacity-70">{{ param.in }}</td>
               <td class="opacity-70">
                 {{ param.descriptionHTML }}
-                {% if param.schema.default != nil %}
+                {%- if param.schema.default != nil -%}
                   Default: <code>{{ param.schema.default }}</code>
-                {% endif %}
+                {%- endif -%}
               </td>
             </tr>
-          {% endfor %}
-          {% for bodyParam in operation.bodyParameters %}
+          {%- endfor -%}
+          {%- for bodyParam in operation.bodyParameters -%}
           <tr>
             <td><code>{{ bodyParam.name }}</code></td>
             <td class="opacity-70">{{ bodyParam.type }}</td>
             <td class="opacity-70">{{ bodyParam.in }}</td>
             <td class="opacity-70">
               {{ bodyParam.description }}
-              {% if bodyParam.default != nil %}
+              {%- if bodyParam.default != nil -%}
               Default: <code>{{ bodyParam.default }}</code>
-              {% endif %}
+              {%- endif -%}
             </td>
           </tr>
-          {% if bodyParam.childParamsGroups.size > 0 %}
+          {%- if bodyParam.childParamsGroups.size > 0 -%}
           <tr style="border: none;">
             <td colspan="4" class="has-nested-table">
-            {% for childParamsGroup in bodyParam.childParamsGroups %}
+            {%- for childParamsGroup in bodyParam.childParamsGroups -%}
             <details>
             <summary class="color-text-secondary"><h4 id="{{ childParamsGroup.id }}">Properties of the <code>{{ childParamsGroup.parentName }}</code> {{ childParamsGroup.parentType }}</h4></summary>
             <table>
               <tbody>
-                {% for childParam in childParamsGroup.params %}
+                {%- for childParam in childParamsGroup.params -%}
                   <tr>
                     <td class="opacity-60"><code>{{ childParam.name }}</code> ({{ childParam.type }})</td>
                     <td class="opacity-60">{{ childParam.description }}</td>
                   </tr>
-                {% endfor %}
+                {%- endfor -%}
               </tbody>
             </table>
             </details>
-            {% endfor %}
+            {%- endfor -%}
             </td>
           </tr>
-          {% endif %}
-          {% endfor %}
+          {%- endif -%}
+          {%- endfor -%}
         </tbody>
       </table>
-    {% endif %}
-    {% if operation.x-codeSamples.size > 0 %}
+    {%- endif -%}
+    {%- if operation.x-codeSamples.size > 0 -%}
       <h4 id="{{operation.slug}}--code-samples">
-        <a href="#{{operation.slug}}--code-samples">{% data ui.products.rest.reference.code_samples %}</a>
+        <a href="#{{operation.slug}}--code-samples">{%- data ui.products.rest.reference.code_samples -%}</a>
       </h4>
-      {% for sample in operation.x-codeSamples %}
-        {% unless sample.lang == "Ruby" %}
+      {%- for sample in operation.x-codeSamples -%}
+        {%- unless sample.lang == "Ruby" -%}
           <h5>
-            {% if sample.lang == "JavaScript" %}
+            {%- if sample.lang == "JavaScript" -%}
               {{sample.lang}} (<a href="https://github.com/octokit/core.js#readme">@octokit/core.js</a>)
-            {% else %}
+            {%- else -%}
               {{sample.lang}}
-            {% endif %}
+            {%- endif -%}
           </h5>
           {{sample.html}}
-        {% endunless %}
-      {% endfor %}
-    {% endif %}
-    {% for response in operation.responses %}
+        {%- endunless -%}
+      {%- endfor -%}
+    {%- endif -%}
+    {%- for response in operation.responses -%}
       <h4>{{ response.description }}</h4>
       <pre><code>Status: {{ response.httpStatusCode }} {{ response.httpStatusMessage }}</code></pre>
       <div class="height-constrained-code-block">{{ response.payload }}</div>
-    {% endfor %}
-    {% if operation.notes.size > 0 or operation.x-github.enabledForGitHubApps %}
+    {%- endfor -%}
+    {%- if operation.notes.size > 0 or operation.x-github.enabledForGitHubApps -%}
       <h4>Notes</h4>
       <ul class="mt-2">
-        {% if operation.x-github.enabledForGitHubApps %}
+        {%- if operation.x-github.enabledForGitHubApps -%}
           <li><a href="{{ restGitHubAppsLink }}">Works with GitHub Apps</a></li>
-        {% endif %}
-        {% for note in operation.notes %}
+        {%- endif -%}
+        {%- for note in operation.notes -%}
           <li>{{ note }}</li>
-        {% endfor %}
+        {%- endfor -%}
       </ul>
-    {% endif %}
-    {% if operation.x-github.previews.size > 0 %}
+    {%- endif -%}
+    {%- if operation.x-github.previews.size > 0 -%}
       <h4 id="{{operation.slug}}-preview-notices">
-        {% if operation.x-github.previews.size > 1 %}
-          {% data ui.products.rest.reference.preview_notices %}
-        {% else %}
-          {% data ui.products.rest.reference.preview_notice %}
-        {% endif %}
+        {%- if operation.x-github.previews.size > 1 -%}
+          {%- data ui.products.rest.reference.preview_notices -%}
+        {%- else -%}
+          {%- data ui.products.rest.reference.preview_notice -%}
+        {%- endif -%}
       </h4>
-      {% for preview in operation.x-github.previews %}
+      {%- for preview in operation.x-github.previews -%}
         <div class="extended-markdown note border rounded-1 mb-4 p-3 color-border-info color-bg-info f5">
           {{ preview.html }}
-          {% if preview.required %}☝️ {% data ui.products.rest.reference.preview_header_is_required %}.{% endif %}
+          {%- if preview.required -%}☝️ {%- data ui.products.rest.reference.preview_header_is_required -%}.{%- endif -%}
         </div>
-      {% endfor %}
-    {% endif %}
+      {%- endfor -%}
+    {%- endif -%}
   </div>
   <hr>
 </div>

lib/get-english-headings.js

@@ -1,6 +1,6 @@
-import astFromMarkdown from 'mdast-util-from-markdown'
-import toString from 'mdast-util-to-string'
-import visit from 'unist-util-visit'
+import { fromMarkdown } from 'mdast-util-from-markdown'
+import { toString } from 'mdast-util-to-string'
+import { visit } from 'unist-util-visit'
 import findPage from './find-page.js'
 
 // for any translated page, first get corresponding English markdown
@@ -41,7 +41,7 @@ export default function getEnglishHeadings(page, context) {
 }
 
 function getHeadings(markdown) {
-  const ast = astFromMarkdown(markdown)
+  const ast = fromMarkdown(markdown)
   const headings = []
 
   visit(ast, (node) => {

lib/render-content/create-processor.js

@@ -1,5 +1,7 @@
-import unified from 'unified'
-import markdown from 'remark-parse'
+import { unified } from 'unified'
+import markdown from 'remark-parse-no-trim'
+import markdownNext from 'remark-parse'
+import gfm from 'remark-gfm'
 import emoji from 'remark-gemoji-to-emoji'
 import remark2rehype from 'remark-rehype'
 import raw from 'rehype-raw'
@@ -18,7 +20,8 @@ const graphql = xHighlightjsGraphql.definer
 
 export default function createProcessor(context) {
   return unified()
-    .use(markdown)
+    .use(process.env.COMMONMARK ? markdownNext : markdown)
+    .use(process.env.COMMONMARK ? gfm : null)
     .use(remarkCodeExtra, { transform: codeHeader })
     .use(emoji)
     .use(remark2rehype, { allowDangerousHtml: true })

lib/render-content/plugins/code-header.js

@@ -1,7 +1,7 @@
-import h from 'hastscript'
+import { h } from 'hastscript'
 import octicons from '@primer/octicons'
 import parse5 from 'parse5'
-import fromParse5 from 'hast-util-from-parse5'
+import { fromParse5 } from 'hast-util-from-parse5'
 
 const LANGUAGE_MAP = {
   asp: 'ASP',
@@ -94,8 +94,8 @@ export default function addCodeHeader(node) {
   const language = LANGUAGE_MAP[node.lang] || node.lang || 'Code'
 
   const btnIconHtml = octicons.clippy.toSVG()
-  const btnIconAst = parse5.parse(String(btnIconHtml))
-  const btnIcon = fromParse5(btnIconAst, btnIconHtml)
+  const btnIconAst = parse5.parse(String(btnIconHtml), { sourceCodeLocationInfo: true })
+  const btnIcon = fromParse5(btnIconAst, { file: btnIconHtml })
 
   // Need to create the header using Markdown AST utilities, to fit
   // into the Unified processor ecosystem.

lib/render-content/plugins/rewrite-legacy-asset-paths.js

@@ -1,4 +1,4 @@
-import visit from 'unist-util-visit'
+import { visit } from 'unist-util-visit'
 import fs from 'fs'
 import { legacyAssetVersions } from '../../enterprise-server-releases.js'
 import { allVersions } from '../../all-versions.js'

lib/render-content/plugins/rewrite-local-links.js

@@ -1,5 +1,5 @@
 import path from 'path'
-import visit from 'unist-util-visit'
+import { visit } from 'unist-util-visit'
 import { getPathWithoutLanguage, getVersionStringFromPath } from '../../path-utils.js'
 import { getNewVersionedPath } from '../../old-versions-utils.js'
 import patterns from '../../patterns.js'

lib/render-content/plugins/use-english-headings.js

@@ -1,7 +1,7 @@
 import GithubSlugger from 'github-slugger'
 import xHtmlEntities from 'html-entities'
 import toString from 'hast-util-to-string'
-import visit from 'unist-util-visit'
+import { visit } from 'unist-util-visit'
 const Entities = xHtmlEntities.XmlEntities
 const slugger = new GithubSlugger()
 const entities = new Entities()

lib/render-content/plugins/wrap-in-element.js

@@ -1,6 +1,6 @@
-import visit from 'unist-util-visit'
+import { visit } from 'unist-util-visit'
 import { selectAll } from 'hast-util-select'
-import parseSelector from 'hast-util-parse-selector'
+import { parseSelector } from 'hast-util-parse-selector'
 
 /*
  * Attacher