mdx-v2: feat: add support for remark, rehype and gatsby-remark plugins (#35751)

* feat: add support for remark, rehype and gatsby-remark plugins

* clean up

* refactor: extract our two custom remark into their own files

* fix: pass path to MDX file for remark plugins

* refactor: remove type assertions

* feat: extend plugin option validation

* remove old comment

Author: axe312ger

e2e-tests/mdx/gatsby-config.js

@@ -18,15 +18,18 @@ module.exports = {
       },
     },
     `gatsby-plugin-test-regression1`,
+    `gatsby-plugin-sharp`,
     {
       resolve: `gatsby-plugin-mdx`,
       options: {
-        lessBabel: false,
         extensions: [`.mdx`, `.md`],
         defaultLayouts: {
           default: require.resolve("./src/components/layout.js"),
         },
-        remarkPlugins: [remarkRequireFilePathPlugin],
+        gatsbyRemarkPlugins: [`gatsby-remark-images`],
+        mdxOptions: {
+          remarkPlugins: [remarkRequireFilePathPlugin],
+        },
       },
     },
     !process.env.CI && `gatsby-plugin-webpack-bundle-analyser-v2`,

e2e-tests/mdx/package.json

@@ -8,7 +8,9 @@
     "fs-extra": "^8.1.0",
     "gatsby": "^4.0.0",
     "gatsby-plugin-mdx": "^3.0.0",
+    "gatsby-plugin-sharp": "^4.0.0",
     "gatsby-plugin-webpack-bundle-analyser-v2": "^1.1.27",
+    "gatsby-remark-images": "^6.15.0",
     "gatsby-source-filesystem": "^4.0.0",
     "react": "^17.0.2",
     "react-dom": "^17.0.2",

e2e-tests/mdx/src/pages/gatsby-logo.png

@@ -10,6 +10,16 @@ Let's go MDX!
 const codefence = true
 ```
 
-<Example/>
+<Example />
 
-<Message data-testid="external">Now an external import</Message>
+<Message data-testid="external">Now an external import</Message>
+
+### An image:
+
+![Alt text here](./gatsby-logo.png)
+
+### HTML
+
+<div data-something="important-information">
+  <strong>This is a test</strong>
+</div>

e2e-tests/mdx/src/pages/index.mdx

@@ -49,15 +49,17 @@
     "testEnvironment": "node"
   },
   "dependencies": {
-    "@mdx-js/loader": "^2.1.1",
     "@mdx-js/mdx": "^2.1.1",
     "change-case": "^4.1.2",
+    "deepmerge": "^4.2.2",
     "fs-extra": "^10.1.0",
+    "gatsby-plugin-utils": "^3.9.0-next.2",
     "gray-matter": "^4.0.3",
-    "mdast-util-from-markdown": "^1.2.0",
     "mdast-util-mdx": "^2.0.0",
     "mdast-util-to-markdown": "^1.3.0",
-    "micromark-extension-mdxjs": "^1.0.0"
+    "unified": "^10.1.2",
+    "unist-util-visit": "^4.1.0",
+    "vfile": "^5.3.2"
   },
   "devDependencies": {
     "babel-preset-gatsby-package": "2.16.0-next.0",

packages/gatsby-plugin-mdx/package.json

@@ -0,0 +1,27 @@
+import type { ProcessorOptions } from "@mdx-js/mdx"
+import type { IFileNode, IMdxNode } from "./types"
+
+// Compiles MDX into JS
+// This could be replaced by @mdx-js/mdx if MDX compile would
+// accept custom data passed to the unified pipeline via processor.data()
+export default async function compileMDX(
+  source: string,
+  mdxNode: IMdxNode,
+  fileNode: IFileNode,
+  options: ProcessorOptions
+): Promise<string> {
+  const { createProcessor } = await import(`@mdx-js/mdx`)
+  const { VFile } = await import(`vfile`)
+
+  const processor = createProcessor(options)
+
+  // If we could pass this via MDX loader config, this whole custom loader might be obsolete.
+  processor.data(`mdxNode`, mdxNode)
+
+  const result = await processor.process(
+    // Inject path to original file for remark plugins. See: https://github.com/gatsbyjs/gatsby/issues/26914
+    new VFile({ value: source, path: fileNode.absolutePath })
+  )
+
+  return result.toString()
+}

packages/gatsby-plugin-mdx/src/compile-mdx.ts

@@ -1,41 +1,34 @@
 /* eslint-disable @babel/no-invalid-this */
-import type { Node } from "gatsby"
 import type { LoaderDefinition } from "webpack"
+import type { Options } from "mdast-util-to-markdown"
+import type { NodeMap } from "./types"
+import type { IMdxPluginOptions } from "./plugin-options"
 
 import grayMatter from "gray-matter"
 import { getOptions } from "loader-utils"
-import type { Options } from "mdast-util-to-markdown"
-
-import { defaultOptions, IMdxPluginOptions } from "./plugin-options"
 
-interface IFileNode extends Node {
-  sourceInstanceName: string
+export interface IGatsbyLayoutLoaderOptions {
+  options: IMdxPluginOptions
+  nodeMap: NodeMap
 }
 
-export interface IMdxLoaderOptions {
-  pluginOptions: IMdxPluginOptions
-  fileMap: Map<string, IFileNode>
-}
-
-// Wrap MDX content with Gatsby Layout component and inject components from `@mdx-js/react`
-const gatsbyMdxLoader: LoaderDefinition = async function (source) {
-  const { pluginOptions, fileMap }: IMdxLoaderOptions = getOptions(this)
-
-  const options = defaultOptions(pluginOptions)
+// Wrap MDX content with Gatsby Layout component
+const gatsbyLayoutLoader: LoaderDefinition = async function (source) {
+  const { options, nodeMap }: IGatsbyLayoutLoaderOptions = getOptions(this)
 
-  const fileNode = fileMap.get(this.resourcePath)
+  const res = nodeMap.get(this.resourcePath)
 
-  if (!fileNode) {
+  if (!res) {
     throw new Error(
       `Unable to locate GraphQL File node for ${this.resourcePath}`
     )
   }
 
-  // get the default layout for the file source group, or if it doesn't
-  // exist, the overall default layout
-  const layoutPath =
-    options.defaultLayouts[fileNode.sourceInstanceName] ||
-    options.defaultLayouts[`default`]
+  // Get the default layout for the file source instance group name,
+  // or fall back to the default
+  const layoutPath = res.fileNode.sourceInstanceName
+    ? options.defaultLayouts[res.fileNode.sourceInstanceName]
+    : options.defaultLayouts[`default`]
 
   // No default layout set? Nothing to do here!
   if (!layoutPath) {
@@ -83,4 +76,4 @@ const gatsbyMdxLoader: LoaderDefinition = async function (source) {
   return out
 }
 
-export default gatsbyMdxLoader
+export default gatsbyLayoutLoader

packages/gatsby-plugin-mdx/src/mdx-loader.ts renamed to packages/gatsby-plugin-mdx/src/gatsby-layout-loader.ts

@@ -0,0 +1,31 @@
+/* eslint-disable @babel/no-invalid-this */
+import type { LoaderDefinition } from "webpack"
+import type { ProcessorOptions } from "@mdx-js/mdx"
+import type { NodeMap } from "./types"
+
+import { getOptions } from "loader-utils"
+
+import compileMDX from "./compile-mdx"
+
+export interface IGatsbyMDXLoaderOptions {
+  options: ProcessorOptions
+  nodeMap: NodeMap
+}
+
+// Custom MDX Loader that injects the GraphQL MDX node into plugin data
+// This whole loaded could be replaced by @mdx-js/loader if MDX would
+// accept custom data passed to the unified pipeline via processor.data()
+const gatsbyMDXLoader: LoaderDefinition = async function (source) {
+  const { options, nodeMap }: IGatsbyMDXLoaderOptions = getOptions(this)
+  const res = nodeMap.get(this.resourcePath)
+
+  if (!res) {
+    return source
+  }
+
+  const { mdxNode, fileNode } = res
+
+  return compileMDX(source, mdxNode, fileNode, options)
+}
+
+export default gatsbyMDXLoader