MDXv2: remove default layouts and add support for pageQueries (#35983)

* WIP

* WIP#2

* ensure loader actually picks up mdx file path and prepare query parsing

* avoid issue with false-flag query duplicates

* properly inject compiled MDX as children

* fix: chunk names now should no more exceed max file path length

* fix too long chunk names

* fix: ensure page component validation uses clear path

* fix: properly set clean path on all page component validations

* test: add unit test for chunk name shortening

* fix: improve chunk name shortening

* fix: (re)support MDX pages automatically created out of /pages directory

* chore: add comments to mdx-layout loader to explain what the AST transformations do

* chore: remove @todo comment

* refactor: convert pluginOptions into MdxPluginOptions instead of extending PluginOptions

* refactor: rename __mdxPath to __contentFilePath

* refactor: move page component path splitting logic to gatsby-core-utils

* refactor: shorten chunk name to 60 characters and hash it

* fix indentation

* remove unused prop

* tmp

* feat: add caching to frontmatter extraction

* feat: support page component URIs in page tree rendering

* perf: ensure frontmatter is parsed as yaml only

* build: downgrade acorn to version compatible with acorn-jsx and yarn-deduplicate

* fix import

* use murmurhash

* change splitComponentPath to check for stricter than only ?

* fix tests

* silence non-page component warning

* use parse-component-path helpers

Co-authored-by: LekoArts <lekoarts@gmail.com>

Author: axe312ger

benchmarks/gabe-fs-mdx/gatsby-config.js

@@ -14,11 +14,6 @@ module.exports = {
     },
     {
       resolve: `gatsby-plugin-mdx`,
-      options: {
-        defaultLayouts: {
-          articles: path.resolve(`./src/templates/blog-post.js`),
-        },
-      },
     },
     !process.env.CI && `gatsby-plugin-webpack-bundle-analyser-v2`,
   ].filter(Boolean),

benchmarks/gabe-fs-mdx/gatsby-node.js

@@ -1,3 +1,5 @@
+const path = require("path")
+
 exports.createPages = async ({ graphql, actions }) => {
   const { createPage } = actions
 
@@ -34,7 +36,9 @@ exports.createPages = async ({ graphql, actions }) => {
 
     createPage({
       path: slug,
-      component: parent.absolutePath,
+      component: `${path.resolve(
+        `./src/templates/blog-post.js`
+      )}?__contentFilePath=${parent.absolutePath}`,
       context: {
         id,
         slug,

benchmarks/gabe-fs-mdx/src/templates/blog-post.js

@@ -15,7 +15,7 @@ const Article = ({ children, pageContext }) => {
 }
 
 export const pageQuery = graphql`
-  query MdxQuery($id: String!) {
+  query($id: String!) {
     mdx(id: { eq: $id }) {
       id
       frontmatter {

packages/gatsby-cli/src/reporter/loggers/ink/components/pageTree.tsx

@@ -2,6 +2,7 @@
 import React, { ReactElement, useContext } from "react"
 import { Box, Text, BoxProps, Spacer } from "ink"
 import path from "path"
+import { getPathToLayoutComponent } from "gatsby-core-utils"
 import StoreStateContext from "../context"
 import {
   generatePageTree,
@@ -117,7 +118,8 @@ const ConnectedPageTree: React.FC = function ConnectedPageTree() {
   const componentWithPages = new Map<string, IComponentWithPageModes>()
 
   for (const { componentPath, pages } of state.pageTree!.components.values()) {
-    const pagesByMode = {
+    const layoutComponent = getPathToLayoutComponent(componentPath)
+    const pagesByMode = componentWithPages.get(layoutComponent) || {
       SSG: new Set<string>(),
       DSG: new Set<string>(),
       SSR: new Set<string>(),
@@ -128,8 +130,7 @@ const ConnectedPageTree: React.FC = function ConnectedPageTree() {
 
       pagesByMode[gatsbyPage!.mode].add(pagePath)
     })
-
-    componentWithPages.set(componentPath, pagesByMode)
+    componentWithPages.set(layoutComponent, pagesByMode)
   }
 
   for (const {

packages/gatsby-cli/src/reporter/loggers/yurnalist/index.ts

@@ -17,6 +17,7 @@ import {
   IComponentWithPageModes,
 } from "../../../util/generate-page-tree"
 import { IRenderPageArgs } from "../../types"
+import { getPathToLayoutComponent } from "gatsby-core-utils"
 
 interface IYurnalistActivities {
   [activityId: string]: {
@@ -34,7 +35,9 @@ function generatePageTreeToConsole(
   const root = state.root
   const componentWithPages = new Map<string, IComponentWithPageModes>()
   for (const { componentPath, pages } of state.components.values()) {
-    const pagesByMode = {
+    const layoutComponent = getPathToLayoutComponent(componentPath)
+    const relativePath = path.posix.relative(root, layoutComponent)
+    const pagesByMode = componentWithPages.get(relativePath) || {
       SSG: new Set<string>(),
       DSG: new Set<string>(),
       SSR: new Set<string>(),
@@ -46,10 +49,7 @@ function generatePageTreeToConsole(
       pagesByMode[gatsbyPage!.mode].add(pagePath)
     })
 
-    componentWithPages.set(
-      path.posix.relative(root, componentPath),
-      pagesByMode
-    )
+    componentWithPages.set(relativePath, pagesByMode)
   }
 
   for (const {

packages/gatsby-core-utils/src/index.ts

@@ -15,6 +15,7 @@ export * from "./service-lock"
 export * from "./site-metadata"
 export * from "./page-data"
 export * from "./page-html"
+export * from "./parse-component-path"
 export { listPlugins } from "./list-plugins"
 export { createFilePath } from "./filename-utils"
 export { readConfigFile, getConfigPath } from "./utils"

packages/gatsby-core-utils/src/parse-component-path.ts

@@ -0,0 +1,45 @@
+// Since gatsby-plugin-mdx v4, we are using the resourceQuery feature
+// of webpack's loaders to inject a content file into a page component.
+// These helper functions are used to simplify working with these new
+// page component path URI's.
+
+const CONTENT_FILE_PATH_QUERY = `?__contentFilePath=`
+
+// Split the path component URI without using the expensive URI.parse()
+export const splitComponentPath = (componentPath: string): Array<string> => {
+  // If the path does not include the contentFilePath query, we can assume its a regular path
+  if (!componentPath.includes(CONTENT_FILE_PATH_QUERY)) {
+    return [componentPath]
+  }
+
+  const splitPath = componentPath.split(CONTENT_FILE_PATH_QUERY)
+
+  // We only support URI paths with the `?__contentFilePath=` parameter
+  if (splitPath.length !== 2) {
+    throw new Error(
+      `The following page component must contain '${CONTENT_FILE_PATH_QUERY}':\n${componentPath}`
+    )
+  }
+
+  // Other URI parameters are not supported
+  if (splitPath[1].includes(`&`)) {
+    throw new Error(
+      `You can not pass any other parameters to a page component URI as 'contentFilePath'. Remove the ampersand (&):\n${componentPath}`
+    )
+  }
+  return splitPath
+}
+
+// Get the path to the actual js page component
+export const getPathToLayoutComponent = (componentPath: string): string =>
+  splitComponentPath(componentPath)[0]
+
+// Get the path to the content file, falling back to the js component if no content file is given.
+// Pages directly created from `.mdx` files
+export const getPathToContentComponent = (componentPath: string): string => {
+  const splitPath = splitComponentPath(componentPath)
+  if (splitPath.length === 1) {
+    return splitPath[0]
+  }
+  return splitPath[1]
+}

packages/gatsby-plugin-mdx/package.json

@@ -61,7 +61,11 @@
     "rehype-infer-description-meta": "^1.0.1",
     "unified": "^10.1.2",
     "unist-util-visit": "^4.1.0",
-    "vfile": "^5.3.2"
+    "vfile": "^5.3.2",
+    "acorn": "^7.4.1",
+    "acorn-jsx": "^5.3.2",
+    "astring": "^1.8.3",
+    "estree-util-build-jsx": "^2.1.0"
   },
   "devDependencies": {
     "@babel/cli": "^7.17.10",
@@ -73,6 +77,7 @@
   },
   "peerDependencies": {
     "@mdx-js/react": "^2.0.0",
+    "@types/estree": "^0.0.50",
     "gatsby": "^4.0.0-next",
     "gatsby-source-filesystem": "^4.0.0-next",
     "loader-utils": "^3.2.0",

packages/gatsby-plugin-mdx/src/compile-mdx.ts

@@ -1,5 +1,5 @@
 import deepmerge from "deepmerge"
-import type { NodePluginArgs } from "gatsby"
+import type { NodePluginArgs, PluginOptions } from "gatsby"
 import type { ProcessorOptions } from "@mdx-js/mdx"
 import type { IFileNode, IMdxMetadata, IMdxNode } from "./types"
 
@@ -77,7 +77,7 @@ export const compileMDXWithCustomOptions = async ({
   cache,
   mdxNode,
 }: {
-  pluginOptions: IMdxPluginOptions
+  pluginOptions: PluginOptions
   customOptions: Partial<IMdxPluginOptions>
   getNode: NodePluginArgs["getNode"]
   getNodesByType: NodePluginArgs["getNodesByType"]

packages/gatsby-plugin-mdx/src/frontmatter.ts

@@ -0,0 +1,29 @@
+import grayMatter from "gray-matter"
+
+const cacheMap = new Map()
+
+export function parseFrontmatter(
+  cacheKey: string,
+  source: string
+): { body: string; frontmatter: { [key: string]: unknown } } {
+  if (cacheMap.has(cacheKey)) {
+    return cacheMap.get(cacheKey)
+  }
+
+  const { content, data } = grayMatter(source, {
+    language: `yaml`,
+    // Disable JS(ON) frontmatter parsing.
+    // See: https://github.com/gatsbyjs/gatsby/pull/35830
+    engines: {
+      js: () => {
+        return {}
+      },
+      javascript: () => {
+        return {}
+      },
+    },
+  })
+  cacheMap.set(cacheKey, { content, data })
+
+  return { body: content, frontmatter: data }
+}