fix(gatsby): handle initializing multiple instances of gatsby-plugin-sharp (#37306)

* fix(gatsby): handle initializing multiple instances of gatsby-plugin-sharp

* fix(gatsby): handle initializing multiple instances of gatsby-plugin-sharp in engines

* update standalone-regenerate

* normalize main config when loading themes

* move deduplicaiton to plugin loading instead of themes loading

* convert load-themes to TS

* lint

* update assertions

* remove test that no longer make sense

Author: pieh

packages/gatsby/src/bootstrap/load-config/index.ts

@@ -3,7 +3,7 @@ import telemetry from "gatsby-telemetry"
 import { preferDefault } from "../prefer-default"
 import { getConfigFile } from "../get-config-file"
 import { internalActions } from "../../redux/actions"
-import loadThemes from "../load-themes"
+import { loadThemes } from "../load-themes"
 import { store } from "../../redux"
 import handleFlags from "../../utils/handle-flags"
 import availableFlags from "../../utils/flags"

packages/gatsby/src/bootstrap/load-plugins/__tests__/load-plugins.ts

@@ -218,9 +218,7 @@ describe(`Load plugins`, () => {
         (plugin: { name: string }) => plugin.name === `gatsby-plugin-typescript`
       )
 
-      // TODO: I think we should probably be de-duping, so this should be 1.
-      // But this test is mostly here to ensure we don't add an _additional_ gatsby-plugin-typescript
-      expect(tsplugins.length).toEqual(2)
+      expect(tsplugins.length).toEqual(1)
     })
   })
 
@@ -351,9 +349,7 @@ describe(`Load plugins`, () => {
           plugin.name === `gatsby-plugin-gatsby-cloud`
       )
 
-      // TODO: I think we should probably be de-duping, so this should be 1.
-      // But this test is mostly here to ensure we don't add an _additional_ gatsby-plugin-typescript
-      expect(cloudPlugins.length).toEqual(2)
+      expect(cloudPlugins.length).toEqual(1)
     })
   })
 

packages/gatsby/src/bootstrap/load-plugins/load-internal-plugins.ts

@@ -1,4 +1,5 @@
 import { slash } from "gatsby-core-utils"
+import { uniqWith, isEqual } from "lodash"
 import path from "path"
 import reporter from "gatsby-cli/lib/reporter"
 import { store } from "../../redux"
@@ -170,5 +171,7 @@ export function loadInternalPlugins(
     )
   )
 
-  return plugins
+  const uniquePlugins = uniqWith(plugins, isEqual)
+
+  return uniquePlugins
 }

packages/gatsby/src/bootstrap/load-themes/__tests__/index.js

@@ -1,4 +1,4 @@
-const loadThemes = require(`..`)
+const { loadThemes } = require(`..`)
 const path = require(`path`)
 
 describe(`loadThemes`, () => {

packages/gatsby/src/bootstrap/load-themes/index.js renamed to packages/gatsby/src/bootstrap/load-themes/index.ts

@@ -1,22 +1,39 @@
-const { createRequireFromPath } = require(`gatsby-core-utils`)
-const path = require(`path`)
-import { mergeGatsbyConfig } from "../../utils/merge-gatsby-config"
-const Promise = require(`bluebird`)
-const _ = require(`lodash`)
-const debug = require(`debug`)(`gatsby:load-themes`)
+import { createRequireFromPath } from "gatsby-core-utils"
+import * as path from "path"
+import {
+  IGatsbyConfigInput,
+  mergeGatsbyConfig,
+  PluginEntry,
+  IPluginEntryWithParentDir,
+} from "../../utils/merge-gatsby-config"
+import { mapSeries } from "bluebird"
+import { flattenDeep, isEqual, isFunction, uniqWith } from "lodash"
+import DebugCtor from "debug"
 import { preferDefault } from "../prefer-default"
 import { getConfigFile } from "../get-config-file"
 import { resolvePlugin } from "../load-plugins/resolve-plugin"
-const reporter = require(`gatsby-cli/lib/reporter`)
+import reporter from "gatsby-cli/lib/reporter"
+
+const debug = DebugCtor(`gatsby:load-themes`)
+
+interface IThemeObj {
+  themeName: string
+  themeConfig: IGatsbyConfigInput
+  themeDir: string
+  themeSpec: PluginEntry
+  parentDir: string
+  configFilePath?: string
+}
 
 // get the gatsby-config file for a theme
 const resolveTheme = async (
-  themeSpec,
-  configFileThatDeclaredTheme,
-  isMainConfig = false,
-  rootDir
-) => {
-  const themeName = themeSpec.resolve || themeSpec
+  themeSpec: PluginEntry,
+  configFileThatDeclaredTheme: string | undefined,
+  isMainConfig: boolean = false,
+  rootDir: string
+): Promise<IThemeObj> => {
+  const themeName =
+    typeof themeSpec === `string` ? themeSpec : themeSpec.resolve
   let themeDir
   try {
     const scopedRequire = createRequireFromPath(`${rootDir}/:internal:`)
@@ -59,13 +76,16 @@ const resolveTheme = async (
     themeDir,
     `gatsby-config`
   )
-  const theme = preferDefault(configModule)
+  const theme:
+    | IGatsbyConfigInput
+    | ((options?: Record<string, unknown>) => IGatsbyConfigInput) =
+    preferDefault(configModule)
 
   // if theme is a function, call it with the themeConfig
-  let themeConfig = theme
-  if (_.isFunction(theme)) {
-    themeConfig = theme(themeSpec.options || {})
-  }
+  const themeConfig = isFunction(theme)
+    ? theme(typeof themeSpec === `string` ? {} : themeSpec.options)
+    : theme
+
   return {
     themeName,
     themeConfig,
@@ -84,34 +104,63 @@ const resolveTheme = async (
 // no use case for a loop so I expect that to only happen if someone is very
 // off track and creating their own set of themes
 const processTheme = (
-  { themeName, themeConfig, themeSpec, themeDir, configFilePath },
-  { rootDir }
-) => {
+  { themeName, themeConfig, themeSpec, themeDir, configFilePath }: IThemeObj,
+  { rootDir }: { rootDir: string }
+): Promise<Array<IThemeObj>> => {
   const themesList = themeConfig && themeConfig.plugins
   // Gatsby themes don't have to specify a gatsby-config.js (they might only use gatsby-node, etc)
   // in this case they're technically plugins, but we should support it anyway
   // because we can't guarantee which files theme creators create first
   if (themeConfig && themesList) {
     // for every parent theme a theme defines, resolve the parent's
     // gatsby config and return it in order [parentA, parentB, child]
-    return Promise.mapSeries(themesList, async spec => {
-      const themeObj = await resolveTheme(spec, configFilePath, false, themeDir)
-      return processTheme(themeObj, { rootDir: themeDir })
-    }).then(arr =>
-      arr.concat([
-        { themeName, themeConfig, themeSpec, themeDir, parentDir: rootDir },
-      ])
+    return mapSeries(
+      themesList,
+      async (spec: PluginEntry): Promise<Array<IThemeObj>> => {
+        const themeObj = await resolveTheme(
+          spec,
+          configFilePath,
+          false,
+          themeDir
+        )
+        return processTheme(themeObj, { rootDir: themeDir })
+      }
+    ).then(arr =>
+      flattenDeep(
+        arr.concat([
+          { themeName, themeConfig, themeSpec, themeDir, parentDir: rootDir },
+        ])
+      )
     )
   } else {
     // if a theme doesn't define additional themes, return the original theme
-    return [{ themeName, themeConfig, themeSpec, themeDir, parentDir: rootDir }]
+    return Promise.resolve([
+      { themeName, themeConfig, themeSpec, themeDir, parentDir: rootDir },
+    ])
   }
 }
 
-module.exports = async (config, { configFilePath, rootDir }) => {
-  const themesA = await Promise.mapSeries(
+function normalizePluginEntry(
+  plugin: PluginEntry,
+  parentDir: string
+): IPluginEntryWithParentDir {
+  return {
+    resolve: typeof plugin === `string` ? plugin : plugin.resolve,
+    options: typeof plugin === `string` ? {} : plugin.options || {},
+    parentDir,
+  }
+}
+
+export async function loadThemes(
+  config: IGatsbyConfigInput,
+  { configFilePath, rootDir }: { configFilePath: string; rootDir: string }
+): Promise<{
+  config: IGatsbyConfigInput
+  themes: Array<IThemeObj>
+}> {
+  const themesA = await mapSeries(
     config.plugins || [],
-    async themeSpec => {
+    async (themeSpec: PluginEntry) => {
       const themeObj = await resolveTheme(
         themeSpec,
         configFilePath,
@@ -120,7 +169,7 @@ module.exports = async (config, { configFilePath, rootDir }) => {
       )
       return processTheme(themeObj, { rootDir })
     }
-  ).then(arr => _.flattenDeep(arr))
+  ).then(arr => flattenDeep(arr))
 
   // log out flattened themes list to aid in debugging
   debug(themesA)
@@ -129,21 +178,21 @@ module.exports = async (config, { configFilePath, rootDir }) => {
   // list in the config for the theme. This enables the usage of
   // gatsby-node, etc in themes.
   return (
-    Promise.mapSeries(
+    mapSeries(
       themesA,
       ({ themeName, themeConfig = {}, themeSpec, themeDir, parentDir }) => {
         return {
           ...themeConfig,
           plugins: [
-            ...(themeConfig.plugins || []).map(plugin => {
-              return {
-                resolve: typeof plugin === `string` ? plugin : plugin.resolve,
-                options: plugin.options || {},
-                parentDir: themeDir,
-              }
-            }),
+            ...(themeConfig.plugins || []).map(plugin =>
+              normalizePluginEntry(plugin, themeDir)
+            ),
             // theme plugin is last so it's gatsby-node, etc can override it's declared plugins, like a normal site.
-            { resolve: themeName, options: themeSpec.options || {}, parentDir },
+            {
+              resolve: themeName,
+              options: typeof themeSpec === `string` ? {} : themeSpec.options,
+              parentDir,
+            },
           ],
         }
       }
@@ -156,8 +205,19 @@ module.exports = async (config, { configFilePath, rootDir }) => {
        */
       .reduce(mergeGatsbyConfig, {})
       .then(newConfig => {
+        const mergedConfig = mergeGatsbyConfig(newConfig, {
+          ...config,
+          plugins: [
+            ...(config.plugins || []).map(plugin =>
+              normalizePluginEntry(plugin, rootDir)
+            ),
+          ],
+        })
+
+        mergedConfig.plugins = uniqWith(mergedConfig.plugins, isEqual)
+
         return {
-          config: mergeGatsbyConfig(newConfig, config),
+          config: mergedConfig,
           themes: themesA,
         }
       })

packages/gatsby/src/schema/graphql-engine/entry.ts

@@ -59,18 +59,20 @@ export class GraphQLEngine {
         payload: flattenedPlugins,
       })
 
-      for (const pluginName of Object.keys(gatsbyNodes)) {
+      for (const plugin of gatsbyNodes) {
+        const { name, module, importKey } = plugin
         setGatsbyPluginCache(
-          { name: pluginName, resolve: `` },
+          { name, resolve: ``, importKey },
           `gatsby-node`,
-          gatsbyNodes[pluginName]
+          module
         )
       }
-      for (const pluginName of Object.keys(gatsbyWorkers)) {
+      for (const plugin of gatsbyWorkers) {
+        const { name, module, importKey } = plugin
         setGatsbyPluginCache(
-          { name: pluginName, resolve: `` },
+          { name, resolve: ``, importKey },
           `gatsby-worker`,
-          gatsbyWorkers[pluginName]
+          module
         )
       }
 

packages/gatsby/src/schema/graphql-engine/print-plugins.ts

@@ -53,24 +53,24 @@ async function render(
   usedPlugins: IGatsbyState["flattenedPlugins"],
   usedSubPlugins: IGatsbyState["flattenedPlugins"]
 ): Promise<string> {
-  const uniqGatsbyNode = uniq(usedPlugins)
   const uniqSubPlugins = uniq(usedSubPlugins)
 
-  const sanitizedUsedPlugins = usedPlugins.map(plugin => {
+  const sanitizedUsedPlugins = usedPlugins.map((plugin, i) => {
     // TODO: We don't support functions in pluginOptions here
     return {
       ...plugin,
       resolve: ``,
       pluginFilepath: ``,
       subPluginPaths: undefined,
+      importKey: i + 1,
     }
   })
 
-  const pluginsWithWorkers = await filterPluginsWithWorkers(uniqGatsbyNode)
+  const pluginsWithWorkers = await filterPluginsWithWorkers(usedPlugins)
 
   const subPluginModuleToImportNameMapping = new Map<string, string>()
   const imports: Array<string> = [
-    ...uniqGatsbyNode.map(
+    ...usedPlugins.map(
       (plugin, i) =>
         `import * as pluginGatsbyNode${i} from "${relativePluginPath(
           plugin.resolve
@@ -90,22 +90,28 @@ async function render(
       )}"`
     }),
   ]
-  const gatsbyNodeExports = uniqGatsbyNode.map(
-    (plugin, i) => `"${plugin.name}": pluginGatsbyNode${i},`
+  const gatsbyNodeExports = usedPlugins.map(
+    (plugin, i) =>
+      `{ name: "${plugin.name}", module: pluginGatsbyNode${i}, importKey: ${
+        i + 1
+      } },`
   )
   const gatsbyWorkerExports = pluginsWithWorkers.map(
-    (plugin, i) => `"${plugin.name}": pluginGatsbyWorker${i},`
+    (plugin, i) =>
+      `{ name: "${plugin.name}", module: pluginGatsbyWorker${i}, importKey: ${
+        i + 1
+      } },`
   )
   const output = `
 ${imports.join(`\n`)}
 
-export const gatsbyNodes = {
+export const gatsbyNodes = [
 ${gatsbyNodeExports.join(`\n`)}
-}
+]
 
-export const gatsbyWorkers = {
+export const gatsbyWorkers = [
 ${gatsbyWorkerExports.join(`\n`)}
-}
+]
 
 export const flattenedPlugins =
   ${JSON.stringify(

packages/gatsby/src/schema/graphql-engine/standalone-regenerate.ts

@@ -26,6 +26,7 @@ import { store } from "../../redux"
 import { validateEngines } from "../../utils/validate-engines"
 
 async function run(): Promise<void> {
+  process.env.GATSBY_SLICES = `1`
   // load config
   console.log(`loading config and plugins`)
   await loadConfigAndPlugins({