[WIP] Replace global data.json with page-data.json per page

examples/gatsbygram/package.json

@@ -6,7 +6,7 @@
   "author": "Kyle Mathews <mathews.kyle@gmail.com>",
   "dependencies": {
     "core-js": "^2.5.5",
-    "gatsby": "^2.0.0",
+    "gatsby": "^2.3.4",
     "gatsby-image": "^2.0.5",
     "gatsby-plugin-glamor": "^2.0.5",
     "gatsby-plugin-google-analytics": "^2.0.5",

packages/gatsby-plugin-guess-js/src/gatsby-ssr.js

@@ -1,5 +1,4 @@
 const _ = require(`lodash`)
-const nodePath = require(`path`)
 const fs = require(`fs`)
 const React = require(`react`)
 
@@ -12,18 +11,6 @@ function urlJoin(...parts) {
   }, ``)
 }
 
-let pd = []
-const readPageData = () => {
-  if (pd.length > 0) {
-    return pd
-  } else {
-    pd = JSON.parse(
-      fs.readFileSync(nodePath.join(process.cwd(), `.cache`, `data.json`))
-    )
-    return pd
-  }
-}
-
 let s
 const readStats = () => {
   if (s) {
@@ -37,19 +24,16 @@ const readStats = () => {
 }
 
 exports.onRenderBody = (
-  { setHeadComponents, pathname, pathPrefix },
+  { setHeadComponents, pathname, pathPrefix, loadPageDataSync },
   pluginOptions
 ) => {
   if (process.env.NODE_ENV === `production`) {
-    const pagesData = readPageData()
     const stats = readStats()
     const matchedPaths = Object.keys(
       guess({ path: pathname, threshold: pluginOptions.minimumThreshold })
     )
     if (!_.isEmpty(matchedPaths)) {
-      const matchedPages = matchedPaths.map(match =>
-        _.find(pagesData.pages, page => page.path === match)
-      )
+      const matchedPages = matchedPaths.map(loadPageDataSync)
       let componentUrls = []
       matchedPages.forEach(p => {
         if (p && p.componentChunkName) {

packages/gatsby-plugin-offline/src/gatsby-browser.js

@@ -1,12 +1,9 @@
 exports.registerServiceWorker = () => true
 
-const prefetchedPathnames = []
+const prefetchedResources = []
 const whitelistedPathnames = []
 
-exports.onServiceWorkerActive = ({
-  getResourceURLsForPathname,
-  serviceWorker,
-}) => {
+exports.onServiceWorkerActive = ({ serviceWorker }) => {
   // if the SW has just updated then reset whitelisted paths and don't cache
   // stuff, since we're on the old revision until we navigate to another page
   if (window.___swUpdated) {
@@ -26,15 +23,6 @@ exports.onServiceWorkerActive = ({
     .call(nodes)
     .map(node => node.src || node.href || node.getAttribute(`data-href`))
 
-  // Loop over all resources and fetch the page component and JSON
-  // to add it to the sw cache.
-  const prefetchedResources = []
-  prefetchedPathnames.forEach(path =>
-    getResourceURLsForPathname(path).forEach(resource =>
-      prefetchedResources.push(resource)
-    )
-  )
-
   const resources = [...headerResources, ...prefetchedResources]
   resources.forEach(resource => {
     // Create a prefetch link for each resource, so Workbox runtime-caches them
@@ -69,12 +57,12 @@ function whitelistPathname(pathname, includesPrefix) {
   }
 }
 
-exports.onPostPrefetchPathname = ({ pathname }) => {
+exports.onPostPrefetch = ({ path, resourceUrls }) => {
   // do nothing if the SW has just updated, since we still have old pages in
   // memory which we don't want to be whitelisted
   if (window.___swUpdated) return
 
-  whitelistPathname(pathname, false)
+  whitelistPathname(path, false)
 
   // if SW is not installed, we need to record any prefetches
   // that happen so we can then add them to SW cache once installed
@@ -85,6 +73,6 @@ exports.onPostPrefetchPathname = ({ pathname }) => {
       navigator.serviceWorker.controller.state === `activated`
     )
   ) {
-    prefetchedPathnames.push(pathname)
+    Array.prototype.push.apply(prefetchedResources, resourceUrls)
   }
 }

packages/gatsby/cache-dir/__tests__/find-page.js

@@ -8,23 +8,19 @@ describe(`find-page`, () => {
       {
         path: `/about/`,
         componentChunkName: `page-component---src-pages-test-js`,
-        jsonName: `about.json`,
       },
       {
         path: `/about/me/`,
         componentChunkName: `page-component---src-pages-test-js`,
-        jsonName: `about-me.json`,
       },
       {
         path: `/about/the best/`,
         componentChunkName: `page-component---src-pages-test-js`,
-        jsonName: `the-best.json`,
       },
       {
         path: `/app/`,
         matchPath: `/app/*`,
         componentChunkName: `page-component---src-pages-app-js`,
-        jsonName: `app.json`,
       },
     ]
     findPage = pageFinderFactory(newPages)
@@ -58,7 +54,6 @@ describe(`find-page`, () => {
       {
         path: `/about/`,
         componentChunkName: `page-component---src-pages-test-js`,
-        jsonName: `about.json`,
       },
     ]
     const findPage2 = pageFinderFactory(newPages, `/my-test-prefix`)

packages/gatsby/cache-dir/__tests__/static-entry.js

@@ -22,29 +22,6 @@ jest.mock(
   }
 )
 
-jest.mock(
-  `../data.json`,
-  () => {
-    return {
-      dataPaths: [
-        {
-          [`about.json`]: `/400/about`,
-        },
-      ],
-      pages: [
-        {
-          path: `/about/`,
-          componentChunkName: `page-component---src-pages-test-js`,
-          jsonName: `about.json`,
-        },
-      ],
-    }
-  },
-  {
-    virtual: true,
-  }
-)
-
 const MOCK_FILE_INFO = {
   [`${process.cwd()}/public/webpack.stats.json`]: `{}`,
   [`${process.cwd()}/public/chunk-map.json`]: `{}`,

packages/gatsby/cache-dir/api-runner-browser.js

@@ -3,6 +3,8 @@ const {
   getResourcesForPathname,
   getResourcesForPathnameSync,
   getResourceURLsForPathname,
+  loadPage,
+  getPage,
 } = require(`./loader`).publicLoader
 
 exports.apiRunner = (api, args = {}, defaultReturn, argTransform) => {
@@ -22,9 +24,14 @@ exports.apiRunner = (api, args = {}, defaultReturn, argTransform) => {
       return undefined
     }
 
+    // Deprecated April 2019. Use `getPage` instead
     args.getResourcesForPathnameSync = getResourcesForPathnameSync
+    // Deprecated April 2019. Use `loadPage` instead
     args.getResourcesForPathname = getResourcesForPathname
+    // Deprecated April 2019. Use resources passed in `onPostPrefetch` instead
     args.getResourceURLsForPathname = getResourceURLsForPathname
+    args.loadPage = loadPage
+    args.getPage = getPage
 
     const result = plugin.plugin[api](args, plugin.options)
     if (result && argTransform) {

packages/gatsby/cache-dir/app.js

@@ -5,9 +5,9 @@ import domReady from "@mikaelkristiansson/domready"
 import socketIo from "./socketIo"
 import emitter from "./emitter"
 import { apiRunner, apiRunnerAsync } from "./api-runner-browser"
-import loader, { setApiRunnerForLoader, postInitialRenderWork } from "./loader"
+import loader, { setApiRunnerForLoader } from "./loader"
+import devLoader from "./dev-loader"
 import syncRequires from "./sync-requires"
-import pages from "./pages.json"
 
 window.___emitter = emitter
 setApiRunnerForLoader(apiRunner)
@@ -46,19 +46,26 @@ apiRunnerAsync(`onClientEntry`).then(() => {
     ReactDOM.render
   )[0]
 
-  loader.addPagesArray(pages)
   loader.addDevRequires(syncRequires)
 
-  loader.getResourcesForPathname(window.location.pathname).then(() => {
-    const preferDefault = m => (m && m.default) || m
-    let Root = preferDefault(require(`./root`))
-    domReady(() => {
-      renderer(<Root />, rootElement, () => {
-        postInitialRenderWork()
-        apiRunner(`onInitialClientRender`)
+  Promise.all([
+    loader.loadPage(window.location.pathname),
+    loader.loadPage(`/dev-404-page/`),
+    loader.loadPage(`/404.html`).catch(err => null),
+    devLoader.loadPages(),
+  ])
+    .then(() => {
+      const preferDefault = m => (m && m.default) || m
+      let Root = preferDefault(require(`./root`))
+      domReady(() => {
+        renderer(<Root />, rootElement, () => {
+          apiRunner(`onInitialClientRender`)
+        })
       })
     })
-  })
+    .catch(err => {
+      console.log(err)
+    })
 })
 
 function supportsServiceWorkers(location, navigator) {

packages/gatsby/cache-dir/dev-loader.js

@@ -0,0 +1,49 @@
+// Initialized by calling loadPages
+let pagesManifest = null
+
+const fetchPages = () =>
+  new Promise((resolve, reject) => {
+    const req = new XMLHttpRequest()
+    req.open(`GET`, `/___pages`, true)
+    req.onreadystatechange = () => {
+      if (req.readyState == 4) {
+        if (req.status === 200) {
+          // TODO is this safe? Maybe just do this check in dev mode?
+          const contentType = req.getResponseHeader(`content-type`)
+          if (!contentType || !contentType.startsWith(`application/json`)) {
+            reject()
+          } else {
+            resolve(JSON.parse(req.responseText))
+          }
+        } else {
+          reject()
+        }
+      }
+    }
+    req.send(null)
+  })
+
+// Returns a promise that fetches the `/___pages` resource from the
+// running `gatsby develop` server. It contains a map of all pages on
+// the site (path -> page). Call `getPagesManifest()` to retrieve the
+// pages. Used by dev-404-page to present a list of all pages
+const loadPages = () =>
+  fetchPages().then(pages => {
+    pagesManifest = pages
+  })
+
+// Returns the map of all pages on the site (path -> page)
+const getPagesManifest = () => {
+  if (pagesManifest === null) {
+    throw new Error(
+      `pages-manifest hasn't been initialized. Ensure the dev-loader/loadPages has been called first`
+    )
+  } else {
+    return pagesManifest
+  }
+}
+
+module.exports = {
+  loadPages,
+  getPagesManifest,
+}

packages/gatsby/cache-dir/ensure-resources.js

@@ -17,7 +17,7 @@ class EnsureResources extends React.Component {
 
     this.state = {
       location: { ...location },
-      pageResources: loader.getResourcesForPathnameSync(location.pathname),
+      pageResources: loader.getPageOr404(location.pathname),
     }
   }
 
@@ -34,9 +34,7 @@ class EnsureResources extends React.Component {
 
   static getDerivedStateFromProps({ location }, prevState) {
     if (prevState.location !== location) {
-      const pageResources = loader.getResourcesForPathnameSync(
-        location.pathname
-      )
+      const pageResources = loader.getPageOr404(location.pathname)
 
       return {
         pageResources,
@@ -62,15 +60,15 @@ class EnsureResources extends React.Component {
   retryResources(nextProps) {
     const { pathname } = nextProps.location
 
-    if (!loader.getResourcesForPathnameSync(pathname)) {
+    if (!loader.getPage(pathname)) {
       // Store the previous and next location before resolving resources
       const prevLocation = this.props.location
       this.nextLocation = nextProps.location
 
       // Page resources won't be set in cases where the browser back button
       // or forward button is pushed as we can't wait as normal for resources
       // to load before changing the page.
-      loader.getResourcesForPathname(pathname).then(pageResources => {
+      loader.loadPage(pathname).then(pageResources => {
         // The page may have changed since we started this, in which case doesn't update
         if (this.nextLocation !== nextProps.location) {
           return