Split out loading/executing resources into functions

packages/gatsby/src/cache-dir/component-renderer.js

@@ -25,6 +25,7 @@ class ComponentRenderer extends React.Component {
   componentDidMount() {
     // listen to events.
     ___emitter.on(`onPostLoadPageResources`, e => {
+      console.log("onPostLoadPageResources", e)
       if (e.page.path === this.props.location.pathname) {
         this.setState({ pageResources: e.pageResources })
       }

packages/gatsby/src/cache-dir/loader.js

@@ -8,6 +8,8 @@ let findPage
 let syncRequires = {}
 let asyncRequires = {}
 let pathScriptsCache = {}
+let resourceStrCache = {}
+let resourceCache = {}
 let pages = []
 // Note we're not actively using the path data atm. There
 // could be future optimizations however around trying to ensure
@@ -38,7 +40,75 @@ const sortPagesByCount = (a, b) => {
   }
 }
 
-const prefetcher = () => {}
+const fetchResource = (resourceName, cb = () => {}) => {
+  if (resourceStrCache[resourceName]) {
+    return resourceStrCache[resourceName]
+  } else {
+    // Find resource
+    const resourceFunction = resourceName.slice(0, 6) === `page-c`
+      ? asyncRequires.components[resourceName]
+      : asyncRequires.json[resourceName]
+
+    // Download the resource
+    console.time(`download resource ${resourceName}`)
+    resourceFunction((err, executeChunk) => {
+      console.timeEnd(`download resource ${resourceName}`)
+      resourceStrCache[resourceName] = executeChunk
+      cb(err, executeChunk)
+    })
+  }
+}
+
+const getResourceModule = (resourceName, cb) => {
+  if (resourceCache[resourceName]) {
+    return cb(null, resourceCache[resourceName])
+  } else {
+    fetchResource(resourceName, (err, executeChunk) => {
+      if (err) {
+        cb(err)
+      } else {
+        console.time(`execute chunk ${resourceName}`)
+        const module = preferDefault(executeChunk())
+        console.timeEnd(`execute chunk ${resourceName}`)
+        resourceCache[resourceName] = module
+        return cb(err, module)
+      }
+    })
+  }
+}
+
+// TODO split this into two functions, one for fetching
+// resource and the other for executing it.
+// This prefetcher just sits around waiting for
+// new resources to be added. When hears onPreLoadPageResources
+// it stops until it hears onPostLoadPageResources
+// actually, make it a state machine like thing that's listening
+// to events and flipping it's state around.
+//
+// This is stupid complex. Need tests somehow.
+// const prefetcher = () => {
+// // Get top resource and start downloading it.
+// const nextResource = resourcesArray.slice(-1)[0]
+// if (nextResource) {
+// console.log("prefetching next resource", nextResource)
+// fetchResource(nextResource, (err, executeChunk) => {
+// console.log("executeChunk", executeChunk)
+// resourcesArray = resourcesArray.filter(r => r !== nextResource)
+// setTimeout(() => {
+// console.time(`execute resource`)
+// const module = getResourceModule(nextResource)
+// console.timeEnd(`execute resource`)
+// console.log("module", module)
+// prefetcher()
+// }, 500)
+// })
+// } else {
+// // Wait a second and try again
+// setTimeout(() => prefetcher(), 1000)
+// }
+// }
+
+// setTimeout(() => prefetcher(), 2500)
 
 const queue = {
   empty: () => {
@@ -141,11 +211,9 @@ const queue = {
       if (pathScriptsCache[path]) {
         return pathScriptsCache[path]
       }
-
       console.log("need to load scripts")
       const page = findPage(path)
       console.log("for page", page)
-      console.log(`async requires`, asyncRequires)
       let component
       let json
       const done = () => {
@@ -157,19 +225,18 @@ const queue = {
           })
         }
       }
-      console.log(asyncRequires.components[page.componentChunkName])
-      console.log(asyncRequires.json[page.jsonName])
-      asyncRequires.components[page.componentChunkName]()(
-        callback => {
-          component = preferDefault(callback())
-          console.log(`page component`, component)
-          done()
-        },
-        () => console.log("error loading page component")
-      )
-      asyncRequires.json[page.jsonName]()(callback => {
-        json = preferDefault(callback())
-        console.log(`json`, json)
+      getResourceModule(page.componentChunkName, (err, c) => {
+        if (err) {
+          return console.log("we failed folks")
+        }
+        component = c
+        done()
+      })
+      getResourceModule(page.jsonName, (err, j) => {
+        if (err) {
+          return console.log("we failed folks")
+        }
+        json = j
         done()
       })
     }

packages/gatsby/src/loaders/gatsby-module-loader/index.js

@@ -6,15 +6,12 @@
 */
 const loaderUtils = require(`loader-utils`)
 const path = require(`path`)
-console.log("in gatsby-module-loader")
 
 module.exports = function() {}
 module.exports.pitch = function(remainingRequest) {
   this.cacheable && this.cacheable()
 
-  console.log("this.query", this.query)
   const query = loaderUtils.parseQuery(this.query)
-  console.log("query", query)
   let chunkName = ``
 
   if (query.name) {
@@ -25,22 +22,19 @@ module.exports.pitch = function(remainingRequest) {
     chunkName = `, ${JSON.stringify(chunkName)}`
   }
 
-  console.log(`chunkName`, chunkName)
-
   const request = loaderUtils.stringifyRequest(this, `!!` + remainingRequest)
 
-  const callback = "callback(function() { return require(" + request + ") })"
+  const callback = "function() { return require(" + request + ") }"
 
   const executor = `
-    return function(callback, errback) {
-      require.ensure([], function(_, error) {
+     return require.ensure([], function(_, error) {
         if (error) {
-          errback()
+          cb(true)
         } else {
-          ${callback}
+          cb(null, ${callback})
         }
       }${chunkName});
-    }`
+    `
 
   const result = `
     require(
@@ -49,10 +43,8 @@ module.exports.pitch = function(remainingRequest) {
         `!${path.join(__dirname, `patch.js`)}`
       )}
     );
-    module.exports = function() { ${executor} }
+    module.exports = function(cb) { ${executor} }
     `
 
-  console.log(`result`, result)
-
   return result
 }

packages/gatsby/src/loaders/gatsby-module-loader/patch.js

@@ -76,5 +76,5 @@ function patch() {
       script.onload = script.onerror = null
       setTimeout(callback, 0)
     }
-  };
-};
+  }
+}