Add prefetcher

packages/gatsby/src/cache-dir/__tests__/prefetcher.js

@@ -0,0 +1,64 @@
+let prefetcher
+let createResourceDownload
+
+describe(`Loader`, () => {
+  beforeEach(() => {
+    createResourceDownload = jest.fn()
+    prefetcher = require(`../prefetcher.js`)({
+      getNextQueuedResources: () => Math.random(),
+      createResourceDownload,
+    })
+    prefetcher.empty()
+  })
+
+  test(`It tracks when pages are loading`, () => {
+    prefetcher.onPreLoadPageResources()
+    prefetcher.onPreLoadPageResources()
+    expect(prefetcher.getState().pagesLoading).toEqual(2)
+  })
+
+  test(`It tracks when pages finish loading`, () => {
+    prefetcher.onPreLoadPageResources()
+    prefetcher.onPostLoadPageResources()
+    expect(prefetcher.getState().pagesLoading).toEqual(0)
+  })
+
+  test(`When a new resource is queued, it tries to download it`, () => {
+    prefetcher.onNewResourcesAdded()
+    expect(createResourceDownload.mock.calls.length).toBe(1)
+  })
+
+  test(`If page resources are downloading, it doesn't prefetch until after it's finished`, () => {
+    prefetcher.onPreLoadPageResources()
+    prefetcher.onNewResourcesAdded()
+    expect(createResourceDownload.mock.calls.length).toBe(0)
+    prefetcher.onPostLoadPageResources()
+    expect(createResourceDownload.mock.calls.length).toBe(1)
+  })
+
+  test(`Once one resource finishes downloading, it starts another`, () => {
+    prefetcher.onNewResourcesAdded()
+    // Get resource name
+    const resourceName = createResourceDownload.mock.calls[0][0]
+
+    // New resources added shouldn't trigger new downloads.
+    prefetcher.onNewResourcesAdded()
+    prefetcher.onNewResourcesAdded()
+    expect(createResourceDownload.mock.calls.length).toBe(1)
+
+    // Finish the first download triggers another download.
+    prefetcher.onResourcedFinished(resourceName)
+    expect(createResourceDownload.mock.calls.length).toBe(2)
+  })
+
+  test(`It stops downloading when the resourcesArray is empty`, () => {
+    prefetcher = require(`../prefetcher.js`)({
+      getNextQueuedResources: () => undefined,
+      createResourceDownload: jest.fn(),
+    })
+    prefetcher.empty()
+
+    prefetcher.onNewResourcesAdded()
+    expect(createResourceDownload.mock.calls.length).toBe(0)
+  })
+})

packages/gatsby/src/cache-dir/component-renderer.js

@@ -1,5 +1,6 @@
 import React, { createElement } from "react"
 import loader from "./loader"
+import emitter from "./emitter"
 
 // Pass pathname in as prop.
 // component will try fetching resources. If they exist,
@@ -24,17 +25,14 @@ class ComponentRenderer extends React.Component {
 
   componentDidMount() {
     // listen to events.
-    ___emitter.on(`onPostLoadPageResources`, e => {
-      console.log("onPostLoadPageResources", e)
+    emitter.on(`onPostLoadPageResources`, e => {
       if (e.page.path === this.props.location.pathname) {
         this.setState({ pageResources: e.pageResources })
       }
     })
   }
 
   render() {
-    console.log("rendering ComponentRenderer")
-    console.log(this)
     if (this.state.pageResources) {
       return createElement(this.state.pageResources.component, {
         ...this.props,

packages/gatsby/src/cache-dir/emitter.js

@@ -0,0 +1,3 @@
+import mitt from "mitt"
+const emitter = mitt()
+module.exports = emitter

packages/gatsby/src/cache-dir/loader.js

@@ -3,6 +3,7 @@ if (typeof window !== `undefined`) {
 }
 
 import pageFinderFactor from "./find-page"
+import emitter from "./emitter"
 let findPage
 
 let syncRequires = {}
@@ -20,6 +21,30 @@ let resourcesArray = []
 let resourcesCount = {}
 const preferDefault = m => (m && m.default) || m
 
+// Prefetcher logic
+const prefetcher = require(`./prefetcher`)({
+  getNextQueuedResources: () => resourcesArray.slice(-1)[0],
+  createResourceDownload: resourceName => {
+    console.log("fetching resource:", resourceName)
+    console.log(`resourcesArray length`, resourcesArray.length)
+    fetchResource(resourceName, () => {
+      resourcesArray = resourcesArray.filter(r => r !== resourceName)
+      console.log(`resourcesArray length`, resourcesArray.length)
+      prefetcher.onResourcedFinished(resourceName)
+      console.log(`resourceStrCache`, resourceStrCache)
+      console.log(`resourceCache`, resourceCache)
+    })
+  },
+})
+emitter.on(`ON_PRE_LOAD_PAGE_RESOURCES`, e => {
+  console.log(`ON_PRE_LOAD_PAGE_RESOURCES`, e)
+  prefetcher.onPreLoadPageResources(e)
+})
+emitter.on(`ON_POST_LOAD_PAGE_RESOURCES`, e => {
+  console.log(`ON_POST_LOAD_PAGE_RESOURCES`, e)
+  prefetcher.onPostLoadPageResources(e)
+})
+
 const sortResourcesByCount = (a, b) => {
   if (resourcesCount[a] > resourcesCount[b]) {
     return 1
@@ -42,7 +67,7 @@ const sortPagesByCount = (a, b) => {
 
 const fetchResource = (resourceName, cb = () => {}) => {
   if (resourceStrCache[resourceName]) {
-    return resourceStrCache[resourceName]
+    return cb(null, resourceStrCache[resourceName])
   } else {
     // Find resource
     const resourceFunction = resourceName.slice(0, 6) === `page-c`
@@ -60,6 +85,7 @@ const fetchResource = (resourceName, cb = () => {}) => {
 }
 
 const getResourceModule = (resourceName, cb) => {
+  console.log(`getting resourceName`, resourceName)
   if (resourceCache[resourceName]) {
     return cb(null, resourceCache[resourceName])
   } else {
@@ -77,39 +103,6 @@ const getResourceModule = (resourceName, cb) => {
   }
 }
 
-// TODO split this into two functions, one for fetching
-// resource and the other for executing it.
-// This prefetcher just sits around waiting for
-// new resources to be added. When hears onPreLoadPageResources
-// it stops until it hears onPostLoadPageResources
-// actually, make it a state machine like thing that's listening
-// to events and flipping it's state around.
-//
-// This is stupid complex. Need tests somehow.
-// const prefetcher = () => {
-// // Get top resource and start downloading it.
-// const nextResource = resourcesArray.slice(-1)[0]
-// if (nextResource) {
-// console.log("prefetching next resource", nextResource)
-// fetchResource(nextResource, (err, executeChunk) => {
-// console.log("executeChunk", executeChunk)
-// resourcesArray = resourcesArray.filter(r => r !== nextResource)
-// setTimeout(() => {
-// console.time(`execute resource`)
-// const module = getResourceModule(nextResource)
-// console.timeEnd(`execute resource`)
-// console.log("module", module)
-// prefetcher()
-// }, 500)
-// })
-// } else {
-// // Wait a second and try again
-// setTimeout(() => prefetcher(), 1000)
-// }
-// }
-
-// setTimeout(() => prefetcher(), 2500)
-
 const queue = {
   empty: () => {
     pathArray = []
@@ -159,7 +152,12 @@ const queue = {
         resourcesCount[page.jsonName] += 1
       }
 
-      if (resourcesArray.indexOf(page.jsonName) === -1) {
+      // Before adding checking that the JSON resource isn't either
+      // already queued or been downloading.
+      if (
+        resourcesArray.indexOf(page.jsonName) === -1 &&
+        !resourceStrCache[page.jsonName]
+      ) {
         resourcesArray.unshift(page.jsonName)
       }
     }
@@ -170,13 +168,19 @@ const queue = {
         resourcesCount[page.componentChunkName] += 1
       }
 
-      if (resourcesArray.indexOf(page.componentChunkName) === -1) {
+      // Before adding checking that the component resource isn't either
+      // already queued or been downloading.
+      if (
+        resourcesArray.indexOf(page.componentChunkName) === -1 &&
+        !resourceStrCache[page.jsonName]
+      ) {
         resourcesArray.unshift(page.componentChunkName)
       }
     }
 
-    // Sort resources by resourcesCount
+    // Sort resources by resourcesCount.
     resourcesArray.sort(sortResourcesByCount)
+    prefetcher.onNewResourcesAdded()
 
     return true
   },
@@ -195,7 +199,7 @@ const queue = {
   hasPage: pathname => findPage(pathname),
   has: path => pathArray.some(p => p === path),
   getResourcesForPathname: path => {
-    ___emitter.emit(`onPreLoadPageResources`, { path })
+    emitter.emit(`onPreLoadPageResources`, { path })
     // In development we know the code is loaded already
     // so we just return with it immediately.
     if (process.env.NODE_ENV !== `production`) {
@@ -204,7 +208,8 @@ const queue = {
         component: syncRequires.components[page.componentChunkName],
         json: syncRequires.json[page.jsonName],
       }
-      ___emitter.emit(`onPostLoadPageResources`, { page, pageResources })
+      console.log(`EMITTING onPostLoadPageResources`, page, pageResources)
+      emitter.emit(`onPostLoadPageResources`, { page, pageResources })
       return pageResources
     } else {
       // Check if it's in the cache already.
@@ -218,24 +223,29 @@ const queue = {
       let json
       const done = () => {
         if (component && json) {
+          console.log("done loading")
           pathScriptsCache[path] = { component, json }
-          ___emitter.emit(`onPostLoadPageResources`, {
+          const pageResources = { component, json }
+          console.log(`EMITTING onPostLoadPageResources`, page, pageResources)
+          emitter.emit(`onPostLoadPageResources`, {
             page,
-            pageResources: { component, json },
+            pageResources,
           })
         }
       }
       getResourceModule(page.componentChunkName, (err, c) => {
         if (err) {
           return console.log("we failed folks")
         }
+        console.log("done getting component")
         component = c
         done()
       })
       getResourceModule(page.jsonName, (err, j) => {
         if (err) {
           return console.log("we failed folks")
         }
+        console.log("done getting json")
         json = j
         done()
       })

packages/gatsby/src/cache-dir/prefetcher.js

@@ -0,0 +1,69 @@
+module.exports = ({ getNextQueuedResources, createResourceDownload }) => {
+  let pagesLoading = 0
+  let resourcesDownloading = []
+
+  // Do things
+  const startResourceDownloading = () => {
+    const nextResource = getNextQueuedResources()
+    if (nextResource) {
+      resourcesDownloading.push(nextResource)
+      createResourceDownload(nextResource)
+      console.log({ pagesLoading, resourcesDownloading })
+    }
+  }
+
+  const reducer = action => {
+    switch (action.type) {
+      case `RESOURCE_FINISHED`:
+        console.log(`reducer RESOURCE_FINISHED`, action, resourcesDownloading)
+        resourcesDownloading = resourcesDownloading.filter(
+          r => r !== action.payload
+        )
+        break
+      case `ON_PRE_LOAD_PAGE_RESOURCES`:
+        pagesLoading += 1
+        break
+      case `ON_POST_LOAD_PAGE_RESOURCES`:
+        pagesLoading -= 1
+        break
+      case `ON_NEW_RESOURCES_ADDED`:
+        break
+    }
+
+    // Take actions.
+    if (resourcesDownloading.length === 0 && pagesLoading === 0) {
+      // Start another resource downloading.
+      startResourceDownloading()
+    }
+  }
+
+  return {
+    onResourcedFinished: event => {
+      // Tell prefetcher that the resource finished downloading
+      // so it can grab the next one.
+      reducer({ type: `RESOURCE_FINISHED`, payload: event })
+    },
+    onPreLoadPageResources: event => {
+      // Tell prefetcher a page load has started so it should stop
+      // loading anything new
+      reducer({ type: `ON_PRE_LOAD_PAGE_RESOURCES`, payload: event })
+    },
+    onPostLoadPageResources: event => {
+      // Tell prefetcher a page load has finished so it should start
+      // loading resources again.
+      reducer({ type: `ON_POST_LOAD_PAGE_RESOURCES`, payload: event })
+    },
+    onNewResourcesAdded: () => {
+      // Tell prefetcher that more resources to be downloaded have
+      // been added.
+      reducer({ type: `ON_NEW_RESOURCES_ADDED` })
+    },
+    getState: () => {
+      return { pagesLoading, resourcesDownloading }
+    },
+    empty: () => {
+      pagesLoading = 0
+      resourcesDownloading = []
+    },
+  }
+}