[1.0] make determining when a given stage is finished much more relia…

…ble (#1080) * Use 'traceId' to track when cascading actions are complete * Add jobs system + move query running into internal plugins * Fixes, remove logging, disable rebuilding schema in dev * Remove unused code/packages * Rename variables * Update snapshots * Fix building + change postBuild to onPostBuild * Fix Drupal source plugin/site by restoring way for plugins to set status + touchNode action creator * Remove console.logs * debugging * Use jest moduleNameMapper to fix failing test * Don't query raw field as we don't use it * Use standard JSDoc notation
gatsbyjs · Jun 1, 2017 · 7b3bcd5 · 7b3bcd5
1 parent aa367c3
commit 7b3bcd5
Show file tree

Hide file tree

Showing 53 changed files with 785 additions and 364 deletions.
diff --git a/jest.config.js b/jest.config.js
@@ -21,5 +21,6 @@ module.exports = {
   moduleNameMapper: {
     "^graphql-skip-limit$": `<rootDir>/packages/graphql-skip-limit/src/index.js`,
     "^gatsby-plugin-sharp$": `<rootDir>/packages/gatsby-plugin-sharp/src/index.js`,
+    "^gatsby/dist/redux/actions$": `<rootDir>/packages/gatsby/src/redux/actions.js`,
   },
 }
diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js
@@ -1,10 +1,7 @@
-import { queue } from "./index"
-import Promise from "bluebird"
-
-async function generateSideEffects() {
-  return new Promise(resolve => {
-    queue.start(() => resolve())
-  })
+// TODO
+exports.formatJobMessage = jobs => {
+  return {
+    progress: 40,
+    message: `3/4`,
+  }
 }
-
-exports.generateSideEffects = generateSideEffects
diff --git a/packages/gatsby-plugin-sharp/src/index.js b/packages/gatsby-plugin-sharp/src/index.js
@@ -7,9 +7,11 @@ const fs = require(`fs`)
 const ProgressBar = require(`progress`)
 const imagemin = require(`imagemin`)
 const imageminPngquant = require(`imagemin-pngquant`)
-const queue = require(`queue`)
+const queue = require(`async/queue`)
 const duotone = require(`./duotone`)
 
+const { boundActionCreators } = require(`gatsby/dist/redux/actions`)
+
 // Promisify the sharp prototype (methods) to promisify the alternative (for
 // raw) callback-accepting toBuffer(...) method
 Promise.promisifyAll(sharp.prototype, { multiArgs: true })
@@ -27,8 +29,18 @@ const bar = new ProgressBar(
     width: 30,
   }
 )
+
 const processFile = (file, jobs, cb) => {
-  Promise.all(jobs.map(job => job.finished)).then(() => cb())
+  totalJobs += jobs.length
+  bar.total = _.sumBy(
+    Object.keys(toProcess),
+    key => _.values(toProcess[key]).length
+  )
+
+  let imagesFinished = 0
+
+  // Wait for each job promise to resolve.
+  Promise.all(jobs.map(job => job.finishedPromise)).then(() => cb())
   const pipeline = sharp(file).rotate()
   jobs.forEach(async job => {
     const args = job.args
@@ -80,7 +92,15 @@ const processFile = (file, jobs, cb) => {
 
     if (job.file.extension.match(/^jp/)) {
       clonedPipeline.toFile(job.outputPath, (err, info) => {
+        imagesFinished += 1
         bar.tick()
+        boundActionCreators.setJob(
+          {
+            id: `processing image ${job.file.absolutePath}`,
+            imagesFinished,
+          },
+          { name: `gatsby-plugin-sharp` }
+        )
         job.outsideResolve(info)
       })
       // Compress pngs
@@ -96,7 +116,15 @@ const processFile = (file, jobs, cb) => {
           })
           .then(imageminBuffer => {
             fs.writeFile(job.outputPath, imageminBuffer, () => {
+              imagesFinished += 1
               bar.tick()
+              boundActionCreators.setJob(
+                {
+                  id: `processing image ${job.file.absolutePath}`,
+                  imagesFinished,
+                },
+                { name: `gatsby-plugin-sharp` }
+              )
               job.outsideResolve()
             })
           })
@@ -105,10 +133,11 @@ const processFile = (file, jobs, cb) => {
   })
 }
 
-let totalCount = 0
+let totalJobs = 0
 const toProcess = {}
-const q = queue()
-q.concurrency = 1
+const q = queue((task, callback) => {
+  task(callback)
+}, 1)
 
 const queueJob = job => {
   const inputFileKey = job.file.absolutePath.replace(/\./g, `%2E`)
@@ -125,8 +154,6 @@ const queueJob = job => {
     return
   }
 
-  totalCount += 1
-
   let notQueued = true
   if (toProcess[inputFileKey]) {
     notQueued = false
@@ -136,20 +163,34 @@ const queueJob = job => {
     `${job.file.absolutePath.replace(/\./g, `%2E`)}.${job.outputPath.replace(/\./g, `%2E`)}`,
     job
   )
+
+  // totalJobs += 1
   if (notQueued) {
     q.push(cb => {
+      // console.log("processing image", job.file.absolutePath)
+      boundActionCreators.createJob(
+        {
+          id: `processing image ${job.file.absolutePath}`,
+          imagesCount: _.values(toProcess[inputFileKey]).length,
+        },
+        { name: `gatsby-plugin-sharp` }
+      )
       // We're now processing the file's jobs.
       processFile(
         job.file.absolutePath,
         _.values(toProcess[inputFileKey]),
         () => {
+          boundActionCreators.endJob(
+            {
+              id: `processing image ${job.file.absolutePath}`,
+            },
+            { name: `gatsby-plugin-sharp` }
+          )
           cb()
         }
       )
     })
   }
-
-  bar.total = totalCount
 }
 
 function queueImageResizing({ file, args = {} }) {
@@ -194,7 +235,7 @@ function queueImageResizing({ file, args = {} }) {
   const filePath = `${process.cwd()}/public${imgSrc}`
   // Create function to call when the image is finished.
   let outsideResolve
-  const finished = new Promise(resolve => {
+  const finishedPromise = new Promise(resolve => {
     outsideResolve = resolve
   })
 
@@ -220,7 +261,7 @@ function queueImageResizing({ file, args = {} }) {
   const job = {
     file,
     args: options,
-    finished,
+    finishedPromise,
     outsideResolve,
     inputPath: file.absolutePath,
     outputPath: filePath,
@@ -237,7 +278,7 @@ function queueImageResizing({ file, args = {} }) {
     width,
     height,
     aspectRatio,
-    finished,
+    finishedPromise,
   }
 }
 
@@ -491,7 +532,6 @@ async function responsiveResolution({ file, args = {} }) {
   }
 }
 
-exports.queue = q
 exports.queueImageResizing = queueImageResizing
 exports.base64 = base64
 exports.responsiveSizes = responsiveSizes

diff --git a/packages/gatsby-source-drupal/src/gatsby-node.js b/packages/gatsby-source-drupal/src/gatsby-node.js
@@ -28,18 +28,7 @@ exports.sourceNodes = async (
   { boundActionCreators, getNode, hasNodeChanged, store },
   { baseUrl }
 ) => {
-  const {
-    createNode,
-    updateSourcePluginStatus,
-    touchNode,
-  } = boundActionCreators
-  updateSourcePluginStatus({
-    plugin: `gatsby-source-drupal`,
-    status: {
-      ...store.getState().status[`gatsby-source-drupal`],
-      ready: false,
-    },
-  })
+  const { createNode, setPluginStatus, touchNode } = boundActionCreators
 
   // Touch existing Drupal nodes so Gatsby doesn't garbage collect them.
   _.values(store.getState().nodes)
@@ -50,9 +39,14 @@ exports.sourceNodes = async (
   console.time(`fetch Drupal data`)
   console.log(`Starting to fetch data from Drupal`)
 
-  const lastFetched = store.getState().status.sourcePlugins[
-    `gatsby-source-drupal`
-  ].lastFetched
+  let lastFetched
+  if (
+    store.getState().status.plugins &&
+    store.getState().status.plugins[`gatsby-source-drupal`]
+  ) {
+    lastFetched = store.getState().status.plugins[`gatsby-source-drupal`]
+      .lastFetched
+  }
 
   let url
   if (lastFetched) {
@@ -70,10 +64,8 @@ exports.sourceNodes = async (
 
   console.log(`articles fetched`, result.data.data.length)
 
-  updateSourcePluginStatus({
-    plugin: `gatsby-source-drupal`,
+  setPluginStatus({
     status: {
-      ...store.getState().status[`gatsby-source-drupal`],
       lastFetched: new Date().toJSON(),
     },
   })
@@ -158,13 +150,5 @@ exports.sourceNodes = async (
     )
   )
 
-  updateSourcePluginStatus({
-    plugin: `gatsby-source-drupal`,
-    status: {
-      ...store.getState().status[`gatsby-source-drupal`],
-      ready: true,
-    },
-  })
-
   return
 }
diff --git a/packages/gatsby-source-filesystem/src/gatsby-node.js b/packages/gatsby-source-filesystem/src/gatsby-node.js
@@ -56,21 +56,13 @@ function readFile(file, pluginOptions, cb) {
 
 exports.sourceNodes = (
   { boundActionCreators, getNode, hasNodeChanged },
-  pluginOptions
+  pluginOptions,
+  done
 ) => {
-  const {
-    createNode,
-    deleteNode,
-    updateSourcePluginStatus,
-  } = boundActionCreators
+  const { createNode, deleteNode } = boundActionCreators
 
   let ready = false
 
-  updateSourcePluginStatus({
-    plugin: `source-filesystem --- ${pluginOptions.name}`,
-    status: { ready },
-  })
-
   const watcher = chokidar.watch(pluginOptions.path, {
     ignored: [
       `**/*.un~`,
@@ -133,10 +125,7 @@ exports.sourceNodes = (
 
     ready = true
     flushPathQueue(() => {
-      updateSourcePluginStatus({
-        plugin: `source-filesystem --- ${pluginOptions.name}`,
-        status: { ready },
-      })
+      done()
     })
   })
 

diff --git a/packages/gatsby-source-hacker-news/src/gatsby-node.js b/packages/gatsby-source-hacker-news/src/gatsby-node.js
@@ -14,10 +14,6 @@ exports.sourceNodes = async ({
   hasNodeChanged,
 }) => {
   const { createNode, updateSourcePluginStatus } = boundActionCreators
-  updateSourcePluginStatus({
-    plugin: `gatsby-source-hacker-news`,
-    ready: false,
-  })
 
   // Do the initial fetch
   console.time(`fetch HN data`)
@@ -169,10 +165,6 @@ fragment commentsFragment on HackerNewsItem {
   })
 
   const ready = true
-  updateSourcePluginStatus({
-    plugin: `gatsby-source-hacker-news`,
-    status: { ready },
-  })
 
   return
 }
diff --git a/packages/gatsby/package.json b/packages/gatsby/package.json
@@ -41,13 +41,11 @@
     "extract-text-webpack-plugin": "^1.0.1",
     "file-loader": "^0.9.0",
     "flat": "^2.0.1",
-    "flux-standard-action": "^1.2.0",
     "front-matter": "^2.1.0",
     "fs-extra": "^3.0.1",
     "glob": "^7.1.1",
     "graphql-relay": "^0.5.1",
     "graphql-skip-limit": "^1.0.0-alpha16",
-    "graphql-union-input-type": "^0.2.1",
     "gray-matter": "^2.1.0",
     "hapi": "^8.5.1",
     "history": "^4.6.1",
@@ -83,7 +81,6 @@
     "postcss-reporter": "^1.4.1",
     "raw-loader": "^0.5.1",
     "react": "^15.4.0",
-    "react-document-title": "^2.0.1",
     "react-dom": "^15.4.0",
     "react-hot-loader": "^3.0.0-beta.6",
     "react-router": "^4.1.1",
@@ -92,7 +89,6 @@
     "redux": "^3.6.0",
     "relay-compiler": "^1.0.0-rc.3",
     "remote-redux-devtools": "^0.5.7",
-    "resolve-tree": "^0.1.13",
     "sift": "^3.2.6",
     "slash": "^1.0.0",
     "static-site-generator-webpack-plugin": "^3.1.0",
@@ -142,8 +138,8 @@
   "scripts": {
     "build": "rimraf dist && npm run build:src && npm run build:cli && npm run build:internal-plugins && npm run build:rawfiles",
     "build:src": "babel src --out-dir dist --source-maps --ignore gatsby-cli.js,raw_*,__tests__",
-    "build:internal-plugins": "copyfiles -u 1 src/bootstrap/internal-plugins/**/package.json dist",
-    "build:rawfiles": "copyfiles -u 1 src/bootstrap/internal-plugins/**/raw_* dist",
+    "build:internal-plugins": "copyfiles -u 1 src/internal-plugins/**/package.json dist",
+    "build:rawfiles": "copyfiles -u 1 src/internal-plugins/**/raw_* dist",
     "build:cli": "babel src/gatsby-cli.js --out-file dist/gatsby-cli.js --presets es2015",
     "clean-test-bundles": "find test/ -type f -name bundle.js* -exec rm -rf {} +",
     "test-coverage": "node_modules/.bin/nyc --reporter=lcov --reporter=text npm test",

diff --git a/packages/gatsby/src/bootstrap/__tests__/__snapshots__/load-plugins.js.snap b/packages/gatsby/src/bootstrap/__tests__/__snapshots__/load-plugins.js.snap
@@ -26,6 +26,14 @@ Array [
     "resolve": "",
     "version": "1.0.0",
   },
+  Object {
+    "name": "query-runner",
+    "pluginOptions": Object {
+      "plugins": Array [],
+    },
+    "resolve": "",
+    "version": "1.0.0",
+  },
   Object {
     "name": "TEST",
     "pluginOptions": Object {
@@ -71,6 +79,14 @@ Array [
     "resolve": "",
     "version": "1.0.0",
   },
+  Object {
+    "name": "query-runner",
+    "pluginOptions": Object {
+      "plugins": Array [],
+    },
+    "resolve": "",
+    "version": "1.0.0",
+  },
   Object {
     "name": "defaultSitePlugin",
     "pluginOptions": Object {