refactor(gatsby-plugin-sharp): split single file into more maintaina…

…ble chunks (#10964) * feat(sharp): thumbnail creation on demand in dev mode * Fix thumbnail generation on gatsby build * fix eslint * fix queue caching * fix queue caching * move comments * fix comments * update cache on query change * update webpack tap name * review changes * review changes * fix review * make cache smaller & refator toFormat * fix testsf * fix schedulejob on demand * update snapshots * add tests to opt out of lazy image resolving * Use schedulejob instead of processFile in devmode * Use schedulejob instead of processFile in devmode Speed up multiple requests with some input files * hide status on develop (clutters the cli) * update snapshots with jest-path-serializer * fix workflow * retrigger ci 🤞 * fix azure tests * revert www/README.md * remove unused src field * convert cache to setPluginStatus * add lazyImageGeneration option * fix yarn * update snapshots * fix image url in test * move develop.js to index.js * revert tests * comment out all the lazy stuff * fix tests * schedule jobs immediately (for now) Signed-off-by: Ward Peeters <ward@coding-tech.com>
gatsbyjs · Mar 11, 2019 · 7aef5ca · 7aef5ca
1 parent 677eda9
commit 7aef5ca
Show file tree

Hide file tree

Showing 21 changed files with 664 additions and 373 deletions.
diff --git a/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js b/integration-tests/gatsby-pipeline/__tests__/lazy-image-build/develop.js
@@ -24,7 +24,7 @@ describe(`Lazy images`, () => {
     const { kill } = await createDevServer()
 
     const response = await request(
-      `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/a2541/gatsby-astronaut.png`,
+      `http://localhost:8000/static/6d91c86c0fde632ba4cd01062fd9ccfa/e4795/gatsby-astronaut.png`,
       {
         resolveWithFullResponse: true,
       }

diff --git a/integration-tests/gatsby-pipeline/gatsby-browser.js b/integration-tests/gatsby-pipeline/gatsby-browser.js
diff --git a/jest.config.js b/jest.config.js
@@ -32,6 +32,7 @@ module.exports = {
   moduleNameMapper: {
     "^highlight.js$": `<rootDir>/node_modules/highlight.js/lib/index.js`,
   },
+  snapshotSerializers: [`jest-serializer-path`],
   collectCoverage: useCoverage,
   coverageReporters: [`json-summary`, `text`, `html`, `cobertura`],
   coverageThreshold: {

diff --git a/package.json b/package.json
@@ -24,6 +24,7 @@
     "jest": "^24.0.0",
     "jest-cli": "^24.0.0",
     "jest-junit": "^6.1.0",
+    "jest-serializer-path": "^0.1.15",
     "lerna": "^3.10.7",
     "lint-staged": "^8.0.4",
     "markdown-magic": "^0.1.25",

diff --git a/packages/gatsby-plugin-sharp/.gitignore b/packages/gatsby-plugin-sharp/.gitignore
@@ -1,4 +1,2 @@
-/index.js
 /node_modules
-/gatsby-node.js
-/duotone.js
+/*.js
diff --git a/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap b/packages/gatsby-plugin-sharp/src/__tests__/__snapshots__/index.js.snap
@@ -15,14 +15,14 @@ Object {
   "aspectRatio": 2.0661764705882355,
   "base64": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAKCAYAAAC0VX7mAAAACXBIWXMAABYlAAAWJQFJUiTwAAABAklEQVQoz61R7WrCQBDM+7+Cv1qhlIogoUWMoH8KrfgOLVXRKDV4d7kkl6/pbsxpqBIo9mBuZ7N3k51bB7TKsrwZVsexyS2rKerwFhYFVI1DnkMQOHIuCaLB+ayuwXlMaApXgvMoxjTU8MIQE4rMx8SnNffUkU/qM/bbiPAeRfgw5n8tB+QgO1kmUvwROdm0kYHmUNoe+NoU2wZTdVjQH7Isg9YRpFIXor8vpGmKjb/FQQgslit8fi3wvQ/OHXLhxRvDHY7Qd5+x3e2udmNzTUN4fZvhoddH566L+8cn9AZuVcvpLR3e9kEAISWEkDAmbRXkGCcJ1r4PRRNPEkPu9Kn2Azs0CqJYU3JKAAAAAElFTkSuQmCC",
   "density": 144,
-  "originalImg": "/static/1234/4e66c/test.png",
+  "originalImg": "/static/1234/c81ba/test.png",
   "originalName": undefined,
   "presentationHeight": 68,
   "presentationWidth": 141,
   "sizes": "(max-width: 141px) 100vw, 141px",
-  "src": "/static/1234/4e66c/test.png",
-  "srcSet": "/static/1234/1c669/test.png 200w,
-/static/1234/4e66c/test.png 281w",
+  "src": "/static/1234/c81ba/test.png",
+  "srcSet": "/static/1234/e4832/test.png 200w,
+/static/1234/c81ba/test.png 281w",
   "srcSetType": "image/png",
   "tracedSVG": undefined,
 }
@@ -33,14 +33,14 @@ Object {
   "aspectRatio": 2.0661764705882355,
   "base64": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAKCAYAAAC0VX7mAAAACXBIWXMAABYlAAAWJQFJUiTwAAABAklEQVQoz61R7WrCQBDM+7+Cv1qhlIogoUWMoH8KrfgOLVXRKDV4d7kkl6/pbsxpqBIo9mBuZ7N3k51bB7TKsrwZVsexyS2rKerwFhYFVI1DnkMQOHIuCaLB+ayuwXlMaApXgvMoxjTU8MIQE4rMx8SnNffUkU/qM/bbiPAeRfgw5n8tB+QgO1kmUvwROdm0kYHmUNoe+NoU2wZTdVjQH7Isg9YRpFIXor8vpGmKjb/FQQgslit8fi3wvQ/OHXLhxRvDHY7Qd5+x3e2udmNzTUN4fZvhoddH566L+8cn9AZuVcvpLR3e9kEAISWEkDAmbRXkGCcJ1r4PRRNPEkPu9Kn2Azs0CqJYU3JKAAAAAElFTkSuQmCC",
   "density": 144,
-  "originalImg": "/static/1234/896a4/test.png",
+  "originalImg": "/static/1234/ec7d1/test.png",
   "originalName": undefined,
   "presentationHeight": 136,
   "presentationWidth": 281,
   "sizes": "(max-width: 281px) 100vw, 281px",
-  "src": "/static/1234/896a4/test.png",
-  "srcSet": "/static/1234/dc730/test.png 200w,
-/static/1234/896a4/test.png 281w",
+  "src": "/static/1234/ec7d1/test.png",
+  "srcSet": "/static/1234/dc107/test.png 200w,
+/static/1234/ec7d1/test.png 281w",
   "srcSetType": "image/png",
   "tracedSVG": undefined,
 }
@@ -51,13 +51,13 @@ Object {
   "aspectRatio": 1,
   "base64": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAUCAIAAAAC64paAAAACXBIWXMAAAsTAAALEwEAmpwYAAAAGElEQVQ4y2P4TwFgGNU8qnlU86jmgdUMAOBEq42KgAyMAAAAAElFTkSuQmCC",
   "density": 72,
-  "originalImg": "/static/1234/d0c1a/test.png",
+  "originalImg": "/static/1234/1581d/test.png",
   "originalName": undefined,
   "presentationHeight": 1,
   "presentationWidth": 1,
   "sizes": "(max-width: 1px) 100vw, 1px",
-  "src": "/static/1234/d0c1a/test.png",
-  "srcSet": "/static/1234/d0c1a/test.png 1w",
+  "src": "/static/1234/1581d/test.png",
+  "srcSet": "/static/1234/1581d/test.png 1w",
   "srcSetType": "image/png",
   "tracedSVG": undefined,
 }
@@ -74,14 +74,78 @@ Object {
 }
 `;
 
+exports[`gatsby-plugin-sharp queueImageResizing should process immediately when asked 1`] = `
+[MockFunction] {
+  "calls": Array [
+    Array [
+      Object {
+        "args": Object {
+          "base64": true,
+          "duotone": false,
+          "grayscale": false,
+          "jpegProgressive": true,
+          "maxWidth": 800,
+          "pathPrefix": "",
+          "pngCompressionLevel": 9,
+          "pngCompressionSpeed": 4,
+          "quality": 50,
+          "sizeByPixelDensity": false,
+          "toFormat": "png",
+          "width": 3,
+        },
+        "inputPath": "<PROJECT_ROOT>/packages/gatsby-plugin-sharp/src/__tests__/images/144-density.png",
+        "outputPath": "<PROJECT_ROOT>/public/static/1234/39ca0/test.png",
+      },
+      Object {
+        "addThirdPartySchema": [Function],
+        "createJob": [Function],
+        "createNode": [Function],
+        "createNodeField": [Function],
+        "createPage": [Function],
+        "createPageDependency": [Function],
+        "createParentChildLink": [Function],
+        "createRedirect": [Function],
+        "deleteComponentsDependencies": [Function],
+        "deleteNode": [Function],
+        "deleteNodes": [Function],
+        "deletePage": [Function],
+        "endJob": [Function],
+        "replaceComponentQuery": [Function],
+        "replaceStaticQuery": [Function],
+        "replaceWebpackConfig": [Function],
+        "setBabelOptions": [Function],
+        "setBabelPlugin": [Function],
+        "setBabelPreset": [Function],
+        "setJob": [Function],
+        "setPluginStatus": [Function],
+        "setWebpackConfig": [Function],
+        "touchNode": [Function],
+      },
+      Object {
+        "defaultQuality": 50,
+        "lazyImageGeneration": true,
+        "stripMetadata": true,
+        "useMozJpeg": false,
+      },
+    ],
+  ],
+  "results": Array [
+    Object {
+      "type": "return",
+      "value": Promise {},
+    },
+  ],
+}
+`;
+
 exports[`gatsby-plugin-sharp tracedSVG runs on demand 1`] = `
 Object {
   "aspectRatio": 1,
   "base64": undefined,
   "height": 1,
   "originalName": undefined,
-  "src": "/static/1234/05eb0/test.png",
-  "srcSet": "/static/1234/05eb0/test.png 1x",
+  "src": "/static/1234/155b3/test.png",
+  "srcSet": "/static/1234/155b3/test.png 1x",
   "tracedSVG": "data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='100' height='100' viewBox='0 0 100 100' version='1'/%3e",
   "width": 1,
 }
@@ -92,13 +156,13 @@ Object {
   "aspectRatio": 1,
   "base64": undefined,
   "density": 72,
-  "originalImg": "/static/1234/c1fac/test.png",
+  "originalImg": "/static/1234/3092c/test.png",
   "originalName": undefined,
   "presentationHeight": 1,
   "presentationWidth": 1,
   "sizes": "(max-width: 1px) 100vw, 1px",
-  "src": "/static/1234/c1fac/test.png",
-  "srcSet": "/static/1234/c1fac/test.png 1w",
+  "src": "/static/1234/3092c/test.png",
+  "srcSet": "/static/1234/3092c/test.png 1w",
   "srcSetType": "image/png",
   "tracedSVG": "data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' width='100' height='100' viewBox='0 0 100 100' version='1'/%3e",
 }

diff --git a/packages/gatsby-plugin-sharp/src/__tests__/index.js b/packages/gatsby-plugin-sharp/src/__tests__/index.js
@@ -1,5 +1,6 @@
 const path = require(`path`)
 const fs = require(`fs-extra`)
+jest.mock(`../scheduler`)
 
 jest.mock(`async/queue`, () => () => {
   return {
@@ -16,6 +17,8 @@ const {
   queueImageResizing,
   getImageSize,
 } = require(`../`)
+const { scheduleJob } = require(`../scheduler`)
+scheduleJob.mockResolvedValue(Promise.resolve())
 
 describe(`gatsby-plugin-sharp`, () => {
   const args = {
@@ -50,7 +53,7 @@ describe(`gatsby-plugin-sharp`, () => {
       // test name encoding with various characters
       const testName = `spaces and '"@#$%^&,`
 
-      const queueResult = await queueImageResizing({
+      const queueResult = queueImageResizing({
         file: getFileObject(
           path.join(__dirname, `images/144-density.png`),
           testName
@@ -68,6 +71,19 @@ describe(`gatsby-plugin-sharp`, () => {
       expect(testName.match(/[!@#$^&," ]/)).not.toBe(false)
       expect(queueResultName.match(/[!@#$^&," ]/)).not.toBe(true)
     })
+
+    // re-enable when image processing on demand is implemented
+    it.skip(`should process immediately when asked`, async () => {
+      scheduleJob.mockClear()
+      const result = queueImageResizing({
+        file: getFileObject(path.join(__dirname, `images/144-density.png`)),
+        args: { width: 3 },
+      })
+
+      await result.finishedPromise
+
+      expect(scheduleJob).toMatchSnapshot()
+    })
   })
 
   describe(`fluid`, () => {

diff --git a/packages/gatsby-plugin-sharp/src/gatsby-node.js b/packages/gatsby-plugin-sharp/src/gatsby-node.js
@@ -1,10 +1,141 @@
-const { setBoundActionCreators, setPluginOptions } = require(`./index`)
+const {
+  setBoundActionCreators,
+  setPluginOptions,
+  // queue: jobQueue,
+  // reportError,
+} = require(`./index`)
+// const { scheduleJob } = require(`./scheduler`)
+// let normalizedOptions = {}
 
-exports.onPreInit = ({ actions }, pluginOptions) => {
+// const getQueueFromCache = store => {
+//   const pluginStatus = store.getState().status.plugins[`gatsby-plugin-sharp`]
+
+//   if (!pluginStatus || !pluginStatus.queue) {
+//     return new Map()
+//   }
+
+//   return new Map(pluginStatus.queue)
+// }
+
+// const saveQueueToCache = async (store, setPluginStatus, queue) => {
+//   const cachedQueue = getQueueFromCache(store)
+
+//   // merge both queues
+//   for (const [key, job] of cachedQueue) {
+//     if (!queue.has(key)) {
+//       queue.set(key, job)
+//     }
+//   }
+
+//   // JSON.stringify doesn't work on an Map so we need to convert it to an array
+//   setPluginStatus({ queue: Array.from(queue) })
+// }
+
+// const processQueue = (store, boundActionCreators, options) => {
+//   const cachedQueue = getQueueFromCache(store)
+
+//   const promises = []
+//   for (const [, job] of cachedQueue) {
+//     promises.push(scheduleJob(job, boundActionCreators, options))
+//   }
+
+//   return promises
+// }
+
+// const cleanupQueueAfterProcess = (imageJobs, setPluginStatus, reporter) =>
+//   Promise.all(imageJobs)
+//     .then(() => setPluginStatus({ queue: [] }))
+//     .catch(({ err, message }) => {
+//       reportError(message || err.message, err, reporter)
+//     })
+
+exports.onPreBootstrap = ({ actions }, pluginOptions) => {
   setBoundActionCreators(actions)
   setPluginOptions(pluginOptions)
+  // normalizedOptions = setPluginOptions(pluginOptions)
 }
 
+// /**
+//  * save queue to the cache or process queue
+//  */
+// exports.onPostBootstrap = ({
+//   // store,
+//   boundActionCreators,
+//   // actions: { setPluginStatus },
+//   // reporter,
+// }) => {
+//   const promises = []
+//   for (const [, job] of jobQueue) {
+//     promises.push(scheduleJob(job, boundActionCreators, normalizedOptions))
+//   }
+
+//   return promises
+//   // // Save queue
+//   // saveQueueToCache(store, setPluginStatus, jobQueue)
+
+//   // if (normalizedOptions.lazyImageGeneration) {
+//   //   return
+//   // }
+
+//   // const imageJobs = processQueue(store, boundActionCreators, normalizedOptions)
+
+//   // cleanupQueueAfterProcess(imageJobs, setPluginStatus, reporter)
+
+//   // return
+// }
+
+/**
+ * Execute all unprocessed images on gatsby build
+ */
+// let promises = []
+// exports.onPreBuild = ({ store, boundActionCreators }) => {
+//   promises = processQueue(store, boundActionCreators, normalizedOptions)
+// }
+
+/**
+ * wait for all images to be processed
+//  */
+// exports.onPostBuild = ({ actions: { setPluginStatus }, reporter }) =>
+//   cleanupQueueAfterProcess(promises, setPluginStatus, reporter)
+
+/**
+ * Build images on the fly when they are requested by the browser
+ */
+// exports.onCreateDevServer = async ({
+//   app,
+//   emitter,
+//   boundActionCreators,
+//   actions: { setPluginStatus },
+//   store,
+// }) => {
+//   // no need to do set things up when people opt out
+//   if (!normalizedOptions.lazyImageGeneration) {
+//     return
+//   }
+
+//   emitter.on(`QUERY_QUEUE_DRAINED`, () =>
+//     saveQueueToCache(store, setPluginStatus, jobQueue)
+//   )
+
+//   app.use(async (req, res, next) => {
+//     const queue = getQueueFromCache(store)
+//     if (!queue.has(req.originalUrl)) {
+//       return next()
+//     }
+
+//     const job = queue.get(req.originalUrl)
+
+//     // wait until the file has been processed and saved to disk
+//     await scheduleJob(job, boundActionCreators, normalizedOptions, false)
+//     // remove job from queue because it has been processed
+//     queue.delete(req.originalUrl)
+
+//     saveQueueToCache(store, setPluginStatus, queue)
+
+//     return res.sendFile(job.outputPath)
+//   })
+// }
+
 // TODO
 // exports.formatJobMessage = jobs => {
 // return {