Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[1.0] make determining when a given stage is finished much more reliable #1080

Merged
merged 13 commits into from
Jun 1, 2017
Merged

[1.0] make determining when a given stage is finished much more reliable #1080

Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
9d7f80d
Use 'traceId' to track when cascading actions are complete
KyleAMathews Jun 1, 2017
0e992c5
Add jobs system + move query running into internal plugins
KyleAMathews Jun 1, 2017
793119b
Fixes, remove logging, disable rebuilding schema in dev
KyleAMathews Jun 1, 2017
d30369c
Remove unused code/packages
KyleAMathews Jun 1, 2017
d918ccd
Rename variables
KyleAMathews Jun 1, 2017
6d10258
Update snapshots
KyleAMathews Jun 1, 2017
a483090
Fix building + change postBuild to onPostBuild
KyleAMathews Jun 1, 2017
2a76f35
Fix Drupal source plugin/site by restoring way for plugins to set sta…
KyleAMathews Jun 1, 2017
4fef002
Remove console.logs
KyleAMathews Jun 1, 2017
4eb84f8
debugging
KyleAMathews Jun 1, 2017
34bdbb9
Use jest moduleNameMapper to fix failing test
KyleAMathews Jun 1, 2017
7137501
Don't query raw field as we don't use it
KyleAMathews Jun 1, 2017
42804c1
Use standard JSDoc notation
KyleAMathews Jun 1, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Rename variables
  • Loading branch information
@KyleAMathews
KyleAMathews committed Jun 1, 2017
commit d918ccdcf6e4c88da8359735f48d35dc6b9cd43d
34 changes: 19 additions & 15 deletions packages/gatsby-plugin-sharp/src/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,17 @@ const bar = new ProgressBar(
}
)

let total = 0
const processFile = (file, jobs, cb) => {
total += jobs.length
bar.total = total
totalJobs += jobs.length
bar.total = _.sumBy(
Object.keys(toProcess),
key => _.values(toProcess[key]).length
)

let imagesFinished = 0

let finished = 0
Promise.all(jobs.map(job => job.finished)).then(() => cb())
// Wait for each job promise to resolve.
Promise.all(jobs.map(job => job.finishedPromise)).then(() => cb())
const pipeline = sharp(file).rotate()
jobs.forEach(async job => {
const args = job.args
Expand Down Expand Up @@ -88,12 +92,12 @@ const processFile = (file, jobs, cb) => {

if (job.file.extension.match(/^jp/)) {
clonedPipeline.toFile(job.outputPath, (err, info) => {
finished += 1
imagesFinished += 1
bar.tick()
boundActionCreators.setJob(
{
id: `processing image ${job.file.absolutePath}`,
finished,
imagesFinished,
},
{ name: `gatsby-plugin-sharp` }
)
Expand All @@ -112,12 +116,12 @@ const processFile = (file, jobs, cb) => {
})
.then(imageminBuffer => {
fs.writeFile(job.outputPath, imageminBuffer, () => {
finished += 1
imagesFinished += 1
bar.tick()
boundActionCreators.setJob(
{
id: `processing image ${job.file.absolutePath}`,
finished,
imagesFinished,
},
{ name: `gatsby-plugin-sharp` }
)
Expand All @@ -129,7 +133,7 @@ const processFile = (file, jobs, cb) => {
})
}

let totalCount = 0
let totalJobs = 0
const toProcess = {}
const q = queue((task, callback) => {
task(callback)
Expand All @@ -150,8 +154,6 @@ const queueJob = job => {
return
}

totalCount += 1

let notQueued = true
if (toProcess[inputFileKey]) {
notQueued = false
Expand All @@ -161,6 +163,8 @@ const queueJob = job => {
`${job.file.absolutePath.replace(/\./g, `%2E`)}.${job.outputPath.replace(/\./g, `%2E`)}`,
job
)

// totalJobs += 1
if (notQueued) {
q.push(cb => {
// console.log("processing image", job.file.absolutePath)
Expand Down Expand Up @@ -231,7 +235,7 @@ function queueImageResizing({ file, args = {} }) {
const filePath = `${process.cwd()}/public${imgSrc}`
// Create function to call when the image is finished.
let outsideResolve
const finished = new Promise(resolve => {
const finishedPromise = new Promise(resolve => {
outsideResolve = resolve
})

Expand All @@ -257,7 +261,7 @@ function queueImageResizing({ file, args = {} }) {
const job = {
file,
args: options,
finished,
finishedPromise,
outsideResolve,
inputPath: file.absolutePath,
outputPath: filePath,
Expand All @@ -274,7 +278,7 @@ function queueImageResizing({ file, args = {} }) {
width,
height,
aspectRatio,
finished,
finishedPromise,
}
}

Expand Down