Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Replace global data.json with page-data.json per page #13004

Closed
wants to merge 44 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
4e6091e
refactor query runner to allow running static/page queries separately
Moocar Mar 24, 2019
071320f
documenting
Moocar Mar 26, 2019
722c45a
Create queue configurations for build/develop
Moocar Mar 27, 2019
8868072
moved page-query-runner to index
Moocar Mar 24, 2019
d7dc21e
move src/internal-plugins/query-runner to src/query
Moocar Mar 27, 2019
6f036a5
add match paths json
Moocar Mar 23, 2019
330b2ba
write page data on query completion
Moocar Mar 24, 2019
034cc78
run page queries after bootstrap
Moocar Mar 25, 2019
9caff6a
use page data instead of data.json
Moocar Mar 25, 2019
d64e6bf
add webpack compilation hash
Moocar Mar 25, 2019
2b1b3c6
use compilation hash to refresh production-app
Moocar Mar 25, 2019
64bd2b5
remove develop-html (use build-html)
Moocar Mar 25, 2019
f3292ff
query-handler -> job handler. And ref ws manager from develop.js
Moocar Mar 25, 2019
738fbb6
invoke dev-404 correctly. Fix loader to handle develop mode
Moocar Mar 26, 2019
854e7b2
post merge fixes
Moocar Mar 28, 2019
54f79eb
only refresh page on compilation hash in production
Moocar Mar 28, 2019
05fe38a
get fetch 404 working
Moocar Mar 28, 2019
76e0b29
serve pages from gatsby develop
Moocar Mar 28, 2019
f35ae9c
404 page is fully working. Introduced loadPage, getPage
Moocar Mar 28, 2019
21aa5b8
bump gatsbygram gatsby version
Moocar Mar 29, 2019
7a12d71
misc fixes
Moocar Mar 30, 2019
ae210a9
fix 404 loader race condition
Moocar Mar 31, 2019
13204aa
add dev loader for 404 pages
Moocar Mar 31, 2019
ae5ddab
deprecate loader functions (getResources...)
Moocar Apr 1, 2019
406f26d
app load pages in parallel
Moocar Apr 1, 2019
21c325e
cleaning up for PR review
Moocar Apr 1, 2019
8a0ed10
remove failed history stuff
Moocar Apr 1, 2019
dcd65ea
cleanup for PR
Moocar Apr 1, 2019
6dd6e1e
standardize staticQueryResult.result
Moocar Apr 1, 2019
6d7504e
dataAndContext => pageData
Moocar Apr 1, 2019
a148f19
add activity back to build html pages
Moocar Apr 1, 2019
ee6a4c8
remove jsonName from Gatsby
Moocar Apr 1, 2019
1ed5e9f
add loadPageDataSync for guess
Moocar Apr 1, 2019
2c9b446
clean up bootstrap
Moocar Apr 1, 2019
ca2b0c4
explicitly save db state post bootstrap and develop
Moocar Apr 1, 2019
2af3786
watch for page deletes during develop
Moocar Apr 1, 2019
4b158e6
move query running out of bootstrap
Moocar Apr 1, 2019
7110cb7
ascertainPort => selectPort
Moocar Apr 1, 2019
2ed38d5
processPageQueries and processStaticQueries
Moocar Apr 1, 2019
8e90225
rewrite compilation hashes when changed
Moocar Apr 1, 2019
227b683
perform page queries before js compilation in develop
Moocar Apr 1, 2019
19fbb00
daemon -> listener
Moocar Apr 1, 2019
bc7118f
minor
Moocar Apr 1, 2019
ce46689
Merge branch 'master' into per-page-manifest2
Moocar Apr 3, 2019
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
processPageQueries and processStaticQueries
  • Loading branch information
Moocar committed Apr 1, 2019
commit 2ed38d5eae44465e738f4598ade498dc72a04928
25 changes: 7 additions & 18 deletions packages/gatsby/src/commands/build.js
Original file line number Diff line number Diff line change
Expand Up @@ -28,18 +28,6 @@ type BuildArgs = {
openTracingConfigFile: string,
}

async function runStaticQueries({ parentSpan }, staticQueryIds) {
let activity = report.activityTimer(`run static queries`, { parentSpan })
activity.start()
await queryRunner.processQueries(
staticQueryIds.map(id =>
queryRunner.makeStaticQueryJob(store.getState(), id)
),
{ activity }
)
activity.end()
}

module.exports = async function build(program: BuildArgs) {
let activity
initTracer(program.openTracingConfigFile)
Expand All @@ -60,7 +48,12 @@ module.exports = async function build(program: BuildArgs) {
const queryIds = queryRunner.calcBootstrapDirtyQueryIds(store.getState())
const { staticQueryIds, pageQueryIds } = queryRunner.groupQueryIds(queryIds)

await runStaticQueries({ parentSpan: buildSpan }, staticQueryIds)
activity = report.activityTimer(`run static queries`, {
parentSpan: buildSpan,
})
activity.start()
await queryRunner.processStaticQueries(staticQueryIds, { activity })
activity.end()

await apiRunnerNode(`onPreBuild`, {
graphql: graphqlRunner,
Expand All @@ -87,13 +80,9 @@ module.exports = async function build(program: BuildArgs) {
payload: webpackCompilationHash,
})

const state = store.getState()
activity = report.activityTimer(`run page queries`)
activity.start()
await queryRunner.processQueries(
pageQueryIds.map(id => queryRunner.makePageQueryJob(state, id)),
{ activity }
)
await queryRunner.processPageQueries(pageQueryIds, { activity })
activity.end()

const waitJobsFinished = () =>
Expand Down
20 changes: 5 additions & 15 deletions packages/gatsby/src/commands/develop.js
Original file line number Diff line number Diff line change
Expand Up @@ -102,23 +102,10 @@ function startQueryDaemon() {
queryRunner.startDaemon(queue)
}

async function runStaticQueries(queryIds) {
let activity = report.activityTimer(`run static queries`)
activity.start()
await queryRunner.processQueries(
queryIds.map(id => queryRunner.makeStaticQueryJob(store.getState(), id)),
{ activity }
)
activity.end()
}

const runPageQueries = async queryIds => {
let activity = report.activityTimer(`run page queries`)
activity.start()
await queryRunner.processQueries(
queryIds.map(id => queryRunner.makePageQueryJob(store.getState(), id)),
{ activity }
)
await queryRunner.processPageQueries(queryIds, { activity })
activity.end()

require(`../redux/actions`).boundActionCreators.setProgramStatus(
Expand Down Expand Up @@ -556,7 +543,10 @@ module.exports = async (program: any) => {

const queryIds = queryRunner.calcBootstrapDirtyQueryIds(store.getState())
const { staticQueryIds, pageQueryIds } = queryRunner.groupQueryIds(queryIds)
await runStaticQueries(staticQueryIds)
let activity = report.activityTimer(`run static queries`)
activity.start()
await queryRunner.processStaticQueries(staticQueryIds, { activity })
activity.end()

const [compiler] = await startServer(program)

Expand Down
30 changes: 23 additions & 7 deletions packages/gatsby/src/query/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -187,11 +187,13 @@ const processQueries = async (queryJobs, { activity }) => {
const queue = queryQueue.create()
queue.on(`task_finish`, () => {
const stats = queue.getStats()
activity.setStatus(
`${stats.total}/${stats.peak} ${(
stats.total / convertHrtime(process.hrtime(startQueries)).seconds
).toFixed(2)} queries/second`
)
if (activity) {
activity.setStatus(
`${stats.total}/${stats.peak} ${(
stats.total / convertHrtime(process.hrtime(startQueries)).seconds
).toFixed(2)} queries/second`
)
}
})
const drainedPromise = new Promise(resolve => {
queue.once(`drain`, resolve)
Expand All @@ -203,6 +205,20 @@ const processQueries = async (queryJobs, { activity }) => {
await drainedPromise
}

const processPageQueries = async (queryIds, { state, activity }) => {
state = state || store.getState()
await processQueries(queryIds.map(id => makePageQueryJob(state, id)), {
activity,
})
}

const processStaticQueries = async (queryIds, { state, activity }) => {
state = state || store.getState()
await processQueries(queryIds.map(id => makeStaticQueryJob(state, id)), {
activity,
})
}

/////////////////////////////////////////////////////////////////////
// Background query daemon (for gatsby develop)

Expand Down Expand Up @@ -254,9 +270,9 @@ module.exports = {
enqueueExtractedQueryId,
calcBootstrapDirtyQueryIds,
groupQueryIds,
makeStaticQueryJob,
makePageQueryJob,
processQueries,
processPageQueries,
processStaticQueries,
runQueries,
startDaemon,
createQueue: queryQueue.create,
Expand Down