Skip to content

Commit d3b1ead

Browse files
arunodarauchg
authored andcommitted
Implement "on demand entries" (#1111)
* Add a plan for dynamic entry middleware. * Use dynamic pages middleware to load pages in dev. * Add the first version of middleware but not tested. * Integrated. * Disable prefetching in development. Otherwise it'll discard the use of dynamic-entries. * Build custom document and error always. * Refactor code base. * Change branding as on-demand entries. * Fix tests. * Add a client side pinger for on-demand-entries. * Dispose inactive entries. * Add proper logs. * Update grammer changes. * Add integration tests for ondemand entries. * Improve ondemand entry disposing logic. * Try to improve testing. * Make sure entries are not getting disposed in basic integration tests. * Resolve conflicts. * Fix tests. * Fix issue when running Router.onRouteChangeComplete * Simplify state management. * Make sure we don't dispose the last active page. * Reload invalid pages detected with the client side ping. * Improve the pinger code. * Touch the first page to speed up the future rebuild times. * Add Websockets based pinger. * Revert "Add Websockets based pinger." This reverts commit f706a49. * Do not send requests per every route change. * Make sure we are completing the middleware request always. * Make sure test pages are prebuilt.
1 parent 48d8b24 commit d3b1ead

File tree

17 files changed

+1155
-793
lines changed

17 files changed

+1155
-793
lines changed

client/on-demand-entries-client.js

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/* global location */
2+
3+
import Router from '../lib/router'
4+
import fetch from 'unfetch'
5+
6+
async function ping () {
7+
try {
8+
const url = `/on-demand-entries-ping?page=${Router.pathname}`
9+
const res = await fetch(url)
10+
const payload = await res.json()
11+
if (payload.invalid) {
12+
location.reload()
13+
}
14+
} catch (err) {
15+
console.error(`Error with on-demand-entries-ping: ${err.message}`)
16+
}
17+
}
18+
19+
async function runPinger () {
20+
while (true) {
21+
await new Promise((resolve) => setTimeout(resolve, 5000))
22+
await ping()
23+
}
24+
}
25+
26+
runPinger()
27+
.catch((err) => {
28+
console.error(err)
29+
})

lib/router/router.js

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,10 @@ export default class Router extends EventEmitter {
224224
}
225225

226226
async prefetch (url) {
227+
// We don't add support for prefetch in the development mode.
228+
// If we do that, our on-demand-entries optimization won't performs better
229+
if (process.env.NODE_ENV === 'development') return
230+
227231
const { pathname } = parse(url)
228232
const route = toRoute(pathname)
229233
return this.prefetchQueue.add(() => this.fetchRoute(route))

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,7 @@
7878
"source-map-support": "0.4.11",
7979
"strip-ansi": "3.0.1",
8080
"styled-jsx": "0.5.7",
81+
"touch": "1.0.0",
8182
"unfetch": "2.1.0",
8283
"url": "0.11.0",
8384
"uuid": "3.0.1",

server/build/webpack.js

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import WriteFilePlugin from 'write-file-webpack-plugin'
66
import FriendlyErrorsWebpackPlugin from 'friendly-errors-webpack-plugin'
77
import CaseSensitivePathPlugin from 'case-sensitive-paths-webpack-plugin'
88
import UnlinkFilePlugin from './plugins/unlink-file-plugin'
9-
import WatchPagesPlugin from './plugins/watch-pages-plugin'
109
import JsonPagesPlugin from './plugins/json-pages-plugin'
1110
import getConfig from '../config'
1211
import * as babelCore from 'babel-core'
@@ -40,8 +39,18 @@ export default async function createCompiler (dir, { dev = false, quiet = false,
4039
const entries = { 'main.js': mainJS }
4140

4241
const pages = await glob('pages/**/*.js', { cwd: dir })
43-
for (const p of pages) {
44-
entries[join('bundles', p)] = [...defaultEntries, `./${p}?entry`]
42+
const devPages = pages.filter((p) => p === 'pages/_document.js' || p === 'pages/_error.js')
43+
44+
// In the dev environment, on-demand-entry-handler will take care of
45+
// managing pages.
46+
if (dev) {
47+
for (const p of devPages) {
48+
entries[join('bundles', p)] = [...defaultEntries, `./${p}?entry`]
49+
}
50+
} else {
51+
for (const p of pages) {
52+
entries[join('bundles', p)] = [...defaultEntries, `./${p}?entry`]
53+
}
4554
}
4655

4756
for (const p of defaultPages) {
@@ -81,6 +90,9 @@ export default async function createCompiler (dir, { dev = false, quiet = false,
8190
return count >= minChunks
8291
}
8392
}),
93+
new webpack.DefinePlugin({
94+
'process.env.NODE_ENV': JSON.stringify(dev ? 'development' : 'production')
95+
}),
8496
new JsonPagesPlugin(),
8597
new CaseSensitivePathPlugin()
8698
]
@@ -89,17 +101,13 @@ export default async function createCompiler (dir, { dev = false, quiet = false,
89101
plugins.push(
90102
new webpack.HotModuleReplacementPlugin(),
91103
new webpack.NoEmitOnErrorsPlugin(),
92-
new UnlinkFilePlugin(),
93-
new WatchPagesPlugin(dir)
104+
new UnlinkFilePlugin()
94105
)
95106
if (!quiet) {
96107
plugins.push(new FriendlyErrorsWebpackPlugin())
97108
}
98109
} else {
99110
plugins.push(
100-
new webpack.DefinePlugin({
101-
'process.env.NODE_ENV': JSON.stringify('production')
102-
}),
103111
new webpack.optimize.UglifyJsPlugin({
104112
compress: { warnings: false },
105113
sourceMap: false

server/hot-reloader.js

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,12 @@
11
import { join, relative, sep } from 'path'
22
import webpackDevMiddleware from 'webpack-dev-middleware'
33
import webpackHotMiddleware from 'webpack-hot-middleware'
4+
import onDemandEntryHandler from './on-demand-entry-handler'
45
import isWindowsBash from 'is-windows-bash'
56
import webpack from './build/webpack'
67
import clean from './build/clean'
78
import readPage from './read-page'
9+
import getConfig from './config'
810

911
export default class HotReloader {
1012
constructor (dir, { quiet } = {}) {
@@ -20,6 +22,8 @@ export default class HotReloader {
2022
this.prevChunkNames = null
2123
this.prevFailedChunkNames = null
2224
this.prevChunkHashes = null
25+
26+
this.config = getConfig(dir)
2327
}
2428

2529
async run (req, res) {
@@ -145,10 +149,16 @@ export default class HotReloader {
145149
})
146150

147151
this.webpackHotMiddleware = webpackHotMiddleware(compiler, { log: false })
152+
this.onDemandEntries = onDemandEntryHandler(this.webpackDevMiddleware, compiler, {
153+
dir: this.dir,
154+
dev: true,
155+
...this.config.onDemandEntries
156+
})
148157

149158
this.middlewares = [
150159
this.webpackDevMiddleware,
151-
this.webpackHotMiddleware
160+
this.webpackHotMiddleware,
161+
this.onDemandEntries.middleware()
152162
]
153163
}
154164

@@ -184,6 +194,10 @@ export default class HotReloader {
184194
send (action, ...args) {
185195
this.webpackHotMiddleware.publish({ action, data: args })
186196
}
197+
198+
ensurePage (page) {
199+
return this.onDemandEntries.ensurePage(page)
200+
}
187201
}
188202

189203
function deleteCache (path) {

server/index.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@ export default class Server {
2222
this.dir = resolve(dir)
2323
this.dev = dev
2424
this.quiet = quiet
25-
this.renderOpts = { dir: this.dir, dev, staticMarkup }
2625
this.router = new Router()
2726
this.hotReloader = dev ? new HotReloader(this.dir, { quiet }) : null
27+
this.renderOpts = { dir: this.dir, dev, staticMarkup, hotReloader: this.hotReloader }
2828
this.http = null
2929
this.config = getConfig(this.dir)
3030

@@ -107,6 +107,7 @@ export default class Server {
107107

108108
const paths = params.path || ['index']
109109
const pathname = `/${paths.join('/')}`
110+
110111
await this.renderJSON(req, res, pathname)
111112
},
112113

server/on-demand-entry-handler.js

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,184 @@
1+
import DynamicEntryPlugin from 'webpack/lib/DynamicEntryPlugin'
2+
import { EventEmitter } from 'events'
3+
import { join } from 'path'
4+
import { parse } from 'url'
5+
import resolvePath from './resolve'
6+
import touch from 'touch'
7+
8+
const ADDED = Symbol()
9+
const BUILDING = Symbol()
10+
const BUILT = Symbol()
11+
12+
export default function onDemandEntryHandler (devMiddleware, compiler, {
13+
dir,
14+
dev,
15+
maxInactiveAge = 1000 * 25
16+
}) {
17+
const entries = {}
18+
const lastAccessPages = ['']
19+
const doneCallbacks = new EventEmitter()
20+
let touchedAPage = false
21+
22+
compiler.plugin('make', function (compilation, done) {
23+
const allEntries = Object.keys(entries).map((page) => {
24+
const { name, entry } = entries[page]
25+
entries[page].status = BUILDING
26+
return addEntry(compilation, this.context, name, entry)
27+
})
28+
29+
Promise.all(allEntries)
30+
.then(() => done())
31+
.catch(done)
32+
})
33+
34+
compiler.plugin('done', function (stats) {
35+
// Call all the doneCallbacks
36+
Object.keys(entries).forEach((page) => {
37+
const entryInfo = entries[page]
38+
if (entryInfo.status !== BUILDING) return
39+
40+
// With this, we are triggering a filesystem based watch trigger
41+
// It'll memorize some timestamp related info related to common files used
42+
// in the page
43+
// That'll reduce the page building time significantly.
44+
if (!touchedAPage) {
45+
setTimeout(() => {
46+
touch.sync(entryInfo.pathname)
47+
}, 0)
48+
touchedAPage = true
49+
}
50+
51+
entryInfo.status = BUILT
52+
entries[page].lastActiveTime = Date.now()
53+
doneCallbacks.emit(page)
54+
})
55+
})
56+
57+
setInterval(function () {
58+
disposeInactiveEntries(devMiddleware, entries, lastAccessPages, maxInactiveAge)
59+
}, 5000)
60+
61+
return {
62+
async ensurePage (page) {
63+
page = normalizePage(page)
64+
65+
const pagePath = join(dir, 'pages', page)
66+
const pathname = await resolvePath(pagePath)
67+
const name = join('bundles', pathname.substring(dir.length))
68+
69+
const entry = [
70+
join(__dirname, '..', 'client/webpack-hot-middleware-client'),
71+
join(__dirname, '..', 'client', 'on-demand-entries-client'),
72+
`${pathname}?entry`
73+
]
74+
75+
await new Promise((resolve, reject) => {
76+
const entryInfo = entries[page]
77+
78+
if (entryInfo) {
79+
if (entryInfo.status === BUILT) {
80+
resolve()
81+
return
82+
}
83+
84+
if (entryInfo.status === BUILDING) {
85+
doneCallbacks.on(page, processCallback)
86+
return
87+
}
88+
}
89+
90+
console.log(`> Building page: ${page}`)
91+
92+
entries[page] = { name, entry, pathname, status: ADDED }
93+
doneCallbacks.on(page, processCallback)
94+
95+
devMiddleware.invalidate()
96+
97+
function processCallback (err) {
98+
if (err) return reject(err)
99+
resolve()
100+
}
101+
})
102+
},
103+
104+
middleware () {
105+
return function (req, res, next) {
106+
if (!/^\/on-demand-entries-ping/.test(req.url)) return next()
107+
108+
const { query } = parse(req.url, true)
109+
const page = normalizePage(query.page)
110+
const entryInfo = entries[page]
111+
112+
// If there's no entry.
113+
// Then it seems like an weird issue.
114+
if (!entryInfo) {
115+
const message = `Client pings, but there's no entry for page: ${page}`
116+
console.error(message)
117+
sendJson(res, { invalid: true })
118+
return
119+
}
120+
121+
sendJson(res, { success: true })
122+
123+
// We don't need to maintain active state of anything other than BUILT entries
124+
if (entryInfo.status !== BUILT) return
125+
126+
// If there's an entryInfo
127+
lastAccessPages.pop()
128+
lastAccessPages.unshift(page)
129+
entryInfo.lastActiveTime = Date.now()
130+
}
131+
}
132+
}
133+
}
134+
135+
function addEntry (compilation, context, name, entry) {
136+
return new Promise((resolve, reject) => {
137+
const dep = DynamicEntryPlugin.createDependency(entry, name)
138+
compilation.addEntry(context, dep, name, (err) => {
139+
if (err) return reject(err)
140+
resolve()
141+
})
142+
})
143+
}
144+
145+
function disposeInactiveEntries (devMiddleware, entries, lastAccessPages, maxInactiveAge) {
146+
const disposingPages = []
147+
148+
Object.keys(entries).forEach((page) => {
149+
const { lastActiveTime, status } = entries[page]
150+
151+
// This means this entry is currently building or just added
152+
// We don't need to dispose those entries.
153+
if (status !== BUILT) return
154+
155+
// We should not build the last accessed page even we didn't get any pings
156+
// Sometimes, it's possible our XHR ping to wait before completing other requests.
157+
// In that case, we should not dispose the current viewing page
158+
if (lastAccessPages[0] === page) return
159+
160+
if (Date.now() - lastActiveTime > maxInactiveAge) {
161+
disposingPages.push(page)
162+
}
163+
})
164+
165+
if (disposingPages.length > 0) {
166+
disposingPages.forEach((page) => {
167+
delete entries[page]
168+
})
169+
console.log(`> Disposing inactive page(s): ${disposingPages.join(', ')}`)
170+
devMiddleware.invalidate()
171+
}
172+
}
173+
174+
// /index and / is the same. So, we need to identify both pages as the same.
175+
// This also applies to sub pages as well.
176+
function normalizePage (page) {
177+
return page.replace(/\/index$/, '/')
178+
}
179+
180+
function sendJson (res, payload) {
181+
res.setHeader('Content-Type', 'application/json')
182+
res.status = 200
183+
res.end(JSON.stringify(payload))
184+
}

server/render.js

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,11 +32,15 @@ async function doRender (req, res, pathname, query, {
3232
err,
3333
page,
3434
buildId,
35+
hotReloader,
3536
dir = process.cwd(),
3637
dev = false,
3738
staticMarkup = false
3839
} = {}) {
3940
page = page || pathname
41+
42+
await ensurePage(page, { dir, hotReloader })
43+
4044
let [Component, Document] = await Promise.all([
4145
requireModule(join(dir, '.next', 'dist', 'pages', page)),
4246
requireModule(join(dir, '.next', 'dist', 'pages', '_document'))
@@ -100,7 +104,8 @@ async function doRender (req, res, pathname, query, {
100104
return '<!DOCTYPE html>' + renderToStaticMarkup(doc)
101105
}
102106

103-
export async function renderJSON (req, res, page, { dir = process.cwd() } = {}) {
107+
export async function renderJSON (req, res, page, { dir = process.cwd(), hotReloader } = {}) {
108+
await ensurePage(page, { dir, hotReloader })
104109
const pagePath = await resolvePath(join(dir, '.next', 'bundles', 'pages', page))
105110
return serveStatic(req, res, pagePath)
106111
}
@@ -158,3 +163,10 @@ export function serveStatic (req, res, path) {
158163
.on('finish', resolve)
159164
})
160165
}
166+
167+
async function ensurePage (page, { dir, hotReloader }) {
168+
if (!hotReloader) return
169+
if (page === '_error' || page === '_document') return
170+
171+
await hotReloader.ensurePage(page)
172+
}

0 commit comments

Comments
 (0)