Simplify build and js serving file structures

A number of different changes here that are unfortunately tied.

1. Move all client source into .next/bundles dir
2. Simplify serving logic where possible to map directly to these paths
3. Define buildId at the start of the build and tag dynamic chunk loading with this.
4. Avoid creating duplicate chunks with the same content when loaded via different explicit require paths.

These changes also make it much easier for 

Potentially breaking changes:
1. Requests for page javascript are now *.js. / is special cased to be index.js.
2. Composite source files are not filtered based on dev or not
3. _error/index.js -> _error.js

Author: kpdecker

server/build/babel/plugins/handle-import.js

@@ -2,7 +2,7 @@
 // We've added support for SSR with this version
 import template from 'babel-template'
 import syntax from 'babel-plugin-syntax-dynamic-import'
-import UUID from 'uuid'
+import { dirname, relative, resolve } from 'path'
 
 const TYPE_IMPORT = 'Import'
 
@@ -43,10 +43,15 @@ export default () => ({
   visitor: {
     CallExpression (path) {
       if (path.node.callee.type === TYPE_IMPORT) {
+        const { opts } = path.hub.file
+
         const moduleName = path.node.arguments[0].value
-        const name = `${moduleName.replace(/[^\w]/g, '-')}-${UUID.v4()}`
+        const currentDir = dirname(opts.filename)
+        const modulePath = resolve(currentDir, moduleName)
+        const chunkName = relative(opts.sourceRoot, modulePath).replace(/[^\w]/g, '-')
+
         const newImport = buildImport({
-          name
+          name: chunkName
         })({
           SOURCE: path.node.arguments
         })

server/build/index.js

@@ -9,12 +9,14 @@ import md5File from 'md5-file/promise'
 
 export default async function build (dir, conf = null) {
   const buildDir = join(tmpdir(), uuid.v4())
-  const compiler = await webpack(dir, { buildDir, conf })
+  const buildId = uuid.v4()
+
+  const compiler = await webpack(dir, { buildDir, buildId, conf })
 
   try {
     await runCompiler(compiler)
     await writeBuildStats(buildDir)
-    await writeBuildId(buildDir)
+    await writeBuildId(buildDir, buildId)
   } catch (err) {
     console.error(`> Failed to build on ${buildDir}`)
     throw err
@@ -52,15 +54,14 @@ async function writeBuildStats (dir) {
   // So, we need to generate the hash ourself.
   const assetHashMap = {
     'app.js': {
-      hash: await md5File(join(dir, '.next', 'app.js'))
+      hash: await md5File(join(dir, '.next', 'bundles', 'app.js'))
     }
   }
   const buildStatsPath = join(dir, '.next', 'build-stats.json')
   await fs.writeFile(buildStatsPath, JSON.stringify(assetHashMap), 'utf8')
 }
 
-async function writeBuildId (dir) {
+async function writeBuildId (dir, buildId) {
   const buildIdPath = join(dir, '.next', 'BUILD_ID')
-  const buildId = uuid.v4()
   await fs.writeFile(buildIdPath, buildId, 'utf8')
 }

server/build/plugins/dynamic-chunks-plugin.js

@@ -26,12 +26,6 @@ export default class PagesPlugin {
           source: () => newContent,
           size: () => newContent.length
         }
-
-        // This is to support, webpack dynamic import support with HMR
-        compilation.assets[`chunks/${chunk.id}`] = {
-          source: () => newContent,
-          size: () => newContent.length
-        }
       })
       callback()
     })

server/build/plugins/watch-pages-plugin.js

@@ -10,7 +10,7 @@ export default class WatchPagesPlugin {
       compilation.plugin('optimize-assets', (assets, callback) => {
         // transpile pages/_document.js and descendants,
         // but don't need the bundle file
-        delete assets[join('bundles', 'pages', '_document.js')]
+        delete assets[join('pages', '_document.js')]
         callback()
       })
     })

server/build/webpack.js

@@ -27,7 +27,7 @@ const interpolateNames = new Map(defaultPages.map((p) => {
 
 const relativeResolve = rootModuleRelativePath(require)
 
-export default async function createCompiler (dir, { dev = false, quiet = false, buildDir, conf = null } = {}) {
+export default async function createCompiler (dir, { dev = false, quiet = false, buildDir, buildId = 'development', conf = null } = {}) {
   dir = resolve(dir)
   const config = getConfig(dir, conf)
   const defaultEntries = dev ? [
@@ -54,16 +54,16 @@ export default async function createCompiler (dir, { dev = false, quiet = false,
     // managing pages.
     if (dev) {
       for (const p of devPages) {
-        entries[join('bundles', p)] = [`./${p}?entry`]
+        entries[p] = [`./${p}?entry`]
       }
     } else {
       for (const p of pages) {
-        entries[join('bundles', p)] = [`./${p}?entry`]
+        entries[p] = [`./${p}?entry`]
       }
     }
 
     for (const p of defaultPages) {
-      const entryName = join('bundles', 'pages', p)
+      const entryName = join('pages', p)
       if (!entries[entryName]) {
         entries[entryName] = [join(nextPagesDir, p) + '?entry']
       }
@@ -207,7 +207,7 @@ export default async function createCompiler (dir, { dev = false, quiet = false,
       return /node_modules/.test(str) && str.indexOf(nextPagesDir) !== 0
     },
     options: {
-      name: 'dist/[path][name].[ext]',
+      name: '../dist/[path][name].[ext]',
       // By default, our babel config does not transpile ES2015 module syntax because
       // webpack knows how to handle them. (That's how it can do tree-shaking)
       // But Node.js doesn't know how to handle them. So, we have to transpile them here.
@@ -279,10 +279,10 @@ export default async function createCompiler (dir, { dev = false, quiet = false,
     context: dir,
     entry,
     output: {
-      path: buildDir ? join(buildDir, '.next') : join(dir, config.distDir),
+      path: buildDir ? join(buildDir, '.next', 'bundles') : join(dir, config.distDir, 'bundles'),
       filename: '[name]',
       libraryTarget: 'commonjs2',
-      publicPath: '/_next/webpack/',
+      publicPath: `/_next/${buildId}/`,
       strictModuleExceptionHandling: true,
       devtoolModuleFilenameTemplate ({ resourcePath }) {
         const hash = createHash('sha1')

server/document.js

@@ -67,12 +67,12 @@ export class Head extends Component {
 
   getPreloadDynamicChunks () {
     const { chunks, __NEXT_DATA__ } = this.context._documentProps
-    let { assetPrefix } = __NEXT_DATA__
+    let { assetPrefix, buildId } = __NEXT_DATA__
     return chunks.map((chunk) => (
       <link
         key={chunk}
         rel='preload'
-        href={`${assetPrefix}/_next/webpack/chunks/${chunk}`}
+        href={`${assetPrefix}/_next/${buildId}/chunks/${chunk}`}
         as='script'
       />
     ))
@@ -85,7 +85,7 @@ export class Head extends Component {
 
     return <head {...this.props}>
       <link rel='preload' href={`${assetPrefix}/_next/${buildId}/page${pagePathname}`} as='script' />
-      <link rel='preload' href={`${assetPrefix}/_next/${buildId}/page/_error/index.js`} as='script' />
+      <link rel='preload' href={`${assetPrefix}/_next/${buildId}/page/_error.js`} as='script' />
       {this.getPreloadDynamicChunks()}
       {this.getPreloadMainLinks()}
       {(head || []).map((h, i) => React.cloneElement(h, { key: i }))}
@@ -148,15 +148,15 @@ export class NextScript extends Component {
 
   getDynamicChunks () {
     const { chunks, __NEXT_DATA__ } = this.context._documentProps
-    let { assetPrefix } = __NEXT_DATA__
+    let { assetPrefix, buildId } = __NEXT_DATA__
     return (
       <div>
         {chunks.map((chunk) => (
           <script
             async
             key={chunk}
             type='text/javascript'
-            src={`${assetPrefix}/_next/webpack/chunks/${chunk}`}
+            src={`${assetPrefix}/_next/${buildId}/chunks/${chunk}`}
           />
         ))}
       </div>
@@ -188,15 +188,16 @@ export class NextScript extends Component {
         `
       }} />}
       <script async id={`__NEXT_PAGE__${pathname}`} type='text/javascript' src={`${assetPrefix}/_next/${buildId}/page${pagePathname}`} />
-      <script async id={`__NEXT_PAGE__/_error`} type='text/javascript' src={`${assetPrefix}/_next/${buildId}/page/_error/index.js`} />
+      <script async defer id={`__NEXT_PAGE__/_error`} type='text/javascript' src={`${assetPrefix}/_next/${buildId}/page/_error.js`} />
       {staticMarkup ? null : this.getDynamicChunks()}
       {staticMarkup ? null : this.getScripts()}
     </div>
   }
 }
 
 function getPagePathname (pathname, nextExport) {
-  if (!nextExport) return pathname
   if (pathname === '/') return '/index.js'
+
+  if (!nextExport) return `${pathname}.js`
   return `${pathname}/index.js`
 }

server/hot-reloader.js

@@ -40,7 +40,7 @@ export default class HotReloader {
 
   async start () {
     const [compiler] = await Promise.all([
-      webpack(this.dir, { dev: true, quiet: this.quiet }),
+      webpack(this.dir, { dev: true, buildId: 'hmr', quiet: this.quiet }),
       clean(this.dir)
     ])
 
@@ -66,7 +66,7 @@ export default class HotReloader {
     this.stats = null
 
     const [compiler] = await Promise.all([
-      webpack(this.dir, { dev: true, quiet: this.quiet }),
+      webpack(this.dir, { dev: true, buildId: 'hmr', quiet: this.quiet }),
       clean(this.dir)
     ])
 
@@ -173,7 +173,7 @@ export default class HotReloader {
     ]
 
     let webpackDevMiddlewareConfig = {
-      publicPath: '/_next/webpack/',
+      publicPath: '/_next/hmr/',
       noInfo: true,
       quiet: true,
       clientLogLevel: 'warning',

server/index.js

@@ -114,68 +114,12 @@ export default class Server {
         await this.serveStatic(req, res, p)
       },
 
-      // This is to support, webpack dynamic imports in production.
-      '/_next/webpack/chunks/:name': async (req, res, params) => {
-        res.setHeader('Cache-Control', 'max-age=365000000, immutable')
-        const p = join(this.dir, this.dist, 'chunks', params.name)
-        await this.serveStatic(req, res, p)
-      },
-
-      // This is to support, webpack dynamic import support with HMR
-      '/_next/webpack/:id': async (req, res, params) => {
-        const p = join(this.dir, this.dist, 'chunks', params.id)
-        await this.serveStatic(req, res, p)
-      },
-
-      '/_next/:hash/manifest.js': async (req, res, params) => {
-        if (!this.dev) return this.send404(res)
-
-        this.handleBuildHash('manifest.js', params.hash, res)
-        const p = join(this.dir, this.dist, 'manifest.js')
-        await this.serveStatic(req, res, p)
-      },
-
-      '/_next/:hash/main.js': async (req, res, params) => {
-        if (!this.dev) return this.send404(res)
-
-        this.handleBuildHash('main.js', params.hash, res)
-        const p = join(this.dir, this.dist, 'main.js')
-        await this.serveStatic(req, res, p)
-      },
-
-      '/_next/:hash/commons.js': async (req, res, params) => {
-        if (!this.dev) return this.send404(res)
-
-        this.handleBuildHash('commons.js', params.hash, res)
-        const p = join(this.dir, this.dist, 'commons.js')
-        await this.serveStatic(req, res, p)
-      },
-
-      '/_next/:hash/app.js': async (req, res, params) => {
-        if (this.dev) return this.send404(res)
-
-        this.handleBuildHash('app.js', params.hash, res)
-        const p = join(this.dir, this.dist, 'app.js')
-        await this.serveStatic(req, res, p)
-      },
-
-      '/_next/:buildId/page/_error*': async (req, res, params) => {
-        if (!this.handleBuildId(params.buildId, res)) {
-          const error = new Error('INVALID_BUILD_ID')
-          const customFields = { buildIdMismatched: true }
-
-          return await renderScriptError(req, res, '/_error', error, customFields, this.renderOpts)
-        }
-
-        const p = join(this.dir, `${this.dist}/bundles/pages/_error.js`)
-        await this.serveStatic(req, res, p)
-      },
-
-      '/_next/:buildId/page/:path*': async (req, res, params) => {
+      '/_next/:hash/page/:path*': async (req, res, params) => {
         const paths = params.path || ['']
         const page = `/${paths.join('/')}`
+        const filename = `pages/${page.replace(/\.js$/, '')}.js`
 
-        if (!this.handleBuildId(params.buildId, res)) {
+        if (!this.handleBuildHash(filename, params.hash, res)) {
           const error = new Error('INVALID_BUILD_ID')
           const customFields = { buildIdMismatched: true }
 
@@ -199,6 +143,20 @@ export default class Server {
         await renderScript(req, res, page, this.renderOpts)
       },
 
+      '/_next/:hash/:name': async (req, res, params) => {
+        if (!this.dev) return this.send404(res)
+
+        if (!this.handleBuildHash(params.name, params.hash, res)) {
+          const error = new Error('INVALID_BUILD_ID')
+          const customFields = { buildIdMismatched: true }
+
+          return await renderScriptError(req, res, params.name, error, customFields, this.renderOpts)
+        }
+
+        const p = join(this.dir, this.dist, 'bundles', params.name)
+        await this.serveStatic(req, res, p)
+      },
+
       // It's very important keep this route's param optional.
       // (but it should support as many as params, seperated by '/')
       // Othewise this will lead to a pretty simple DOS attack.
@@ -379,16 +337,6 @@ export default class Server {
     return buildId.trim()
   }
 
-  handleBuildId (buildId, res) {
-    if (this.dev) return true
-    if (buildId !== this.renderOpts.buildId) {
-      return false
-    }
-
-    res.setHeader('Cache-Control', 'max-age=365000000, immutable')
-    return true
-  }
-
   async getCompilationError () {
     if (!this.hotReloader) return
 
@@ -400,13 +348,15 @@ export default class Server {
   }
 
   handleBuildHash (filename, hash, res) {
-    if (this.dev) return
+    if (this.dev) return true
 
-    if (hash !== this.buildStats[filename].hash) {
-      throw new Error(`Invalid Build File Hash(${hash}) for chunk: ${filename}`)
+    if (hash !== this.renderOpts.buildId &&
+        hash !== (this.buildStats[filename] || {}).hash) {
+      return false
     }
 
     res.setHeader('Cache-Control', 'max-age=365000000, immutable')
+    return true
   }
 
   send404 (res) {

server/on-demand-entry-handler.js

@@ -129,7 +129,7 @@ export default function onDemandEntryHandler (devMiddleware, compiler, {
 
       const pagePath = join(dir, 'pages', page)
       const pathname = await resolvePath(pagePath)
-      const name = join('bundles', pathname.substring(dir.length))
+      const name = join('.', pathname.substring(dir.length))
 
       const entry = [`${pathname}?entry`]
 

server/utils.js

@@ -1,8 +1,8 @@
 import { join } from 'path'
 import { readdirSync, existsSync } from 'fs'
 
-export const IS_BUNDLED_PAGE = /^bundles[/\\]pages.*\.js$/
-export const MATCH_ROUTE_NAME = /^bundles[/\\]pages[/\\](.*)\.js$/
+export const IS_BUNDLED_PAGE = /^pages.*\.js$/
+export const MATCH_ROUTE_NAME = /^pages[/\\](.*)\.js$/
 
 export function getAvailableChunks (dir, dist) {
   const chunksDir = join(dir, dist, 'chunks')