Add build manifest (#4119)

* Add build manifest

* Split out css since they don’t have exact name

* Remove pages map

* Fix locations test

* Re-run tests

* Get consistent open ports

* Fix static tests

* Add comment about Cache-Control header

bin/next-export

@@ -57,8 +57,6 @@ const options = {
   outdir: argv.outdir ? resolve(argv.outdir) : resolve(dir, 'out')
 }
 
-exportApp(dir, options)
+exportApp(dir, options).catch((err) => {
-  .catch((err) => {
+  printAndExit(err)
-    console.error(err)
+})
-    process.exit(1)
-  })

lib/constants.js

@@ -3,3 +3,4 @@ export const PHASE_PRODUCTION_BUILD = 'phase-production-build'
 export const PHASE_PRODUCTION_SERVER = 'phase-production-server'
 export const PHASE_DEVELOPMENT_SERVER = 'phase-development-server'
 export const PAGES_MANIFEST = 'pages-manifest.json'
+export const BUILD_MANIFEST = 'build-manifest.json'

package.json

@@ -134,7 +134,6 @@
     "node-fetch": "1.7.3",
     "node-notifier": "5.1.2",
     "nyc": "11.2.1",
-    "portfinder": "1.0.13",
     "react": "16.2.0",
     "react-dom": "16.2.0",
     "rimraf": "2.6.2",

readme.md

@@ -943,9 +943,9 @@ import flush from 'styled-jsx/server'
 
 export default class MyDocument extends Document {
   static getInitialProps({ renderPage }) {
-    const { html, head, errorHtml, chunks } = renderPage()
+    const { html, head, errorHtml, chunks, buildManifest } = renderPage()
     const styles = flush()
-    return { html, head, errorHtml, chunks, styles }
+    return { html, head, errorHtml, chunks, styles, buildManifest }
   }
 
   render() {

server/build/plugins/build-manifest-plugin.js

@@ -0,0 +1,46 @@
+// @flow
+import { RawSource } from 'webpack-sources'
+import {BUILD_MANIFEST} from '../../../lib/constants'
+
+// This plugin creates a build-manifest.json for all assets that are being output
+// It has a mapping of "entry" filename to real filename. Because the real filename can be hashed in production
+export default class BuildManifestPlugin {
+  apply (compiler: any) {
+    compiler.plugin('emit', (compilation, callback) => {
+      const {chunks} = compilation
+      const assetMap = {pages: {}, css: []}
+
+      for (const chunk of chunks) {
+        if (!chunk.name || !chunk.files) {
+          continue
+        }
+
+        const files = []
+
+        for (const file of chunk.files) {
+          if (/\.map$/.test(file)) {
+            continue
+          }
+
+          if (/\.hot-update\.js$/.test(file)) {
+            continue
+          }
+
+          if (/\.css$/.exec(file)) {
+            assetMap.css.push(file)
+            continue
+          }
+
+          files.push(file)
+        }
+
+        if (files.length > 0) {
+          assetMap[chunk.name] = files
+        }
+      }
+
+      compilation.assets[BUILD_MANIFEST] = new RawSource(JSON.stringify(assetMap))
+      callback()
+    })
+  }
+}

server/build/webpack.js

@@ -12,6 +12,7 @@ import NextJsSsrImportPlugin from './plugins/nextjs-ssr-import'
 import DynamicChunksPlugin from './plugins/dynamic-chunks-plugin'
 import UnlinkFilePlugin from './plugins/unlink-file-plugin'
 import PagesManifestPlugin from './plugins/pages-manifest-plugin'
+import BuildManifestPlugin from './plugins/build-manifest-plugin'
 
 const presetItem = createConfigItem(require('./babel/preset'), {type: 'preset'})
 const hotLoaderItem = createConfigItem(require('react-hot-loader/babel'), {type: 'plugin'})
@@ -259,6 +260,7 @@ export default async function getBaseWebpackConfig (dir, {dev = false, isServer
       }),
       !dev && new webpack.optimize.ModuleConcatenationPlugin(),
       isServer && new PagesManifestPlugin(),
+      !isServer && new BuildManifestPlugin(),
       !isServer && new PagesPlugin(),
       !isServer && new DynamicChunksPlugin(),
       isServer && new NextJsSsrImportPlugin(),
@@ -266,7 +268,7 @@ export default async function getBaseWebpackConfig (dir, {dev = false, isServer
       // In production we move common modules into the existing main.js bundle
       !isServer && new webpack.optimize.CommonsChunkPlugin({
         name: 'main.js',
-        filename: 'main.js',
+        filename: dev ? 'static/commons/main.js' : 'static/commons/main-[chunkhash].js',
         minChunks (module, count) {
           // React and React DOM are used everywhere in Next.js. So they should always be common. Even in development mode, to speed up compilation.
           if (module.resource && module.resource.includes(`${sep}react-dom${sep}`) && count >= 0) {
@@ -297,8 +299,8 @@ export default async function getBaseWebpackConfig (dir, {dev = false, isServer
       }),
       // We use a manifest file in development to speed up HMR
       dev && !isServer && new webpack.optimize.CommonsChunkPlugin({
-        name: 'manifest',
+        name: 'manifest.js',
-        filename: 'manifest.js'
+        filename: dev ? 'static/commons/manifest.js' : 'static/commons/manifest-[chunkhash].js'
       })
     ].filter(Boolean)
   }

server/document.js

@@ -10,9 +10,9 @@ const Fragment = React.Fragment || function Fragment ({ children }) {
 
 export default class Document extends Component {
   static getInitialProps ({ renderPage }) {
-    const { html, head, errorHtml, chunks } = renderPage()
+    const { html, head, errorHtml, chunks, buildManifest } = renderPage()
     const styles = flush()
-    return { html, head, errorHtml, chunks, styles }
+    return { html, head, errorHtml, chunks, styles, buildManifest }
   }
 
   static childContextTypes = {
@@ -40,32 +40,33 @@ export class Head extends Component {
   }
 
   getChunkPreloadLink (filename) {
-    const { __NEXT_DATA__ } = this.context._documentProps
+    const { __NEXT_DATA__, buildManifest } = this.context._documentProps
     let { assetPrefix, buildId } = __NEXT_DATA__
-    const hash = buildId
 
-    return (
+    const files = buildManifest[filename]
-      <link
+
+    return files.map(file => {
+      return <link
         key={filename}
         rel='preload'
-        href={`${assetPrefix}/_next/${hash}/${filename}`}
+        href={`${assetPrefix}/_next/${file}`}
         as='script'
       />
-    )
+    })
   }
 
   getPreloadMainLinks () {
     const { dev } = this.context._documentProps
     if (dev) {
       return [
-        this.getChunkPreloadLink('manifest.js'),
+        ...this.getChunkPreloadLink('manifest.js'),
-        this.getChunkPreloadLink('main.js')
+        ...this.getChunkPreloadLink('main.js')
       ]
     }
 
     // In the production mode, we have a single asset with all the JS content.
     return [
-      this.getChunkPreloadLink('main.js')
+      ...this.getChunkPreloadLink('main.js')
     ]
   }
 
@@ -125,31 +126,32 @@ export class NextScript extends Component {
   }
 
   getChunkScript (filename, additionalProps = {}) {
-    const { __NEXT_DATA__ } = this.context._documentProps
+    const { __NEXT_DATA__, buildManifest } = this.context._documentProps
     let { assetPrefix, buildId } = __NEXT_DATA__
-    const hash = buildId
 
-    return (
+    const files = buildManifest[filename]
+
+    return files.map((file) => (
       <script
         key={filename}
-        src={`${assetPrefix}/_next/${hash}/${filename}`}
+        src={`${assetPrefix}/_next/${file}`}
         {...additionalProps}
       />
-    )
+    ))
   }
 
   getScripts () {
     const { dev } = this.context._documentProps
     if (dev) {
       return [
-        this.getChunkScript('manifest.js'),
+        ...this.getChunkScript('manifest.js'),
-        this.getChunkScript('main.js')
+        ...this.getChunkScript('main.js')
       ]
     }
 
     // In the production mode, we have a single asset with all the JS content.
     // So, we can load the script with async
-    return [this.getChunkScript('main.js', { async: true })]
+    return [...this.getChunkScript('main.js', { async: true })]
   }
 
   getDynamicChunks () {

server/export.js

@@ -19,10 +19,7 @@ export default async function (dir, options, configuration) {
   log(`> using build directory: ${nextDir}`)
 
   if (!existsSync(nextDir)) {
-    console.error(
+    throw new Error(`Build directory ${nextDir} does not exist. Make sure you run "next build" before running "next start" or "next export".`)
-      `Build directory ${nextDir} does not exist. Make sure you run "next build" before running "next start" or "next export".`
-    )
-    process.exit(1)
   }
 
   const buildId = readFileSync(join(nextDir, 'BUILD_ID'), 'utf8')
@@ -53,12 +50,6 @@ export default async function (dir, options, configuration) {
     )
   }
 
-  // Copy main.js
-  await cp(
-    join(nextDir, 'main.js'),
-    join(outDir, '_next', buildId, 'main.js')
-  )
-
   // Copy .next/static directory
   if (existsSync(join(nextDir, 'static'))) {
     log('  copying "static build" directory')

server/index.js

@@ -162,57 +162,6 @@ export default class Server {
         await this.serveStatic(req, res, p)
       },
 
-      '/_next/:buildId/manifest.js': async (req, res, params) => {
-        if (!this.dev) return this.send404(res)
-
-        this.handleBuildId(params.buildId, res)
-        const p = join(this.dir, this.dist, 'manifest.js')
-        await this.serveStatic(req, res, p)
-      },
-
-      '/_next/:buildId/manifest.js.map': async (req, res, params) => {
-        if (!this.dev) return this.send404(res)
-
-        this.handleBuildId(params.buildId, res)
-        const p = join(this.dir, this.dist, 'manifest.js.map')
-        await this.serveStatic(req, res, p)
-      },
-
-      '/_next/:buildId/main.js': async (req, res, params) => {
-        if (this.dev) {
-          this.handleBuildId(params.buildId, res)
-          const p = join(this.dir, this.dist, 'main.js')
-          await this.serveStatic(req, res, p)
-        } else {
-          const buildId = params.buildId
-          if (!this.handleBuildId(buildId, res)) {
-            const error = new Error('INVALID_BUILD_ID')
-            const customFields = { buildIdMismatched: true }
-
-            return await renderScriptError(req, res, '/_error', error, customFields, this.renderOpts)
-          }
-
-          const p = join(this.dir, this.dist, 'main.js')
-          await this.serveStatic(req, res, p)
-        }
-      },
-
-      '/_next/:buildId/main.js.map': async (req, res, params) => {
-        if (this.dev) {
-          this.handleBuildId(params.buildId, res)
-          const p = join(this.dir, this.dist, 'main.js.map')
-          await this.serveStatic(req, res, p)
-        } else {
-          const buildId = params.buildId
-          if (!this.handleBuildId(buildId, res)) {
-            return await this.render404(req, res)
-          }
-
-          const p = join(this.dir, this.dist, 'main.js.map')
-          await this.serveStatic(req, res, p)
-        }
-      },
-
       '/_next/:buildId/page/:path*.js.map': async (req, res, params) => {
         const paths = params.path || ['']
         const page = `/${paths.join('/')}`
@@ -279,6 +228,11 @@ export default class Server {
       },
 
       '/_next/static/:path*': async (req, res, params) => {
+        // The commons folder holds commonschunk files
+        // In development they don't have a hash, and shouldn't be cached by the browser.
+        if (this.dev && params.path[0] === 'commons') {
+          res.setHeader('Cache-Control', 'no-store, must-revalidate')
+        }
         const p = join(this.dir, this.dist, 'static', ...(params.path || []))
         await this.serveStatic(req, res, p)
       },

server/render.js

@@ -12,6 +12,7 @@ import Head, { defaultHead } from '../lib/head'
 import App from '../lib/app'
 import ErrorDebug from '../lib/error-debug'
 import { flushChunks } from '../lib/dynamic'
+import { BUILD_MANIFEST } from '../lib/constants'
 
 const logger = console
 
@@ -54,6 +55,7 @@ async function doRender (req, res, pathname, query, {
   }
 
   const documentPath = join(dir, dist, 'dist', 'bundles', 'pages', '_document')
+  const buildManifest = require(join(dir, dist, BUILD_MANIFEST))
 
   let [Component, Document] = await Promise.all([
     requirePage(page, {dir, dist}),
@@ -94,7 +96,7 @@ async function doRender (req, res, pathname, query, {
     }
     const chunks = loadChunks({ dev, dir, dist, availableChunks })
 
-    return { html, head, errorHtml, chunks }
+    return { html, head, errorHtml, chunks, buildManifest }
   }
 
   const docProps = await loadGetInitialProps(Document, { ...ctx, renderPage })
@@ -117,6 +119,7 @@ async function doRender (req, res, pathname, query, {
     dev,
     dir,
     staticMarkup,
+    buildManifest,
     ...docProps
   })
 

test/integration/dist-dir/test/index.test.js

@@ -39,11 +39,10 @@ describe('Production Usage', () => {
 
   describe('File locations', () => {
     it('should build the app within the given `dist` directory', () => {
-      expect(existsSync(join(__dirname, '/../dist/main.js'))).toBeTruthy()
+      expect(existsSync(join(__dirname, '/../dist/BUILD_ID'))).toBeTruthy()
     })
-
     it('should not build the app within the default `.next` directory', () => {
-      expect(existsSync(join(__dirname, '/../.next/main.js'))).toBeFalsy()
+      expect(existsSync(join(__dirname, '/../.next/BUILD_ID'))).toBeFalsy()
     })
   })
 })

test/integration/static/.gitignore

@@ -0,0 +1 @@
+.next-dev

test/integration/static/next.config.js

@@ -1,17 +1,22 @@
-module.exports = {
+const {PHASE_DEVELOPMENT_SERVER} = require('next/constants')
-  exportPathMap: function () {
+
-    return {
+module.exports = (phase) => {
-      '/': { page: '/' },
+  return {
-      '/about': { page: '/about' },
+    distDir: phase === PHASE_DEVELOPMENT_SERVER ? '.next-dev' : '.next',
-      '/asset': { page: '/asset' },
+    exportPathMap: function () {
-      '/button-link': { page: '/button-link' },
+      return {
-      '/get-initial-props-with-no-query': { page: '/get-initial-props-with-no-query' },
+        '/': { page: '/' },
-      '/counter': { page: '/counter' },
+        '/about': { page: '/about' },
-      '/dynamic-imports': { page: '/dynamic-imports' },
+        '/asset': { page: '/asset' },
-      '/dynamic': { page: '/dynamic', query: { text: 'cool dynamic text' } },
+        '/button-link': { page: '/button-link' },
-      '/dynamic/one': { page: '/dynamic', query: { text: 'next export is nice' } },
+        '/get-initial-props-with-no-query': { page: '/get-initial-props-with-no-query' },
-      '/dynamic/two': { page: '/dynamic', query: { text: 'zeit is awesome' } },
+        '/counter': { page: '/counter' },
-      '/file-name.md': { page: '/dynamic', query: { text: 'this file has an extension' } }
+        '/dynamic-imports': { page: '/dynamic-imports' },
+        '/dynamic': { page: '/dynamic', query: { text: 'cool dynamic text' } },
+        '/dynamic/one': { page: '/dynamic', query: { text: 'next export is nice' } },
+        '/dynamic/two': { page: '/dynamic', query: { text: 'zeit is awesome' } },
+        '/file-name.md': { page: '/dynamic', query: { text: 'this file has an extension' } }
+      }
     }
   }
 }

test/integration/static/test/index.test.js

@@ -31,13 +31,13 @@ describe('Static Export', () => {
     context.server = await startStaticServer(join(appDir, 'out'))
     context.port = context.server.address().port
 
-    devContext.appPort = await findPort()
+    devContext.port = await findPort()
-    devContext.server = await launchApp(join(__dirname, '../'), devContext.appPort, true)
+    devContext.server = await launchApp(join(__dirname, '../'), devContext.port, true)
 
     // pre-build all pages at the start
     await Promise.all([
-      renderViaHTTP(devContext.appPort, '/'),
+      renderViaHTTP(devContext.port, '/'),
-      renderViaHTTP(devContext.appPort, '/dynamic/one')
+      renderViaHTTP(devContext.port, '/dynamic/one')
     ])
   })
   afterAll(() => {
@@ -47,5 +47,5 @@ describe('Static Export', () => {
 
   ssr(context)
   browser(context)
-  dev(context)
+  dev(devContext)
 })

test/lib/next-test-utils.js

@@ -3,7 +3,7 @@ import qs from 'querystring'
 import http from 'http'
 import express from 'express'
 import path from 'path'
-import portfinder from 'portfinder'
+import getPort from 'get-port'
 import { spawn } from 'child_process'
 import { readFileSync, writeFileSync, existsSync, unlinkSync } from 'fs'
 import fkill from 'fkill'
@@ -63,8 +63,7 @@ export function fetchViaHTTP (appPort, pathname, query) {
 }
 
 export function findPort () {
-  portfinder.basePort = 20000 + Math.ceil(Math.random() * 10000)
+  return getPort()
-  return portfinder.getPortPromise()
 }
 
 // Launch the app in dev mode.

test/lib/next-webdriver.js

@@ -1,6 +1,10 @@
 import wd from 'wd'
 
 export default async function (appPort, pathname) {
+  if (typeof appPort === 'undefined') {
+    throw new Error('appPort is undefined')
+  }
+
   const url = `http://localhost:${appPort}${pathname}`
   console.log(`> Start loading browser with url: ${url}`)
 

yarn.lock

@@ -902,7 +902,7 @@ ansi-styles@^3.2.0, ansi-styles@^3.2.1:
   dependencies:
     color-convert "^1.9.0"
 
-any-promise@^1.0.0, any-promise@^1.1.0:
+any-promise@^1.1.0:
   version "1.3.0"
   resolved "https://registry.yarnpkg.com/any-promise/-/any-promise-1.3.0.tgz#abc6afeedcea52e809cdc0376aed3ce39635d17f"
 
@@ -1083,7 +1083,7 @@ async@2.0.1:
   dependencies:
     lodash "^4.8.0"
 
-async@^1.4.0, async@^1.5.2:
+async@^1.4.0:
   version "1.5.2"
   resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a"
 
@@ -3440,10 +3440,6 @@ glob-parent@^3.1.0:
     is-glob "^3.1.0"
     path-dirname "^1.0.0"
 
-glob-promise@3.3.0:
-  version "3.3.0"
-  resolved "https://registry.yarnpkg.com/glob-promise/-/glob-promise-3.3.0.tgz#d1eb3625c4e6dcbb9b96eeae4425d5a3b135fed2"
-
 glob@7.1.2, glob@^7.0.0, glob@^7.0.3, glob@^7.0.5, glob@^7.0.6, glob@^7.1.1, glob@^7.1.2:
   version "7.1.2"
   resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15"
@@ -3675,11 +3671,7 @@ hoek@4.x.x:
   version "4.2.1"
   resolved "https://registry.yarnpkg.com/hoek/-/hoek-4.2.1.tgz#9634502aa12c445dd5a7c5734b572bb8738aacbb"
 
-hoist-non-react-statics@2.3.1:
+hoist-non-react-statics@2.5.0, hoist-non-react-statics@^2.5.0:
-  version "2.3.1"
-  resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-2.3.1.tgz#343db84c6018c650778898240135a1420ee22ce0"
-
-hoist-non-react-statics@^2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/hoist-non-react-statics/-/hoist-non-react-statics-2.5.0.tgz#d2ca2dfc19c5a91c5a6615ce8e564ef0347e2a40"
 
@@ -5163,7 +5155,7 @@ mkdirp@0.5.0:
   dependencies:
     minimist "0.0.8"
 
-mkdirp@0.5.1, mkdirp@0.5.x, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1:
+mkdirp@0.5.1, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1:
   version "0.5.1"
   resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903"
   dependencies:
@@ -5200,14 +5192,6 @@ mute-stream@0.0.7:
   version "0.0.7"
   resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.7.tgz#3075ce93bc21b8fab43e1bc4da7e8115ed1e7bab"
 
-mz@2.7.0:
-  version "2.7.0"
-  resolved "https://registry.yarnpkg.com/mz/-/mz-2.7.0.tgz#95008057a56cafadc2bc63dde7f9ff6955948e32"
-  dependencies:
-    any-promise "^1.0.0"
-    object-assign "^4.0.1"
-    thenify-all "^1.0.0"
-
 nan@^2.3.0:
   version "2.10.0"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.10.0.tgz#96d0cd610ebd58d4b4de9cc0c6828cda99c7548f"
@@ -5790,14 +5774,6 @@ pluralize@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-7.0.0.tgz#298b89df8b93b0221dbf421ad2b1b1ea23fc6777"
 
-portfinder@1.0.13:
-  version "1.0.13"
-  resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.13.tgz#bb32ecd87c27104ae6ee44b5a3ccbf0ebb1aede9"
-  dependencies:
-    async "^1.5.2"
-    debug "^2.2.0"
-    mkdirp "0.5.x"
-
 posix-character-classes@^0.1.0:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab"
@@ -7358,18 +7334,6 @@ text-table@~0.2.0:
   version "0.2.0"
   resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4"
 
-thenify-all@^1.0.0:
-  version "1.6.0"
-  resolved "https://registry.yarnpkg.com/thenify-all/-/thenify-all-1.6.0.tgz#1a1918d402d8fc3f98fbf234db0bcc8cc10e9726"
-  dependencies:
-    thenify ">= 3.1.0 < 4"
-
-"thenify@>= 3.1.0 < 4":
-  version "3.3.0"
-  resolved "https://registry.yarnpkg.com/thenify/-/thenify-3.3.0.tgz#e69e38a1babe969b0108207978b9f62b88604839"
-  dependencies:
-    any-promise "^1.0.0"
-
 throat@^4.0.0:
   version "4.1.0"
   resolved "https://registry.yarnpkg.com/throat/-/throat-4.1.0.tgz#89037cbc92c56ab18926e6ba4cbb200e15672a6a"