rsnext/packages/next/server/dev/hot-reloader.ts
Shu Ding 99d4d6c5a4
Implement web server as the request handler for edge SSR (#33635)
(#31506 for context)

This PR implements the minimum viable web server on top of the Next.js base server, and integrates it into our middleware (edge) SSR runtime to handle all the requests.

This also addresses problems like missing dynamic routes support in our current handler.

Note that this is the initial implementation with the assumption that the web server is running under minimal mode. Also later we can refactor the `__server_context` environment to properly passing the context via the constructor or methods.
2022-01-26 06:22:11 +00:00

835 lines
26 KiB
TypeScript

import { getOverlayMiddleware } from 'next/dist/compiled/@next/react-dev-overlay/middleware'
import { IncomingMessage, ServerResponse } from 'http'
import { WebpackHotMiddleware } from './hot-middleware'
import { join, relative, isAbsolute } from 'path'
import { UrlObject } from 'url'
import { webpack } from 'next/dist/compiled/webpack/webpack'
import type { webpack5 } from 'next/dist/compiled/webpack/webpack'
import {
createEntrypoints,
createPagesMapping,
finalizeEntrypoint,
PagesMapping,
} from '../../build/entries'
import { watchCompilers } from '../../build/output'
import getBaseWebpackConfig from '../../build/webpack-config'
import { API_ROUTE, MIDDLEWARE_ROUTE } from '../../lib/constants'
import { recursiveDelete } from '../../lib/recursive-delete'
import { BLOCKED_PAGES } from '../../shared/lib/constants'
import { __ApiPreviewProps } from '../api-utils'
import { route } from '../router'
import { findPageFile } from '../lib/find-page-file'
import onDemandEntryHandler, {
entries,
BUILDING,
} from './on-demand-entry-handler'
import { denormalizePagePath, normalizePathSep } from '../normalize-page-path'
import getRouteFromEntrypoint from '../get-route-from-entrypoint'
import { fileExists } from '../../lib/file-exists'
import { ClientPagesLoaderOptions } from '../../build/webpack/loaders/next-client-pages-loader'
import { ssrEntries } from '../../build/webpack/plugins/middleware-plugin'
import { stringify } from 'querystring'
import {
difference,
isCustomErrorPage,
isFlightPage,
isReservedPage,
} from '../../build/utils'
import { NextConfigComplete } from '../config-shared'
import { CustomRoutes } from '../../lib/load-custom-routes'
import { DecodeError } from '../../shared/lib/utils'
import { Span, trace } from '../../trace'
import { getProperError } from '../../lib/is-error'
import ws from 'next/dist/compiled/ws'
const wsServer = new ws.Server({ noServer: true })
export async function renderScriptError(
res: ServerResponse,
error: Error,
{ verbose = true } = {}
) {
// Asks CDNs and others to not to cache the errored page
res.setHeader(
'Cache-Control',
'no-cache, no-store, max-age=0, must-revalidate'
)
if ((error as any).code === 'ENOENT') {
res.statusCode = 404
res.end('404 - Not Found')
return
}
if (verbose) {
console.error(error.stack)
}
res.statusCode = 500
res.end('500 - Internal Error')
}
function addCorsSupport(req: IncomingMessage, res: ServerResponse) {
const isApiRoute = req.url!.match(API_ROUTE)
// API routes handle their own CORS headers
if (isApiRoute) {
return { preflight: false }
}
if (!req.headers.origin) {
return { preflight: false }
}
res.setHeader('Access-Control-Allow-Origin', req.headers.origin)
res.setHeader('Access-Control-Allow-Methods', 'OPTIONS, GET')
// Based on https://github.com/primus/access-control/blob/4cf1bc0e54b086c91e6aa44fb14966fa5ef7549c/index.js#L158
if (req.headers['access-control-request-headers']) {
res.setHeader(
'Access-Control-Allow-Headers',
req.headers['access-control-request-headers'] as string
)
}
if (req.method === 'OPTIONS') {
res.writeHead(200)
res.end()
return { preflight: true }
}
return { preflight: false }
}
const matchNextPageBundleRequest = route(
'/_next/static/chunks/pages/:path*.js(\\.map|)'
)
// Recursively look up the issuer till it ends up at the root
function findEntryModule(issuer: any): any {
if (issuer.issuer) {
return findEntryModule(issuer.issuer)
}
return issuer
}
function erroredPages(compilation: webpack5.Compilation) {
const failedPages: { [page: string]: any[] } = {}
for (const error of compilation.errors) {
if (!error.module) {
continue
}
const entryModule = findEntryModule(error.module)
const { name } = entryModule
if (!name) {
continue
}
// Only pages have to be reloaded
const enhancedName = getRouteFromEntrypoint(name)
if (!enhancedName) {
continue
}
if (!failedPages[enhancedName]) {
failedPages[enhancedName] = []
}
failedPages[enhancedName].push(error)
}
return failedPages
}
export default class HotReloader {
private dir: string
private buildId: string
private middlewares: any[]
private pagesDir: string
private webpackHotMiddleware?: WebpackHotMiddleware
private config: NextConfigComplete
private webServerRuntime: boolean
private hasServerComponents: boolean
public clientStats: webpack5.Stats | null
public serverStats: webpack5.Stats | null
private clientError: Error | null = null
private serverError: Error | null = null
private serverPrevDocumentHash: string | null
private prevChunkNames?: Set<any>
private onDemandEntries?: ReturnType<typeof onDemandEntryHandler>
private previewProps: __ApiPreviewProps
private watcher: any
private rewrites: CustomRoutes['rewrites']
private fallbackWatcher: any
private hotReloaderSpan: Span
private pagesMapping: PagesMapping = {}
constructor(
dir: string,
{
config,
pagesDir,
buildId,
previewProps,
rewrites,
}: {
config: NextConfigComplete
pagesDir: string
buildId: string
previewProps: __ApiPreviewProps
rewrites: CustomRoutes['rewrites']
}
) {
this.buildId = buildId
this.dir = dir
this.middlewares = []
this.pagesDir = pagesDir
this.clientStats = null
this.serverStats = null
this.serverPrevDocumentHash = null
this.config = config
this.webServerRuntime = !!config.experimental.concurrentFeatures
this.hasServerComponents = !!(
config.experimental.concurrentFeatures &&
config.experimental.serverComponents
)
this.previewProps = previewProps
this.rewrites = rewrites
this.hotReloaderSpan = trace('hot-reloader', undefined, {
version: process.env.__NEXT_VERSION as string,
})
// Ensure the hotReloaderSpan is flushed immediately as it's the parentSpan for all processing
// of the current `next dev` invocation.
this.hotReloaderSpan.stop()
}
public async run(
req: IncomingMessage,
res: ServerResponse,
parsedUrl: UrlObject
): Promise<{ finished?: true }> {
// Usually CORS support is not needed for the hot-reloader (this is dev only feature)
// With when the app runs for multi-zones support behind a proxy,
// the current page is trying to access this URL via assetPrefix.
// That's when the CORS support is needed.
const { preflight } = addCorsSupport(req, res)
if (preflight) {
return {}
}
// When a request comes in that is a page bundle, e.g. /_next/static/<buildid>/pages/index.js
// we have to compile the page using on-demand-entries, this middleware will handle doing that
// by adding the page to on-demand-entries, waiting till it's done
// and then the bundle will be served like usual by the actual route in server/index.js
const handlePageBundleRequest = async (
pageBundleRes: ServerResponse,
parsedPageBundleUrl: UrlObject
): Promise<{ finished?: true }> => {
const { pathname } = parsedPageBundleUrl
const params: { path: string[] } | null =
matchNextPageBundleRequest(pathname)
if (!params) {
return {}
}
let decodedPagePath: string
try {
decodedPagePath = `/${params.path
.map((param) => decodeURIComponent(param))
.join('/')}`
} catch (_) {
throw new DecodeError('failed to decode param')
}
const page = denormalizePagePath(decodedPagePath)
if (page === '/_error' || BLOCKED_PAGES.indexOf(page) === -1) {
try {
await this.ensurePage(page, true)
} catch (error) {
await renderScriptError(pageBundleRes, getProperError(error))
return { finished: true }
}
const errors = await this.getCompilationErrors(page)
if (errors.length > 0) {
await renderScriptError(pageBundleRes, errors[0], { verbose: false })
return { finished: true }
}
}
return {}
}
const { finished } = await handlePageBundleRequest(res, parsedUrl)
for (const fn of this.middlewares) {
await new Promise<void>((resolve, reject) => {
fn(req, res, (err: Error) => {
if (err) return reject(err)
resolve()
})
})
}
return { finished }
}
public onHMR(req: IncomingMessage, _res: ServerResponse, head: Buffer) {
wsServer.handleUpgrade(req, req.socket, head, (client) => {
this.webpackHotMiddleware?.onHMR(client)
this.onDemandEntries?.onHMR(client)
})
}
private async clean(span: Span): Promise<void> {
return span
.traceChild('clean')
.traceAsyncFn(() =>
recursiveDelete(join(this.dir, this.config.distDir), /^cache/)
)
}
private async getWebpackConfig(span: Span) {
const webpackConfigSpan = span.traceChild('get-webpack-config')
return webpackConfigSpan.traceAsyncFn(async () => {
const pagePaths = await webpackConfigSpan
.traceChild('get-page-paths')
.traceAsyncFn(() =>
Promise.all([
findPageFile(this.pagesDir, '/_app', this.config.pageExtensions),
findPageFile(
this.pagesDir,
'/_document',
this.config.pageExtensions
),
])
)
this.pagesMapping = webpackConfigSpan
.traceChild('create-pages-mapping')
.traceFn(() =>
createPagesMapping(
pagePaths.filter((i) => i !== null) as string[],
this.config.pageExtensions,
{
isDev: true,
hasConcurrentFeatures: this.webServerRuntime,
hasServerComponents: this.hasServerComponents,
}
)
)
const entrypoints = webpackConfigSpan
.traceChild('create-entrypoints')
.traceFn(() =>
createEntrypoints(
this.pagesMapping,
'server',
this.buildId,
this.previewProps,
this.config,
[]
)
)
return webpackConfigSpan
.traceChild('generate-webpack-config')
.traceAsyncFn(() =>
Promise.all(
[
getBaseWebpackConfig(this.dir, {
dev: true,
isServer: false,
config: this.config,
buildId: this.buildId,
pagesDir: this.pagesDir,
rewrites: this.rewrites,
entrypoints: entrypoints.client,
runWebpackSpan: this.hotReloaderSpan,
}),
getBaseWebpackConfig(this.dir, {
dev: true,
isServer: true,
config: this.config,
buildId: this.buildId,
pagesDir: this.pagesDir,
rewrites: this.rewrites,
entrypoints: entrypoints.server,
runWebpackSpan: this.hotReloaderSpan,
}),
this.webServerRuntime
? getBaseWebpackConfig(this.dir, {
dev: true,
isServer: true,
webServerRuntime: true,
config: this.config,
buildId: this.buildId,
pagesDir: this.pagesDir,
rewrites: this.rewrites,
entrypoints: entrypoints.serverWeb,
runWebpackSpan: this.hotReloaderSpan,
})
: null,
].filter(Boolean) as webpack.Configuration[]
)
)
})
}
public async buildFallbackError(): Promise<void> {
if (this.fallbackWatcher) return
const fallbackConfig = await getBaseWebpackConfig(this.dir, {
runWebpackSpan: this.hotReloaderSpan,
dev: true,
isServer: false,
config: this.config,
buildId: this.buildId,
pagesDir: this.pagesDir,
rewrites: {
beforeFiles: [],
afterFiles: [],
fallback: [],
},
isDevFallback: true,
entrypoints: createEntrypoints(
{
'/_app': 'next/dist/pages/_app',
'/_error': 'next/dist/pages/_error',
},
'server',
this.buildId,
this.previewProps,
this.config,
[]
).client,
})
const fallbackCompiler = webpack(fallbackConfig)
this.fallbackWatcher = await new Promise((resolve) => {
let bootedFallbackCompiler = false
fallbackCompiler.watch(
// @ts-ignore webpack supports an array of watchOptions when using a multiCompiler
fallbackConfig.watchOptions,
// Errors are handled separately
(_err: any) => {
if (!bootedFallbackCompiler) {
bootedFallbackCompiler = true
resolve(true)
}
}
)
})
}
public async start(): Promise<void> {
const startSpan = this.hotReloaderSpan.traceChild('start')
startSpan.stop() // Stop immediately to create an artificial parent span
await this.clean(startSpan)
const configs = await this.getWebpackConfig(startSpan)
for (const config of configs) {
const defaultEntry = config.entry
config.entry = async (...args) => {
// @ts-ignore entry is always a function
const entrypoints = await defaultEntry(...args)
const isClientCompilation = config.name === 'client'
const isServerCompilation = config.name === 'server'
const isServerWebCompilation = config.name === 'server-web'
await Promise.all(
Object.keys(entries).map(async (pageKey) => {
const isClientKey = pageKey.startsWith('client')
const isServerWebKey = pageKey.startsWith('server-web')
if (isClientKey !== isClientCompilation) return
if (isServerWebKey !== isServerWebCompilation) return
const page = pageKey.slice(
isClientKey
? 'client'.length
: isServerWebKey
? 'server-web'.length
: 'server'.length
)
const isMiddleware = !!page.match(MIDDLEWARE_ROUTE)
if (isClientCompilation && page.match(API_ROUTE) && !isMiddleware) {
return
}
const isApiRoute = page.match(API_ROUTE)
if (!isClientCompilation && isMiddleware) {
return
}
const { bundlePath, absolutePagePath, dispose } = entries[pageKey]
const pageExists = !dispose && (await fileExists(absolutePagePath))
if (!pageExists) {
// page was removed or disposed
delete entries[pageKey]
return
}
const isCustomError = isCustomErrorPage(page)
const isReserved = isReservedPage(page)
const isServerComponent =
this.hasServerComponents &&
isFlightPage(this.config, absolutePagePath)
if (
isServerCompilation &&
this.webServerRuntime &&
!isApiRoute &&
!isCustomError
) {
return
}
entries[pageKey].status = BUILDING
const pageLoaderOpts: ClientPagesLoaderOptions = {
page,
absolutePagePath,
}
if (isClientCompilation && isMiddleware) {
entrypoints[bundlePath] = finalizeEntrypoint({
name: bundlePath,
value: `next-middleware-loader?${stringify(pageLoaderOpts)}!`,
isServer: false,
isMiddleware: true,
})
} else if (isClientCompilation) {
entrypoints[bundlePath] = finalizeEntrypoint({
name: bundlePath,
value: `next-client-pages-loader?${stringify(pageLoaderOpts)}!`,
isServer: false,
})
if (isServerComponent) {
ssrEntries.set(bundlePath, { requireFlightManifest: true })
} else if (
this.webServerRuntime &&
!isReserved &&
!isCustomError
) {
ssrEntries.set(bundlePath, { requireFlightManifest: false })
}
} else if (isServerWebCompilation) {
if (!isReserved) {
entrypoints[bundlePath] = finalizeEntrypoint({
name: '[name].js',
value: `next-middleware-ssr-loader?${stringify({
dev: true,
page,
stringifiedConfig: JSON.stringify(this.config),
absoluteAppPath: this.pagesMapping['/_app'],
absoluteDocumentPath: this.pagesMapping['/_document'],
absoluteErrorPath: this.pagesMapping['/_error'],
absolute404Path: this.pagesMapping['/404'] || '',
absolutePagePath,
isServerComponent,
buildId: this.buildId,
} as any)}!`,
isServer: false,
isServerWeb: true,
})
}
} else {
let request = relative(config.context!, absolutePagePath)
if (!isAbsolute(request) && !request.startsWith('../')) {
request = `./${request}`
}
entrypoints[bundlePath] = finalizeEntrypoint({
name: bundlePath,
value: request,
isServer: true,
})
}
})
)
return entrypoints
}
}
// Enable building of client compilation before server compilation in development
// @ts-ignore webpack 5
configs.parallelism = 1
const multiCompiler = webpack(configs) as unknown as webpack5.MultiCompiler
watchCompilers(
multiCompiler.compilers[0],
multiCompiler.compilers[1],
multiCompiler.compilers[2] || null
)
// Watch for changes to client/server page files so we can tell when just
// the server file changes and trigger a reload for GS(S)P pages
const changedClientPages = new Set<string>()
const changedServerPages = new Set<string>()
const prevClientPageHashes = new Map<string, string>()
const prevServerPageHashes = new Map<string, string>()
const trackPageChanges =
(pageHashMap: Map<string, string>, changedItems: Set<string>) =>
(stats: webpack5.Compilation) => {
stats.entrypoints.forEach((entry, key) => {
if (key.startsWith('pages/')) {
// TODO this doesn't handle on demand loaded chunks
entry.chunks.forEach((chunk: any) => {
if (chunk.id === key) {
const prevHash = pageHashMap.get(key)
if (prevHash && prevHash !== chunk.hash) {
changedItems.add(key)
}
pageHashMap.set(key, chunk.hash)
}
})
}
})
}
multiCompiler.compilers[0].hooks.emit.tap(
'NextjsHotReloaderForClient',
trackPageChanges(prevClientPageHashes, changedClientPages)
)
multiCompiler.compilers[1].hooks.emit.tap(
'NextjsHotReloaderForServer',
trackPageChanges(prevServerPageHashes, changedServerPages)
)
// This plugin watches for changes to _document.js and notifies the client side that it should reload the page
multiCompiler.compilers[1].hooks.failed.tap(
'NextjsHotReloaderForServer',
(err: Error) => {
this.serverError = err
this.serverStats = null
}
)
multiCompiler.compilers[1].hooks.done.tap(
'NextjsHotReloaderForServer',
(stats) => {
this.serverError = null
this.serverStats = stats
const { compilation } = stats
// We only watch `_document` for changes on the server compilation
// the rest of the files will be triggered by the client compilation
const documentChunk = compilation.namedChunks.get('pages/_document')
// If the document chunk can't be found we do nothing
if (!documentChunk) {
console.warn('_document.js chunk not found')
return
}
// Initial value
if (this.serverPrevDocumentHash === null) {
this.serverPrevDocumentHash = documentChunk.hash || null
return
}
// If _document.js didn't change we don't trigger a reload
if (documentChunk.hash === this.serverPrevDocumentHash) {
return
}
// Notify reload to reload the page, as _document.js was changed (different hash)
this.send('reloadPage')
this.serverPrevDocumentHash = documentChunk.hash || null
}
)
multiCompiler.hooks.done.tap('NextjsHotReloaderForServer', () => {
const serverOnlyChanges = difference<string>(
changedServerPages,
changedClientPages
)
const middlewareChanges = Array.from(changedClientPages).filter((name) =>
name.match(MIDDLEWARE_ROUTE)
)
changedClientPages.clear()
changedServerPages.clear()
if (middlewareChanges.length > 0) {
this.send({
event: 'middlewareChanges',
})
}
if (serverOnlyChanges.length > 0) {
this.send({
event: 'serverOnlyChanges',
pages: serverOnlyChanges.map((pg) =>
denormalizePagePath(pg.substr('pages'.length))
),
})
}
})
multiCompiler.compilers[0].hooks.failed.tap(
'NextjsHotReloaderForClient',
(err: Error) => {
this.clientError = err
this.clientStats = null
}
)
multiCompiler.compilers[0].hooks.done.tap(
'NextjsHotReloaderForClient',
(stats) => {
this.clientError = null
this.clientStats = stats
const { compilation } = stats
const chunkNames = new Set(
[...compilation.namedChunks.keys()].filter(
(name) => !!getRouteFromEntrypoint(name)
)
)
if (this.prevChunkNames) {
// detect chunks which have to be replaced with a new template
// e.g, pages/index.js <-> pages/_error.js
const addedPages = diff(chunkNames, this.prevChunkNames!)
const removedPages = diff(this.prevChunkNames!, chunkNames)
if (addedPages.size > 0) {
for (const addedPage of addedPages) {
const page = getRouteFromEntrypoint(addedPage)
this.send('addedPage', page)
}
}
if (removedPages.size > 0) {
for (const removedPage of removedPages) {
const page = getRouteFromEntrypoint(removedPage)
this.send('removedPage', page)
}
}
}
this.prevChunkNames = chunkNames
}
)
this.webpackHotMiddleware = new WebpackHotMiddleware(
multiCompiler.compilers
)
let booted = false
this.watcher = await new Promise((resolve) => {
const watcher = multiCompiler.watch(
// @ts-ignore webpack supports an array of watchOptions when using a multiCompiler
configs.map((config) => config.watchOptions!),
// Errors are handled separately
(_err: any) => {
if (!booted) {
booted = true
resolve(watcher)
}
}
)
})
this.onDemandEntries = onDemandEntryHandler(this.watcher, multiCompiler, {
pagesDir: this.pagesDir,
nextConfig: this.config,
...(this.config.onDemandEntries as {
maxInactiveAge: number
pagesBufferLength: number
}),
})
this.middlewares = [
getOverlayMiddleware({
rootDirectory: this.dir,
stats: () => this.clientStats,
serverStats: () => this.serverStats,
}),
]
}
public async stop(): Promise<void> {
await new Promise((resolve, reject) => {
this.watcher.close((err: any) => (err ? reject(err) : resolve(true)))
})
if (this.fallbackWatcher) {
await new Promise((resolve, reject) => {
this.fallbackWatcher.close((err: any) =>
err ? reject(err) : resolve(true)
)
})
}
}
public async getCompilationErrors(page: string) {
const normalizedPage = normalizePathSep(page)
if (this.clientError || this.serverError) {
return [this.clientError || this.serverError]
} else if (this.clientStats?.hasErrors()) {
const { compilation } = this.clientStats
const failedPages = erroredPages(compilation)
// If there is an error related to the requesting page we display it instead of the first error
if (
failedPages[normalizedPage] &&
failedPages[normalizedPage].length > 0
) {
return failedPages[normalizedPage]
}
// If none were found we still have to show the other errors
return this.clientStats.compilation.errors
} else if (this.serverStats?.hasErrors()) {
const { compilation } = this.serverStats
const failedPages = erroredPages(compilation)
// If there is an error related to the requesting page we display it instead of the first error
if (
failedPages[normalizedPage] &&
failedPages[normalizedPage].length > 0
) {
return failedPages[normalizedPage]
}
// If none were found we still have to show the other errors
return this.serverStats.compilation.errors
}
return []
}
public send(action?: string | any, ...args: any[]): void {
this.webpackHotMiddleware!.publish(
action && typeof action === 'object' ? action : { action, data: args }
)
}
public async ensurePage(page: string, clientOnly: boolean = false) {
// Make sure we don't re-build or dispose prebuilt pages
if (page !== '/_error' && BLOCKED_PAGES.indexOf(page) !== -1) {
return
}
const error = clientOnly
? this.clientError
: this.serverError || this.clientError
if (error) {
return Promise.reject(error)
}
return this.onDemandEntries?.ensurePage(page, clientOnly) as any
}
}
function diff(a: Set<any>, b: Set<any>) {
return new Set([...a].filter((v) => !b.has(v)))
}