Add experimental options for more parallelization in webpack builds (#60177)
This PR introduces 2 experimental options for doing more work in the webpack build in parallel instead of in serial. These options may improve the performance of builds at the cost of more memory. `parallelServerAndEdgeCompiles`: This option kicks off the builds for both `server` and `edge-server` at the same time instead of waiting for each to complete before the next one. In applications that have many server and edge functions, this can increase performance by doing that work in parallel. This can be used with `next build` or `next experimental-compile`. `parallelServerBuildTraces`: This option starts the server build traces as soon as the server compile completes and runs it in the background while the other compilations are happening. With this option enabled, some unnecessary work may be done since ordinarily the client compilation provides information that can reduce the amount of tracing necessary. However, since it is in parallel with the other work, it may still result in a faster build in total at the cost of more memory. This option is already the default when using `next experimental-compile` but can now be used when `next build` is used also. --------- Co-authored-by: Delba de Oliveira <32464864+delbaoliveira@users.noreply.github.com> Co-authored-by: JJ Kasper <jj@jjsweb.site>
This commit is contained in:
parent
a29bf3373f
commit
ca5bc989d1
7 changed files with 110 additions and 26 deletions
18
errors/parallel-build-without-worker.mdx
Normal file
18
errors/parallel-build-without-worker.mdx
Normal file
|
@ -0,0 +1,18 @@
|
||||||
|
---
|
||||||
|
title: Parallel Build Without Build Worker
|
||||||
|
---
|
||||||
|
|
||||||
|
## Why This Error Occurred
|
||||||
|
|
||||||
|
The `experimental.parallelServerCompiles` and `experimental.parallelServerBuildTraces`
|
||||||
|
options require that the `experimental.webpackBuildWorker` option is set to `true`. These
|
||||||
|
options use workers to improve the parallelization of the build which may improve performance,
|
||||||
|
but the build may use more memory at the same time.
|
||||||
|
|
||||||
|
## Possible Ways to Fix It
|
||||||
|
|
||||||
|
Build workers are enabled by default unless you have a custom webpack config. You can force enable the option by setting `experimental.webpackBuildWorker: true` in your `next.config.js` file, but some webpack configuration options may not be compatible.
|
||||||
|
|
||||||
|
## Useful Links
|
||||||
|
|
||||||
|
Also see https://nextjs.org/docs/messages/webpack-build-worker-opt-out
|
|
@ -1395,6 +1395,12 @@ export default async function build(
|
||||||
config.experimental.webpackBuildWorker ||
|
config.experimental.webpackBuildWorker ||
|
||||||
(config.experimental.webpackBuildWorker === undefined &&
|
(config.experimental.webpackBuildWorker === undefined &&
|
||||||
!config.webpack)
|
!config.webpack)
|
||||||
|
const runServerAndEdgeInParallel =
|
||||||
|
config.experimental.parallelServerCompiles
|
||||||
|
const collectServerBuildTracesInParallel =
|
||||||
|
config.experimental.parallelServerBuildTraces ||
|
||||||
|
(config.experimental.parallelServerBuildTraces === undefined &&
|
||||||
|
isCompileMode)
|
||||||
|
|
||||||
nextBuildSpan.setAttribute(
|
nextBuildSpan.setAttribute(
|
||||||
'has-custom-webpack-config',
|
'has-custom-webpack-config',
|
||||||
|
@ -1410,45 +1416,67 @@ export default async function build(
|
||||||
'Custom webpack configuration is detected. When using a custom webpack configuration, the Webpack build worker is disabled by default. To force enable it, set the "experimental.webpackBuildWorker" option to "true". Read more: https://nextjs.org/docs/messages/webpack-build-worker-opt-out'
|
'Custom webpack configuration is detected. When using a custom webpack configuration, the Webpack build worker is disabled by default. To force enable it, set the "experimental.webpackBuildWorker" option to "true". Read more: https://nextjs.org/docs/messages/webpack-build-worker-opt-out'
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
if (
|
||||||
|
!useBuildWorker &&
|
||||||
|
(runServerAndEdgeInParallel || collectServerBuildTracesInParallel)
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
'The "parallelServerBuildTraces" and "parallelServerCompiles" options may only be used when build workers can be used. Read more: https://nextjs.org/docs/messages/parallel-build-without-worker'
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
Log.info('Creating an optimized production build ...')
|
Log.info('Creating an optimized production build ...')
|
||||||
|
|
||||||
if (!isGenerateMode) {
|
if (!isGenerateMode) {
|
||||||
if (isCompileMode && useBuildWorker) {
|
if (runServerAndEdgeInParallel || collectServerBuildTracesInParallel) {
|
||||||
let durationInSeconds = 0
|
let durationInSeconds = 0
|
||||||
|
|
||||||
await webpackBuild(useBuildWorker, ['server']).then((res) => {
|
const serverBuildPromise = webpackBuild(useBuildWorker, [
|
||||||
|
'server',
|
||||||
|
]).then((res) => {
|
||||||
buildTraceContext = res.buildTraceContext
|
buildTraceContext = res.buildTraceContext
|
||||||
durationInSeconds += res.duration
|
durationInSeconds += res.duration
|
||||||
const buildTraceWorker = new Worker(
|
|
||||||
require.resolve('./collect-build-traces'),
|
|
||||||
{
|
|
||||||
numWorkers: 1,
|
|
||||||
exposedMethods: ['collectBuildTraces'],
|
|
||||||
}
|
|
||||||
) as Worker & typeof import('./collect-build-traces')
|
|
||||||
|
|
||||||
buildTracesPromise = buildTraceWorker
|
if (collectServerBuildTracesInParallel) {
|
||||||
.collectBuildTraces({
|
const buildTraceWorker = new Worker(
|
||||||
dir,
|
require.resolve('./collect-build-traces'),
|
||||||
config,
|
{
|
||||||
distDir,
|
numWorkers: 1,
|
||||||
// Serialize Map as this is sent to the worker.
|
exposedMethods: ['collectBuildTraces'],
|
||||||
pageInfos: serializePageInfos(new Map()),
|
}
|
||||||
staticPages: [],
|
) as Worker & typeof import('./collect-build-traces')
|
||||||
hasSsrAmpPages: false,
|
|
||||||
buildTraceContext,
|
buildTracesPromise = buildTraceWorker
|
||||||
outputFileTracingRoot,
|
.collectBuildTraces({
|
||||||
})
|
dir,
|
||||||
.catch((err) => {
|
config,
|
||||||
console.error(err)
|
distDir,
|
||||||
process.exit(1)
|
// Serialize Map as this is sent to the worker.
|
||||||
})
|
pageInfos: serializePageInfos(new Map()),
|
||||||
|
staticPages: [],
|
||||||
|
hasSsrAmpPages: false,
|
||||||
|
buildTraceContext,
|
||||||
|
outputFileTracingRoot,
|
||||||
|
})
|
||||||
|
.catch((err) => {
|
||||||
|
console.error(err)
|
||||||
|
process.exit(1)
|
||||||
|
})
|
||||||
|
}
|
||||||
})
|
})
|
||||||
|
if (!runServerAndEdgeInParallel) {
|
||||||
|
await serverBuildPromise
|
||||||
|
}
|
||||||
|
|
||||||
await webpackBuild(useBuildWorker, ['edge-server']).then((res) => {
|
const edgeBuildPromise = webpackBuild(useBuildWorker, [
|
||||||
|
'edge-server',
|
||||||
|
]).then((res) => {
|
||||||
durationInSeconds += res.duration
|
durationInSeconds += res.duration
|
||||||
})
|
})
|
||||||
|
if (runServerAndEdgeInParallel) {
|
||||||
|
await serverBuildPromise
|
||||||
|
}
|
||||||
|
await edgeBuildPromise
|
||||||
|
|
||||||
await webpackBuild(useBuildWorker, ['client']).then((res) => {
|
await webpackBuild(useBuildWorker, ['client']).then((res) => {
|
||||||
durationInSeconds += res.duration
|
durationInSeconds += res.duration
|
||||||
|
|
|
@ -88,6 +88,8 @@ const supportedTurbopackNextConfigOptions = [
|
||||||
'experimental.memoryBasedWorkersCount',
|
'experimental.memoryBasedWorkersCount',
|
||||||
'experimental.clientRouterFilterRedirects',
|
'experimental.clientRouterFilterRedirects',
|
||||||
'experimental.webpackBuildWorker',
|
'experimental.webpackBuildWorker',
|
||||||
|
'experimental.parallelServerCompiles',
|
||||||
|
'experimental.parallelServerBuildTraces',
|
||||||
'experimental.appDocumentPreloading',
|
'experimental.appDocumentPreloading',
|
||||||
'experimental.incrementalCacheHandlerPath',
|
'experimental.incrementalCacheHandlerPath',
|
||||||
'experimental.amp',
|
'experimental.amp',
|
||||||
|
|
|
@ -285,6 +285,8 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
|
||||||
outputFileTracingIncludes: z
|
outputFileTracingIncludes: z
|
||||||
.record(z.string(), z.array(z.string()))
|
.record(z.string(), z.array(z.string()))
|
||||||
.optional(),
|
.optional(),
|
||||||
|
parallelServerCompiles: z.boolean().optional(),
|
||||||
|
parallelServerBuildTraces: z.boolean().optional(),
|
||||||
ppr: z.boolean().optional(),
|
ppr: z.boolean().optional(),
|
||||||
taint: z.boolean().optional(),
|
taint: z.boolean().optional(),
|
||||||
proxyTimeout: z.number().gte(0).optional(),
|
proxyTimeout: z.number().gte(0).optional(),
|
||||||
|
|
|
@ -282,6 +282,35 @@ export interface ExperimentalConfig {
|
||||||
*/
|
*/
|
||||||
typedRoutes?: boolean
|
typedRoutes?: boolean
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Runs the compilations for server and edge in parallel instead of in serial.
|
||||||
|
* This will make builds faster if there is enough server and edge functions
|
||||||
|
* in the application at the cost of more memory.
|
||||||
|
*
|
||||||
|
* NOTE: This option is only valid when the build process can use workers. See
|
||||||
|
* the documentation for `webpackBuildWorker` for more details.
|
||||||
|
*/
|
||||||
|
parallelServerCompiles?: boolean
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Runs the logic to collect build traces for the server routes in parallel
|
||||||
|
* with other work during the compilation. This will increase the speed of
|
||||||
|
* the build at the cost of more memory. This option may incur some additional
|
||||||
|
* work compared to if the option was disabled since the work is started
|
||||||
|
* before data from the client compilation is available to potentially reduce
|
||||||
|
* the amount of code that needs to be traced. Despite that, this may still
|
||||||
|
* result in faster builds for some applications.
|
||||||
|
*
|
||||||
|
* Valid values are:
|
||||||
|
* - `true`: Collect the server build traces in parallel.
|
||||||
|
* - `false`: Do not collect the server build traces in parallel.
|
||||||
|
* - `undefined`: Collect server build traces in parallel only in the `experimental-compile` mode.
|
||||||
|
*
|
||||||
|
* NOTE: This option is only valid when the build process can use workers. See
|
||||||
|
* the documentation for `webpackBuildWorker` for more details.
|
||||||
|
*/
|
||||||
|
parallelServerBuildTraces?: boolean
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Run the Webpack build in a separate process to optimize memory usage during build.
|
* Run the Webpack build in a separate process to optimize memory usage during build.
|
||||||
* Valid values are:
|
* Valid values are:
|
||||||
|
@ -811,6 +840,8 @@ export const defaultConfig: NextConfig = {
|
||||||
typedRoutes: false,
|
typedRoutes: false,
|
||||||
instrumentationHook: false,
|
instrumentationHook: false,
|
||||||
bundlePagesExternals: false,
|
bundlePagesExternals: false,
|
||||||
|
parallelServerCompiles: false,
|
||||||
|
parallelServerBuildTraces: false,
|
||||||
ppr:
|
ppr:
|
||||||
// TODO: remove once we've made PPR default
|
// TODO: remove once we've made PPR default
|
||||||
// If we're testing, and the `__NEXT_EXPERIMENTAL_PPR` environment variable
|
// If we're testing, and the `__NEXT_EXPERIMENTAL_PPR` environment variable
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
module.exports = {
|
module.exports = {
|
||||||
experimental: {
|
experimental: {
|
||||||
clientRouterFilterRedirects: true,
|
clientRouterFilterRedirects: true,
|
||||||
|
parallelServerCompiles: true,
|
||||||
|
parallelServerBuildTraces: true,
|
||||||
webpackBuildWorker: true,
|
webpackBuildWorker: true,
|
||||||
},
|
},
|
||||||
// output: 'standalone',
|
// output: 'standalone',
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
module.exports = {
|
module.exports = {
|
||||||
experimental: {
|
experimental: {
|
||||||
typedRoutes: true,
|
typedRoutes: true,
|
||||||
|
parallelServerBuildTraces: true,
|
||||||
webpackBuildWorker: true,
|
webpackBuildWorker: true,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue