diff --git a/errors/webpack-build-worker-opt-out.mdx b/errors/webpack-build-worker-opt-out.mdx new file mode 100644 index 0000000000..66c628e82a --- /dev/null +++ b/errors/webpack-build-worker-opt-out.mdx @@ -0,0 +1,15 @@ +--- +title: Webpack Build Worker automatic opt-out +--- + +## Webpack Build Worker Opt-out + +#### Why This Error Occurred + +The Next.js config contains custom webpack configuration, as a result, the webpack build worker optimization was disabled. + +The webpack build worker optimization helps alleviate memory stress during builds and reduce out-of-memory errors although some custom configurations may not be compatible. + +#### Possible Ways to Fix It + +You can force enable the option by setting `config.experimental.webpackBuildWorker: true` in the config. diff --git a/packages/next/src/build/build-context.ts b/packages/next/src/build/build-context.ts index 393f0ed17e..e7cf889c29 100644 --- a/packages/next/src/build/build-context.ts +++ b/packages/next/src/build/build-context.ts @@ -5,7 +5,8 @@ import type { __ApiPreviewProps } from '../server/api-utils' import type { NextConfigComplete } from '../server/config-shared' import type { Span } from '../trace' import type getBaseWebpackConfig from './webpack-config' -import type { TelemetryPlugin } from './webpack/plugins/telemetry-plugin' +import type { TelemetryPluginState } from './webpack/plugins/telemetry-plugin' +import type { Telemetry } from '../telemetry/storage' // A layer for storing data that is used by plugins to communicate with each // other between different steps of the build process. This is only internal @@ -82,7 +83,8 @@ export const NextBuildContext: Partial<{ hasInstrumentationHook: boolean // misc fields - telemetryPlugin: TelemetryPlugin + telemetry: Telemetry + telemetryState: TelemetryPluginState buildSpinner: Ora nextBuildSpan: Span diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 90d279cc32..cc60df026f 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -161,6 +161,7 @@ import type { BuildTraceContext } from './webpack/plugins/next-trace-entrypoints import { formatManifest } from './manifests/formatter/format-manifest' import { getStartServerInfo, logStartInfo } from '../server/lib/app-info-log' import type { NextEnabledDirectories } from '../server/base-server' +import { hasCustomExportOutput } from '../export/utils' import { RouteKind } from '../server/future/route-kind' interface ExperimentalBypassForInfo { @@ -398,12 +399,7 @@ export default async function build( NextBuildContext.config = config let configOutDir = 'out' - if (config.output === 'export' && config.distDir !== '.next') { - // In the past, a user had to run "next build" to generate - // ".next" (or whatever the distDir) followed by "next export" - // to generate "out" (or whatever the outDir). However, when - // "output: export" is configured, "next build" does both steps. - // So the user-configured distDir is actually the outDir. + if (hasCustomExportOutput(config)) { configOutDir = config.distDir config.distDir = '.next' } @@ -1086,16 +1082,38 @@ export default async function build( }) const [duration] = process.hrtime(turboNextBuildStart) - return { duration, buildTraceContext: null } + return { duration, buildTraceContext: undefined } } let buildTraceContext: undefined | BuildTraceContext let buildTracesPromise: Promise | undefined = undefined + // If there's has a custom webpack config and disable the build worker. + // Otherwise respect the option if it's set. + const useBuildWorker = + config.experimental.webpackBuildWorker || + (config.experimental.webpackBuildWorker === undefined && + !config.webpack) + + nextBuildSpan.setAttribute( + 'has-custom-webpack-config', + String(!!config.webpack) + ) + nextBuildSpan.setAttribute('use-build-worker', String(useBuildWorker)) + + if ( + config.webpack && + config.experimental.webpackBuildWorker === undefined + ) { + Log.warn( + 'Custom webpack configuration is detected. When using a custom webpack configuration, the Webpack build worker is disabled by default. To force enable it, set the "experimental.webpackBuildWorker" option to "true". Read more: https://nextjs.org/docs/messages/webpack-build-worker-opt-out' + ) + } + if (!isGenerate) { - if (isCompile && config.experimental.webpackBuildWorker) { + if (isCompile && useBuildWorker) { let durationInSeconds = 0 - await webpackBuild(['server']).then((res) => { + await webpackBuild(useBuildWorker, ['server']).then((res) => { buildTraceContext = res.buildTraceContext durationInSeconds += res.duration const buildTraceWorker = new Worker( @@ -1123,11 +1141,11 @@ export default async function build( }) }) - await webpackBuild(['edge-server']).then((res) => { + await webpackBuild(useBuildWorker, ['edge-server']).then((res) => { durationInSeconds += res.duration }) - await webpackBuild(['client']).then((res) => { + await webpackBuild(useBuildWorker, ['client']).then((res) => { durationInSeconds += res.duration }) @@ -1143,7 +1161,7 @@ export default async function build( } else { const { duration: webpackBuildDuration, ...rest } = turboNextBuild ? await turbopackBuild() - : await webpackBuild() + : await webpackBuild(useBuildWorker, null) buildTraceContext = rest.buildTraceContext @@ -2801,11 +2819,15 @@ export default async function build( }) ) - if (NextBuildContext.telemetryPlugin) { - const events = eventBuildFeatureUsage(NextBuildContext.telemetryPlugin) + if (NextBuildContext.telemetryState) { + const events = eventBuildFeatureUsage( + NextBuildContext.telemetryState.usages + ) telemetry.record(events) telemetry.record( - eventPackageUsedInGetServerSideProps(NextBuildContext.telemetryPlugin) + eventPackageUsedInGetServerSideProps( + NextBuildContext.telemetryState.packagesUsedInServerSideProps + ) ) } diff --git a/packages/next/src/build/webpack-build/impl.ts b/packages/next/src/build/webpack-build/impl.ts index f081388c74..e89132f7d5 100644 --- a/packages/next/src/build/webpack-build/impl.ts +++ b/packages/next/src/build/webpack-build/impl.ts @@ -13,7 +13,10 @@ import { runCompiler } from '../compiler' import * as Log from '../output/log' import getBaseWebpackConfig, { loadProjectInfo } from '../webpack-config' import type { NextError } from '../../lib/is-error' -import { TelemetryPlugin } from '../webpack/plugins/telemetry-plugin' +import { + TelemetryPlugin, + type TelemetryPluginState, +} from '../webpack/plugins/telemetry-plugin' import { NextBuildContext, resumePluginState, @@ -24,6 +27,7 @@ import loadConfig from '../../server/config' import { getTraceEvents, initializeTraceState, + setGlobal, trace, type TraceEvent, type TraceState, @@ -34,6 +38,7 @@ import type { BuildTraceContext } from '../webpack/plugins/next-trace-entrypoint import type { UnwrapPromise } from '../../lib/coalesced-function' import origDebug from 'next/dist/compiled/debug' +import { Telemetry } from '../../telemetry/storage' const debug = origDebug('next:build:webpack-build') @@ -60,11 +65,12 @@ function isTraceEntryPointsPlugin( } export async function webpackBuildImpl( - compilerName?: keyof typeof COMPILER_INDEXES + compilerName: keyof typeof COMPILER_INDEXES | null ): Promise<{ duration: number pluginState: any buildTraceContext?: BuildTraceContext + telemetryState?: TelemetryPluginState }> { let result: CompilerResult | null = { warnings: [], @@ -267,9 +273,9 @@ export async function webpackBuildImpl( .traceChild('format-webpack-messages') .traceFn(() => formatWebpackMessages(result, true)) as CompilerResult - NextBuildContext.telemetryPlugin = ( - clientConfig as webpack.Configuration - ).plugins?.find(isTelemetryPlugin) + const telemetryPlugin = (clientConfig as webpack.Configuration).plugins?.find( + isTelemetryPlugin + ) const traceEntryPointsPlugin = ( serverConfig as webpack.Configuration @@ -329,6 +335,11 @@ export async function webpackBuildImpl( duration: webpackBuildEnd[0], buildTraceContext: traceEntryPointsPlugin?.buildTraceContext, pluginState: getPluginState(), + telemetryState: { + usages: telemetryPlugin?.usages() || [], + packagesUsedInServerSideProps: + telemetryPlugin?.packagesUsedInServerSideProps() || [], + }, } } } @@ -343,6 +354,11 @@ export async function workerMain(workerData: { debugTraceEvents: TraceEvent[] } > { + // Clone the telemetry for worker + const telemetry = new Telemetry({ + distDir: workerData.buildContext.config!.distDir, + }) + setGlobal('telemetry', telemetry) // setup new build context from the serialized data passed from the parent Object.assign(NextBuildContext, workerData.buildContext) diff --git a/packages/next/src/build/webpack-build/index.ts b/packages/next/src/build/webpack-build/index.ts index 8e9675d1a2..93884ffaf6 100644 --- a/packages/next/src/build/webpack-build/index.ts +++ b/packages/next/src/build/webpack-build/index.ts @@ -31,15 +31,11 @@ function deepMerge(target: any, source: any) { } async function webpackBuildWithWorker( - compilerNames: typeof ORDERED_COMPILER_NAMES = ORDERED_COMPILER_NAMES + compilerNamesArg: typeof ORDERED_COMPILER_NAMES | null ) { - const { - config, - telemetryPlugin, - buildSpinner, - nextBuildSpan, - ...prunedBuildContext - } = NextBuildContext + const compilerNames = compilerNamesArg || ORDERED_COMPILER_NAMES + const { buildSpinner, nextBuildSpan, ...prunedBuildContext } = + NextBuildContext prunedBuildContext.pluginState = pluginState @@ -100,6 +96,10 @@ async function webpackBuildWithWorker( pluginState = deepMerge(pluginState, curResult.pluginState) prunedBuildContext.pluginState = pluginState + if (curResult.telemetryState) { + NextBuildContext.telemetryState = curResult.telemetryState + } + combinedResult.duration += curResult.duration if (curResult.buildTraceContext?.entriesTrace) { @@ -134,17 +134,16 @@ async function webpackBuildWithWorker( return combinedResult } -export async function webpackBuild( - compilerNames?: typeof ORDERED_COMPILER_NAMES -) { - const config = NextBuildContext.config! - - if (config.experimental.webpackBuildWorker) { +export function webpackBuild( + withWorker: boolean, + compilerNames: typeof ORDERED_COMPILER_NAMES | null +): ReturnType { + if (withWorker) { debug('using separate compiler workers') - return await webpackBuildWithWorker(compilerNames) + return webpackBuildWithWorker(compilerNames) } else { debug('building all compilers in same process') const webpackBuildImpl = require('./impl').webpackBuildImpl - return await webpackBuildImpl() + return webpackBuildImpl(null, null) } } diff --git a/packages/next/src/build/webpack-config.ts b/packages/next/src/build/webpack-config.ts index 75b0ebdd46..e3030b9ea1 100644 --- a/packages/next/src/build/webpack-config.ts +++ b/packages/next/src/build/webpack-config.ts @@ -75,6 +75,7 @@ import { createWebpackAliases } from './create-compiler-aliases' import { createServerOnlyClientOnlyAliases } from './create-compiler-aliases' import { createRSCAliases } from './create-compiler-aliases' import { createServerComponentsNoopAliases } from './create-compiler-aliases' +import { hasCustomExportOutput } from '../export/utils' type ExcludesFalse = (x: T | false) => x is T type ClientEntries = { @@ -352,6 +353,10 @@ export default async function getBaseWebpackConfig( : '' const babelConfigFile = getBabelConfigFile(dir) + + if (hasCustomExportOutput(config)) { + config.distDir = '.next' + } const distDir = path.join(dir, config.distDir) let useSWCLoader = !babelConfigFile || config.experimental.forceSwcTransforms diff --git a/packages/next/src/build/webpack/plugins/pages-manifest-plugin.ts b/packages/next/src/build/webpack/plugins/pages-manifest-plugin.ts index 1645cc17a4..09be591ac4 100644 --- a/packages/next/src/build/webpack/plugins/pages-manifest-plugin.ts +++ b/packages/next/src/build/webpack/plugins/pages-manifest-plugin.ts @@ -132,17 +132,18 @@ export default class PagesManifestPlugin ...nodeServerPages, }) } else { - assets[(!this.dev && !this.isEdgeRuntime ? '../' : '') + PAGES_MANIFEST] = - new sources.RawSource( - JSON.stringify( - { - ...edgeServerPages, - ...nodeServerPages, - }, - null, - 2 - ) + const pagesManifestPath = + (!this.dev && !this.isEdgeRuntime ? '../' : '') + PAGES_MANIFEST + assets[pagesManifestPath] = new sources.RawSource( + JSON.stringify( + { + ...edgeServerPages, + ...nodeServerPages, + }, + null, + 2 ) + ) } if (this.appDirEnabled) { diff --git a/packages/next/src/build/webpack/plugins/telemetry-plugin.ts b/packages/next/src/build/webpack/plugins/telemetry-plugin.ts index 9299dd7820..701cfa0c9b 100644 --- a/packages/next/src/build/webpack/plugins/telemetry-plugin.ts +++ b/packages/next/src/build/webpack/plugins/telemetry-plugin.ts @@ -111,7 +111,7 @@ const BUILD_FEATURES: Array = [ 'modularizeImports', ] -const ELIMINATED_PACKAGES = new Set() +const eliminatedPackages = new Set() /** * Determine if there is a feature of interest in the specified 'module'. @@ -160,7 +160,10 @@ function findUniqueOriginModulesInConnections( * they are imported. */ export class TelemetryPlugin implements webpack.WebpackPluginInstance { - private usageTracker = new Map() + private usageTracker: Map = new Map< + Feature, + FeatureUsage + >() // Build feature usage is on/off and is known before the build starts constructor(buildFeaturesMap: Map) { @@ -218,7 +221,7 @@ export class TelemetryPlugin implements webpack.WebpackPluginInstance { compiler.hooks.compilation.tap(TelemetryPlugin.name, (compilation) => { const moduleHooks = NormalModule.getCompilationHooks(compilation) moduleHooks.loader.tap(TelemetryPlugin.name, (loaderContext: any) => { - loaderContext.eliminatedPackages = ELIMINATED_PACKAGES + loaderContext.eliminatedPackages = eliminatedPackages }) }) } @@ -229,6 +232,13 @@ export class TelemetryPlugin implements webpack.WebpackPluginInstance { } packagesUsedInServerSideProps(): string[] { - return Array.from(ELIMINATED_PACKAGES) + return Array.from(eliminatedPackages) } } + +export type TelemetryPluginState = { + usages: ReturnType + packagesUsedInServerSideProps: ReturnType< + TelemetryPlugin['packagesUsedInServerSideProps'] + > +} diff --git a/packages/next/src/export/utils.ts b/packages/next/src/export/utils.ts new file mode 100644 index 0000000000..20ade4b43a --- /dev/null +++ b/packages/next/src/export/utils.ts @@ -0,0 +1,14 @@ +import type { NextConfigComplete } from '../server/config-shared' + +export function hasCustomExportOutput(config: NextConfigComplete) { + // In the past, a user had to run "next build" to generate + // ".next" (or whatever the distDir) followed by "next export" + // to generate "out" (or whatever the outDir). However, when + // "output: export" is configured, "next build" does both steps. + // So the user-configured distDir is actually the outDir. + // We'll do some custom logic when meeting this condition. + // e.g. + // Will set config.distDir to .next to make sure the manifests + // are still reading from temporary .next directory. + return config.output === 'export' && config.distDir !== '.next' +} diff --git a/packages/next/src/server/config-shared.ts b/packages/next/src/server/config-shared.ts index 748311ac03..2ab2c13167 100644 --- a/packages/next/src/server/config-shared.ts +++ b/packages/next/src/server/config-shared.ts @@ -283,7 +283,11 @@ export interface ExperimentalConfig { typedRoutes?: boolean /** - * This option is to enable running the Webpack build in a worker thread. + * Run the Webpack build in a separate process to optimize memory usage during build. + * Valid values are: + * - `false`: Disable the Webpack build worker + * - `true`: Enable the Webpack build worker + * - `undefined`: Enable the Webpack build worker only if the webpack config is not customized */ webpackBuildWorker?: boolean @@ -806,7 +810,7 @@ export const defaultConfig: NextConfig = { process.env.__NEXT_EXPERIMENTAL_PPR === 'true' ? true : false, - webpackBuildWorker: false, + webpackBuildWorker: undefined, }, } diff --git a/packages/next/src/telemetry/events/build.ts b/packages/next/src/telemetry/events/build.ts index 9019fdb0c2..9276e07af8 100644 --- a/packages/next/src/telemetry/events/build.ts +++ b/packages/next/src/telemetry/events/build.ts @@ -175,9 +175,9 @@ export type EventBuildFeatureUsage = { invocationCount: number } export function eventBuildFeatureUsage( - telemetryPlugin: TelemetryPlugin + usages: ReturnType ): Array<{ eventName: string; payload: EventBuildFeatureUsage }> { - return telemetryPlugin.usages().map(({ featureName, invocationCount }) => ({ + return usages.map(({ featureName, invocationCount }) => ({ eventName: EVENT_BUILD_FEATURE_USAGE, payload: { featureName, @@ -194,9 +194,11 @@ export type EventPackageUsedInGetServerSideProps = { } export function eventPackageUsedInGetServerSideProps( - telemetryPlugin: TelemetryPlugin + packagesUsedInServerSideProps: ReturnType< + TelemetryPlugin['packagesUsedInServerSideProps'] + > ): Array<{ eventName: string; payload: EventPackageUsedInGetServerSideProps }> { - return telemetryPlugin.packagesUsedInServerSideProps().map((packageName) => ({ + return packagesUsedInServerSideProps.map((packageName) => ({ eventName: EVENT_NAME_PACKAGE_USED_IN_GET_SERVER_SIDE_PROPS, payload: { package: packageName, diff --git a/test/integration/next-image-legacy/svgo-webpack/test/index.test.ts b/test/integration/next-image-legacy/svgo-webpack/test/index.test.ts index ef8b01c012..6b8bbd01f0 100644 --- a/test/integration/next-image-legacy/svgo-webpack/test/index.test.ts +++ b/test/integration/next-image-legacy/svgo-webpack/test/index.test.ts @@ -23,7 +23,10 @@ let devOutput () => { it('should not fail to build invalid usage of the Image component', async () => { const { stderr, code } = await nextBuild(appDir, [], { stderr: true }) - expect(stderr).toBeFalsy() + const errors = stderr + .split('\n') + .filter((line: string) => line && !line.trim().startsWith('⚠')) + expect(errors).toEqual([]) expect(code).toBe(0) }) } diff --git a/test/integration/next-image-new/svgo-webpack/test/index.test.ts b/test/integration/next-image-new/svgo-webpack/test/index.test.ts index 6962b82650..35c209c8ce 100644 --- a/test/integration/next-image-new/svgo-webpack/test/index.test.ts +++ b/test/integration/next-image-new/svgo-webpack/test/index.test.ts @@ -25,7 +25,7 @@ let devOutput const { stderr, code } = await nextBuild(appDir, [], { stderr: true }) const errors = stderr .split('\n') - .filter((line) => line && !line.trim().startsWith('⚠️')) + .filter((line: string) => line && !line.trim().startsWith('⚠')) expect(errors).toEqual([]) expect(code).toBe(0) }) diff --git a/test/integration/webpack-require-hook/test/index.test.js b/test/integration/webpack-require-hook/test/index.test.js index 0353483575..959c11160a 100644 --- a/test/integration/webpack-require-hook/test/index.test.js +++ b/test/integration/webpack-require-hook/test/index.test.js @@ -22,8 +22,10 @@ const appDir = path.join(__dirname, '..') stdout: true, stderr: true, }) - console.log(stderr) - expect(stderr.length).toStrictEqual(0) + const errors = stderr + .split('\n') + .filter((line) => line && !line.trim().startsWith('⚠')) + expect(errors).toEqual([]) expect(stdout).toMatch(/Initialized config/) }) })