Enable build worker by default (#59405)

errors/webpack-build-worker-opt-out.mdx

@@ -0,0 +1,15 @@
+---
+title: Webpack Build Worker automatic opt-out
+---
+
+## Webpack Build Worker Opt-out
+
+#### Why This Error Occurred
+
+The Next.js config contains custom webpack configuration, as a result, the webpack build worker optimization was disabled.
+
+The webpack build worker optimization helps alleviate memory stress during builds and reduce out-of-memory errors although some custom configurations may not be compatible.
+
+#### Possible Ways to Fix It
+
+You can force enable the option by setting `config.experimental.webpackBuildWorker: true` in the config.

packages/next/src/build/build-context.ts

@@ -5,7 +5,8 @@ import type { __ApiPreviewProps } from '../server/api-utils'
 import type { NextConfigComplete } from '../server/config-shared'
 import type { Span } from '../trace'
 import type getBaseWebpackConfig from './webpack-config'
-import type { TelemetryPlugin } from './webpack/plugins/telemetry-plugin'
+import type { TelemetryPluginState } from './webpack/plugins/telemetry-plugin'
+import type { Telemetry } from '../telemetry/storage'
 
 // A layer for storing data that is used by plugins to communicate with each
 // other between different steps of the build process. This is only internal
@@ -82,7 +83,8 @@ export const NextBuildContext: Partial<{
   hasInstrumentationHook: boolean
 
   // misc fields
-  telemetryPlugin: TelemetryPlugin
+  telemetry: Telemetry
+  telemetryState: TelemetryPluginState
   buildSpinner: Ora
   nextBuildSpan: Span
 

packages/next/src/build/index.ts

@@ -161,6 +161,7 @@ import type { BuildTraceContext } from './webpack/plugins/next-trace-entrypoints
 import { formatManifest } from './manifests/formatter/format-manifest'
 import { getStartServerInfo, logStartInfo } from '../server/lib/app-info-log'
 import type { NextEnabledDirectories } from '../server/base-server'
+import { hasCustomExportOutput } from '../export/utils'
 import { RouteKind } from '../server/future/route-kind'
 
 interface ExperimentalBypassForInfo {
@@ -398,12 +399,7 @@ export default async function build(
       NextBuildContext.config = config
 
       let configOutDir = 'out'
-      if (config.output === 'export' && config.distDir !== '.next') {
-        // In the past, a user had to run "next build" to generate
-        // ".next" (or whatever the distDir) followed by "next export"
-        // to generate "out" (or whatever the outDir). However, when
-        // "output: export" is configured, "next build" does both steps.
-        // So the user-configured distDir is actually the outDir.
+      if (hasCustomExportOutput(config)) {
         configOutDir = config.distDir
         config.distDir = '.next'
       }
@@ -1086,16 +1082,38 @@ export default async function build(
         })
 
         const [duration] = process.hrtime(turboNextBuildStart)
-        return { duration, buildTraceContext: null }
+        return { duration, buildTraceContext: undefined }
       }
       let buildTraceContext: undefined | BuildTraceContext
       let buildTracesPromise: Promise<any> | undefined = undefined
 
+      // If there's has a custom webpack config and disable the build worker.
+      // Otherwise respect the option if it's set.
+      const useBuildWorker =
+        config.experimental.webpackBuildWorker ||
+        (config.experimental.webpackBuildWorker === undefined &&
+          !config.webpack)
+
+      nextBuildSpan.setAttribute(
+        'has-custom-webpack-config',
+        String(!!config.webpack)
+      )
+      nextBuildSpan.setAttribute('use-build-worker', String(useBuildWorker))
+
+      if (
+        config.webpack &&
+        config.experimental.webpackBuildWorker === undefined
+      ) {
+        Log.warn(
+          'Custom webpack configuration is detected. When using a custom webpack configuration, the Webpack build worker is disabled by default. To force enable it, set the "experimental.webpackBuildWorker" option to "true". Read more: https://nextjs.org/docs/messages/webpack-build-worker-opt-out'
+        )
+      }
+
       if (!isGenerate) {
-        if (isCompile && config.experimental.webpackBuildWorker) {
+        if (isCompile && useBuildWorker) {
           let durationInSeconds = 0
 
-          await webpackBuild(['server']).then((res) => {
+          await webpackBuild(useBuildWorker, ['server']).then((res) => {
             buildTraceContext = res.buildTraceContext
             durationInSeconds += res.duration
             const buildTraceWorker = new Worker(
@@ -1123,11 +1141,11 @@ export default async function build(
               })
           })
 
-          await webpackBuild(['edge-server']).then((res) => {
+          await webpackBuild(useBuildWorker, ['edge-server']).then((res) => {
             durationInSeconds += res.duration
           })
 
-          await webpackBuild(['client']).then((res) => {
+          await webpackBuild(useBuildWorker, ['client']).then((res) => {
             durationInSeconds += res.duration
           })
 
@@ -1143,7 +1161,7 @@ export default async function build(
         } else {
           const { duration: webpackBuildDuration, ...rest } = turboNextBuild
             ? await turbopackBuild()
-            : await webpackBuild()
+            : await webpackBuild(useBuildWorker, null)
 
           buildTraceContext = rest.buildTraceContext
 
@@ -2801,11 +2819,15 @@ export default async function build(
         })
       )
 
-      if (NextBuildContext.telemetryPlugin) {
-        const events = eventBuildFeatureUsage(NextBuildContext.telemetryPlugin)
+      if (NextBuildContext.telemetryState) {
+        const events = eventBuildFeatureUsage(
+          NextBuildContext.telemetryState.usages
+        )
         telemetry.record(events)
         telemetry.record(
-          eventPackageUsedInGetServerSideProps(NextBuildContext.telemetryPlugin)
+          eventPackageUsedInGetServerSideProps(
+            NextBuildContext.telemetryState.packagesUsedInServerSideProps
+          )
         )
       }
 

packages/next/src/build/webpack-build/impl.ts

@@ -13,7 +13,10 @@ import { runCompiler } from '../compiler'
 import * as Log from '../output/log'
 import getBaseWebpackConfig, { loadProjectInfo } from '../webpack-config'
 import type { NextError } from '../../lib/is-error'
-import { TelemetryPlugin } from '../webpack/plugins/telemetry-plugin'
+import {
+  TelemetryPlugin,
+  type TelemetryPluginState,
+} from '../webpack/plugins/telemetry-plugin'
 import {
   NextBuildContext,
   resumePluginState,
@@ -24,6 +27,7 @@ import loadConfig from '../../server/config'
 import {
   getTraceEvents,
   initializeTraceState,
+  setGlobal,
   trace,
   type TraceEvent,
   type TraceState,
@@ -34,6 +38,7 @@ import type { BuildTraceContext } from '../webpack/plugins/next-trace-entrypoint
 import type { UnwrapPromise } from '../../lib/coalesced-function'
 
 import origDebug from 'next/dist/compiled/debug'
+import { Telemetry } from '../../telemetry/storage'
 
 const debug = origDebug('next:build:webpack-build')
 
@@ -60,11 +65,12 @@ function isTraceEntryPointsPlugin(
 }
 
 export async function webpackBuildImpl(
-  compilerName?: keyof typeof COMPILER_INDEXES
+  compilerName: keyof typeof COMPILER_INDEXES | null
 ): Promise<{
   duration: number
   pluginState: any
   buildTraceContext?: BuildTraceContext
+  telemetryState?: TelemetryPluginState
 }> {
   let result: CompilerResult | null = {
     warnings: [],
@@ -267,9 +273,9 @@ export async function webpackBuildImpl(
     .traceChild('format-webpack-messages')
     .traceFn(() => formatWebpackMessages(result, true)) as CompilerResult
 
-  NextBuildContext.telemetryPlugin = (
-    clientConfig as webpack.Configuration
-  ).plugins?.find(isTelemetryPlugin)
+  const telemetryPlugin = (clientConfig as webpack.Configuration).plugins?.find(
+    isTelemetryPlugin
+  )
 
   const traceEntryPointsPlugin = (
     serverConfig as webpack.Configuration
@@ -329,6 +335,11 @@ export async function webpackBuildImpl(
       duration: webpackBuildEnd[0],
       buildTraceContext: traceEntryPointsPlugin?.buildTraceContext,
       pluginState: getPluginState(),
+      telemetryState: {
+        usages: telemetryPlugin?.usages() || [],
+        packagesUsedInServerSideProps:
+          telemetryPlugin?.packagesUsedInServerSideProps() || [],
+      },
     }
   }
 }
@@ -343,6 +354,11 @@ export async function workerMain(workerData: {
     debugTraceEvents: TraceEvent[]
   }
 > {
+  // Clone the telemetry for worker
+  const telemetry = new Telemetry({
+    distDir: workerData.buildContext.config!.distDir,
+  })
+  setGlobal('telemetry', telemetry)
   // setup new build context from the serialized data passed from the parent
   Object.assign(NextBuildContext, workerData.buildContext)
 

packages/next/src/build/webpack-build/index.ts

@@ -31,15 +31,11 @@ function deepMerge(target: any, source: any) {
 }
 
 async function webpackBuildWithWorker(
-  compilerNames: typeof ORDERED_COMPILER_NAMES = ORDERED_COMPILER_NAMES
+  compilerNamesArg: typeof ORDERED_COMPILER_NAMES | null
 ) {
-  const {
-    config,
-    telemetryPlugin,
-    buildSpinner,
-    nextBuildSpan,
-    ...prunedBuildContext
-  } = NextBuildContext
+  const compilerNames = compilerNamesArg || ORDERED_COMPILER_NAMES
+  const { buildSpinner, nextBuildSpan, ...prunedBuildContext } =
+    NextBuildContext
 
   prunedBuildContext.pluginState = pluginState
 
@@ -100,6 +96,10 @@ async function webpackBuildWithWorker(
     pluginState = deepMerge(pluginState, curResult.pluginState)
     prunedBuildContext.pluginState = pluginState
 
+    if (curResult.telemetryState) {
+      NextBuildContext.telemetryState = curResult.telemetryState
+    }
+
     combinedResult.duration += curResult.duration
 
     if (curResult.buildTraceContext?.entriesTrace) {
@@ -134,17 +134,16 @@ async function webpackBuildWithWorker(
   return combinedResult
 }
 
-export async function webpackBuild(
-  compilerNames?: typeof ORDERED_COMPILER_NAMES
-) {
-  const config = NextBuildContext.config!
-
-  if (config.experimental.webpackBuildWorker) {
+export function webpackBuild(
+  withWorker: boolean,
+  compilerNames: typeof ORDERED_COMPILER_NAMES | null
+): ReturnType<typeof webpackBuildWithWorker> {
+  if (withWorker) {
     debug('using separate compiler workers')
-    return await webpackBuildWithWorker(compilerNames)
+    return webpackBuildWithWorker(compilerNames)
   } else {
     debug('building all compilers in same process')
     const webpackBuildImpl = require('./impl').webpackBuildImpl
-    return await webpackBuildImpl()
+    return webpackBuildImpl(null, null)
   }
 }

packages/next/src/build/webpack-config.ts

@@ -75,6 +75,7 @@ import { createWebpackAliases } from './create-compiler-aliases'
 import { createServerOnlyClientOnlyAliases } from './create-compiler-aliases'
 import { createRSCAliases } from './create-compiler-aliases'
 import { createServerComponentsNoopAliases } from './create-compiler-aliases'
+import { hasCustomExportOutput } from '../export/utils'
 
 type ExcludesFalse = <T>(x: T | false) => x is T
 type ClientEntries = {
@@ -352,6 +353,10 @@ export default async function getBaseWebpackConfig(
     : ''
 
   const babelConfigFile = getBabelConfigFile(dir)
+
+  if (hasCustomExportOutput(config)) {
+    config.distDir = '.next'
+  }
   const distDir = path.join(dir, config.distDir)
 
   let useSWCLoader = !babelConfigFile || config.experimental.forceSwcTransforms

packages/next/src/build/webpack/plugins/pages-manifest-plugin.ts

@@ -132,17 +132,18 @@ export default class PagesManifestPlugin
         ...nodeServerPages,
       })
     } else {
-      assets[(!this.dev && !this.isEdgeRuntime ? '../' : '') + PAGES_MANIFEST] =
-        new sources.RawSource(
-          JSON.stringify(
-            {
-              ...edgeServerPages,
-              ...nodeServerPages,
-            },
-            null,
-            2
-          )
+      const pagesManifestPath =
+        (!this.dev && !this.isEdgeRuntime ? '../' : '') + PAGES_MANIFEST
+      assets[pagesManifestPath] = new sources.RawSource(
+        JSON.stringify(
+          {
+            ...edgeServerPages,
+            ...nodeServerPages,
+          },
+          null,
+          2
         )
+      )
     }
 
     if (this.appDirEnabled) {

packages/next/src/build/webpack/plugins/telemetry-plugin.ts

@@ -111,7 +111,7 @@ const BUILD_FEATURES: Array<Feature> = [
   'modularizeImports',
 ]
 
-const ELIMINATED_PACKAGES = new Set<string>()
+const eliminatedPackages = new Set<string>()
 
 /**
  * Determine if there is a feature of interest in the specified 'module'.
@@ -160,7 +160,10 @@ function findUniqueOriginModulesInConnections(
  * they are imported.
  */
 export class TelemetryPlugin implements webpack.WebpackPluginInstance {
-  private usageTracker = new Map<Feature, FeatureUsage>()
+  private usageTracker: Map<Feature, FeatureUsage> = new Map<
+    Feature,
+    FeatureUsage
+  >()
 
   // Build feature usage is on/off and is known before the build starts
   constructor(buildFeaturesMap: Map<Feature, boolean>) {
@@ -218,7 +221,7 @@ export class TelemetryPlugin implements webpack.WebpackPluginInstance {
       compiler.hooks.compilation.tap(TelemetryPlugin.name, (compilation) => {
         const moduleHooks = NormalModule.getCompilationHooks(compilation)
         moduleHooks.loader.tap(TelemetryPlugin.name, (loaderContext: any) => {
-          loaderContext.eliminatedPackages = ELIMINATED_PACKAGES
+          loaderContext.eliminatedPackages = eliminatedPackages
         })
       })
     }
@@ -229,6 +232,13 @@ export class TelemetryPlugin implements webpack.WebpackPluginInstance {
   }
 
   packagesUsedInServerSideProps(): string[] {
-    return Array.from(ELIMINATED_PACKAGES)
+    return Array.from(eliminatedPackages)
   }
 }
+
+export type TelemetryPluginState = {
+  usages: ReturnType<TelemetryPlugin['usages']>
+  packagesUsedInServerSideProps: ReturnType<
+    TelemetryPlugin['packagesUsedInServerSideProps']
+  >
+}

packages/next/src/export/utils.ts

@@ -0,0 +1,14 @@
+import type { NextConfigComplete } from '../server/config-shared'
+
+export function hasCustomExportOutput(config: NextConfigComplete) {
+  // In the past, a user had to run "next build" to generate
+  // ".next" (or whatever the distDir) followed by "next export"
+  // to generate "out" (or whatever the outDir). However, when
+  // "output: export" is configured, "next build" does both steps.
+  // So the user-configured distDir is actually the outDir.
+  // We'll do some custom logic when meeting this condition.
+  // e.g.
+  // Will set config.distDir to .next to make sure the manifests
+  // are still reading from temporary .next directory.
+  return config.output === 'export' && config.distDir !== '.next'
+}

packages/next/src/server/config-shared.ts

@@ -283,7 +283,11 @@ export interface ExperimentalConfig {
   typedRoutes?: boolean
 
   /**
-   * This option is to enable running the Webpack build in a worker thread.
+   * Run the Webpack build in a separate process to optimize memory usage during build.
+   * Valid values are:
+   * - `false`: Disable the Webpack build worker
+   * - `true`: Enable the Webpack build worker
+   * - `undefined`: Enable the Webpack build worker only if the webpack config is not customized
    */
   webpackBuildWorker?: boolean
 
@@ -806,7 +810,7 @@ export const defaultConfig: NextConfig = {
       process.env.__NEXT_EXPERIMENTAL_PPR === 'true'
         ? true
         : false,
-    webpackBuildWorker: false,
+    webpackBuildWorker: undefined,
   },
 }
 

packages/next/src/telemetry/events/build.ts

@@ -175,9 +175,9 @@ export type EventBuildFeatureUsage = {
   invocationCount: number
 }
 export function eventBuildFeatureUsage(
-  telemetryPlugin: TelemetryPlugin
+  usages: ReturnType<TelemetryPlugin['usages']>
 ): Array<{ eventName: string; payload: EventBuildFeatureUsage }> {
-  return telemetryPlugin.usages().map(({ featureName, invocationCount }) => ({
+  return usages.map(({ featureName, invocationCount }) => ({
     eventName: EVENT_BUILD_FEATURE_USAGE,
     payload: {
       featureName,
@@ -194,9 +194,11 @@ export type EventPackageUsedInGetServerSideProps = {
 }
 
 export function eventPackageUsedInGetServerSideProps(
-  telemetryPlugin: TelemetryPlugin
+  packagesUsedInServerSideProps: ReturnType<
+    TelemetryPlugin['packagesUsedInServerSideProps']
+  >
 ): Array<{ eventName: string; payload: EventPackageUsedInGetServerSideProps }> {
-  return telemetryPlugin.packagesUsedInServerSideProps().map((packageName) => ({
+  return packagesUsedInServerSideProps.map((packageName) => ({
     eventName: EVENT_NAME_PACKAGE_USED_IN_GET_SERVER_SIDE_PROPS,
     payload: {
       package: packageName,

test/integration/next-image-legacy/svgo-webpack/test/index.test.ts

@@ -23,7 +23,10 @@ let devOutput
       () => {
         it('should not fail to build invalid usage of the Image component', async () => {
           const { stderr, code } = await nextBuild(appDir, [], { stderr: true })
-          expect(stderr).toBeFalsy()
+          const errors = stderr
+            .split('\n')
+            .filter((line: string) => line && !line.trim().startsWith('⚠'))
+          expect(errors).toEqual([])
           expect(code).toBe(0)
         })
       }

test/integration/next-image-new/svgo-webpack/test/index.test.ts

@@ -25,7 +25,7 @@ let devOutput
           const { stderr, code } = await nextBuild(appDir, [], { stderr: true })
           const errors = stderr
             .split('\n')
-            .filter((line) => line && !line.trim().startsWith('⚠️'))
+            .filter((line: string) => line && !line.trim().startsWith('⚠'))
           expect(errors).toEqual([])
           expect(code).toBe(0)
         })

test/integration/webpack-require-hook/test/index.test.js

@@ -22,8 +22,10 @@ const appDir = path.join(__dirname, '..')
           stdout: true,
           stderr: true,
         })
-        console.log(stderr)
-        expect(stderr.length).toStrictEqual(0)
+        const errors = stderr
+          .split('\n')
+          .filter((line) => line && !line.trim().startsWith('⚠'))
+        expect(errors).toEqual([])
         expect(stdout).toMatch(/Initialized config/)
       })
     })