[Memory] Add option to reduce memory usage caused by duplicate strings in webpack-sources (#66003)

This PR adds a flag to Next.js to enable Webpack options to improve
memory usage. See https://github.com/webpack/webpack-sources/pull/155
for a full description of the changes and impact on memory.

This PR adds a patch to `webpack-sources` temporarily that contains the
fixes as the real changes are iterated on to merge upstream in the
`webpack/webpack-sources` repository. After that is done, the patch will
be reverted and the latest `webpack-sources` version will be updated in
Next.js.

package.json

@@ -262,5 +262,10 @@
     "node": ">=18.17.0",
     "pnpm": "8.15.7"
   },
-  "packageManager": "pnpm@8.15.7"
+  "packageManager": "pnpm@8.15.7",
+  "pnpm": {
+    "patchedDependencies": {
+      "webpack-sources@3.2.3": "patches/webpack-sources@3.2.3.patch"
+    }
+  }
 }

packages/next/src/build/webpack-build/impl.ts

@@ -1,4 +1,5 @@
 import type { webpack } from 'next/dist/compiled/webpack/webpack'
+import { stringBufferUtils } from 'next/dist/compiled/webpack-sources3'
 import { red } from '../../lib/picocolors'
 import formatWebpackMessages from '../../client/components/react-dev-overlay/internal/helpers/format-webpack-messages'
 import { nonNullable } from '../../lib/non-nullable'
@@ -185,6 +186,11 @@ export async function webpackBuildImpl(
   debug(`starting compiler`, compilerName)
   // We run client and server compilation separately to optimize for memory usage
   await runWebpackSpan.traceAsyncFn(async () => {
+    if (config.experimental.webpackMemoryOptimizations) {
+      stringBufferUtils.disableDualStringBufferCaching()
+      stringBufferUtils.enableStringInterning()
+    }
+
     // Run the server compilers first and then the client
     // compiler to track the boundary of server/client components.
     let clientResult: SingleCompilerResult | null = null
@@ -254,6 +260,9 @@ export async function webpackBuildImpl(
       }
     }
 
+    if (config.experimental.webpackMemoryOptimizations) {
+      stringBufferUtils.disableStringInterning()
+    }
     inputFileSystem?.purge?.()
 
     result = {

packages/next/src/server/config-schema.ts

@@ -365,6 +365,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
           .optional(),
         typedRoutes: z.boolean().optional(),
         webpackBuildWorker: z.boolean().optional(),
+        webpackMemoryOptimizations: z.boolean().optional(),
         turbo: z
           .object({
             loaders: z.record(z.string(), z.array(zTurboLoaderItem)).optional(),

packages/next/src/server/config-shared.ts

@@ -372,6 +372,15 @@ export interface ExperimentalConfig {
    */
   webpackBuildWorker?: boolean
 
+  /**
+   * Enables optimizations to reduce memory usage in Webpack. This reduces the max size of the heap
+   * but may increase compile times slightly.
+   * Valid values are:
+   * - `false`: Disable Webpack memory optimizations (default).
+   * - `true`: Enables Webpack memory optimizations.
+   */
+  webpackMemoryOptimizations?: boolean
+
   /**
    *
    */
@@ -961,6 +970,7 @@ export const defaultConfig: NextConfig = {
         process.env.__NEXT_EXPERIMENTAL_PPR === 'true'
       ),
     webpackBuildWorker: undefined,
+    webpackMemoryOptimizations: false,
     optimizeServerReact: true,
     useEarlyImport: false,
     staleTimes: {

packages/next/types/$$compiled.internal.d.ts

@@ -450,6 +450,16 @@ declare module 'next/dist/compiled/zod' {
 declare module 'mini-css-extract-plugin'
 declare module 'next/dist/compiled/loader-utils3'
 
+declare module 'next/dist/compiled/webpack-sources3' {
+  interface StringBufferUtils {
+    disableDualStringBufferCaching: () => boolean
+    disableStringInterning: () => boolean
+    enableDualStringBufferCaching: () => boolean
+    enableStringInterning: () => boolean
+  }
+  export let stringBufferUtils: StringBufferUtils
+}
+
 declare module 'next/dist/compiled/webpack/webpack' {
   import type webpackSources from 'webpack-sources1'
   export function init(): void

patches/webpack-sources@3.2.3.patch

@@ -0,0 +1,218 @@
+diff --git a/lib/RawSource.js b/lib/RawSource.js
+index 098d317..06e6b8f 100644
+--- a/lib/RawSource.js
++++ b/lib/RawSource.js
+@@ -6,6 +6,10 @@
+ "use strict";
+ 
+ const streamChunksOfRawSource = require("./helpers/streamChunksOfRawSource");
++const {
++	internString,
++	isDualStringBufferCachingEnabled
++} = require("./helpers/stringBufferUtils");
+ const Source = require("./Source");
+ 
+ class RawSource extends Source {
+@@ -17,8 +21,13 @@ class RawSource extends Source {
+ 		}
+ 		this._valueIsBuffer = !convertToString && isBuffer;
+-		this._value = convertToString && isBuffer ? undefined : value;
++		this._value =
++			convertToString && isBuffer
++				? undefined
++				: typeof value === "string"
++				? internString(value)
++				: value;
+ 		this._valueAsBuffer = isBuffer ? value : undefined;
+-		this._valueAsString = isBuffer ? undefined : value;
++		this._valueAsString = isBuffer ? undefined : internString(value);
+ 	}
+ 
+ 	isBuffer() {
+@@ -27,14 +36,22 @@ class RawSource extends Source {
+ 
+ 	source() {
+ 		if (this._value === undefined) {
+-			this._value = this._valueAsBuffer.toString("utf-8");
++			const value = internString(this._valueAsBuffer.toString("utf-8"));
++			if (isDualStringBufferCachingEnabled()) {
++				this._value = value;
++			}
++			return value;
+ 		}
+ 		return this._value;
+ 	}
+ 
+ 	buffer() {
+ 		if (this._valueAsBuffer === undefined) {
+-			this._valueAsBuffer = Buffer.from(this._value, "utf-8");
++			const value = Buffer.from(this._value, "utf-8");
++			if (isDualStringBufferCachingEnabled()) {
++				this._valueAsBuffer = value;
++			}
++			return value;
+ 		}
+ 		return this._valueAsBuffer;
+ 	}
+@@ -51,17 +68,21 @@ class RawSource extends Source {
+ 	 * @returns {void}
+ 	 */
+ 	streamChunks(options, onChunk, onSource, onName) {
+-		if (this._value === undefined) {
++		if (this._value === undefined && isDualStringBufferCachingEnabled()) {
+ 			this._value = Buffer.from(this._valueAsBuffer, "utf-8");
+ 		}
+-		if (this._valueAsString === undefined) {
+-			this._valueAsString =
++		let strValue = this._valueAsString;
++		if (strValue === undefined) {
++			strValue =
+ 				typeof this._value === "string"
+ 					? this._value
+-					: this._value.toString("utf-8");
++					: internString(this._value.toString("utf-8"));
++			if (isDualStringBufferCachingEnabled()) {
++				this._valueAsString = strValue;
++			}
+ 		}
+ 		return streamChunksOfRawSource(
+-			this._valueAsString,
++			strValue,
+ 			onChunk,
+ 			onSource,
+ 			onName,
+@@ -70,11 +91,8 @@ class RawSource extends Source {
+ 	}
+ 
+ 	updateHash(hash) {
+-		if (this._valueAsBuffer === undefined) {
+-			this._valueAsBuffer = Buffer.from(this._value, "utf-8");
+-		}
+ 		hash.update("RawSource");
+-		hash.update(this._valueAsBuffer);
++		hash.update(this.buffer());
+ 	}
+ }
+ 
+diff --git a/lib/helpers/stringBufferUtils.js b/lib/helpers/stringBufferUtils.js
+new file mode 100644
+index 0000000..3b210f1
+--- /dev/null
++++ b/lib/helpers/stringBufferUtils.js
+@@ -0,0 +1,107 @@
++/*
++	MIT License http://www.opensource.org/licenses/mit-license.php
++	Author Mark Knichel @mknichel
++*/
++
++"use strict";
++
++let dualStringBufferCaching = true;
++
++/**
++ * @returns {boolean} Whether the optimization to cache copies of both the
++ * string and buffer version of source content is enabled. This is enabled by
++ * default to improve performance but can consume more memory since values are
++ * stored twice.
++ */
++function isDualStringBufferCachingEnabled() {
++	return dualStringBufferCaching;
++}
++
++/**
++ * Enables an optimization to save both string and buffer in memory to avoid
++ * repeat conversions between the two formats when they are requested. This
++ * is enabled by default. This option can improve performance but can consume
++ * additional memory since values are stored twice.
++ *
++ * @returns {void}
++ */
++function enableDualStringBufferCaching() {
++	dualStringBufferCaching = true;
++}
++
++/**
++ * Disables the optimization to save both string and buffer in memory. This
++ * may increase performance but should reduce memory usage in the Webpack
++ * compiler.
++ *
++ * @returns {void}
++ */
++function disableDualStringBufferCaching() {
++	dualStringBufferCaching = false;
++}
++
++const interningStringMap = new Map();
++
++/**
++ * Saves the string in a map to ensure that only one copy of the string exists
++ * in memory at a given time. This is controlled by {@link enableStringInterning}
++ * and {@link disableStringInterning}. Callers are expect to manage the memory
++ * of the interned strings by calling {@link disableStringInterning} after the
++ * compiler no longer needs to save the interned memory.
++ *
++ * @param {string} str A string to be interned.
++ * @returns {string} The original string or a reference to an existing string
++ * of the same value if it has already been interned.
++ */
++function internString(str) {
++	if (!isStringInterningEnabled() || !str || typeof str !== "string") {
++		return str;
++	}
++	let internedString = interningStringMap.get(str);
++	if (internedString === undefined) {
++		internedString = str;
++		interningStringMap.set(str, internedString);
++	}
++	return internedString;
++}
++
++let enableStringInterningRefCount = 0;
++
++function isStringInterningEnabled() {
++	return enableStringInterningRefCount > 0;
++}
++
++/**
++ * Enables a memory optimization to avoid repeat copies of the same string in
++ * memory by caching a single reference to the string. This can reduce memory
++ * usage if the same string is repeated many times in the compiler, such as
++ * when Webpack layers are used with the same files.
++ *
++ * @returns {void}
++ */
++function enableStringInterning() {
++	enableStringInterningRefCount++;
++}
++
++/**
++ * Disables string interning. This should be called to free the memory used by
++ * the interned strings after the compiler no longer needs to reuse the
++ * interned strings such as at the end of the compilation.
++ *
++ * @returns {void}
++ */
++function disableStringInterning() {
++	if (--enableStringInterningRefCount <= 0) {
++		interningStringMap.clear();
++		enableStringInterningRefCount = 0;
++	}
++}
++
++module.exports = {
++	disableDualStringBufferCaching,
++	disableStringInterning,
++	enableDualStringBufferCaching,
++	enableStringInterning,
++	internString,
++	isDualStringBufferCachingEnabled
++};
+diff --git a/lib/index.js b/lib/index.js
+index 0c11c2f..86a7234 100644
+--- a/lib/index.js
++++ b/lib/index.js
+@@ -28,3 +28,4 @@ defineExport("ReplaceSource", () => require("./ReplaceSource"));
+ defineExport("PrefixSource", () => require("./PrefixSource"));
+ defineExport("SizeOnlySource", () => require("./SizeOnlySource"));
+ defineExport("CompatSource", () => require("./CompatSource"));
++defineExport("stringBufferUtils", () => require("./helpers/stringBufferUtils"));

pnpm-lock.yaml

@@ -20,6 +20,11 @@ overrides:
   react-is: 19.0.0-rc-81c5ff2e04-20240521
   scheduler: 0.25.0-rc-81c5ff2e04-20240521
 
+patchedDependencies:
+  webpack-sources@3.2.3:
+    hash: exarmjd4pnde4auoeobjrmju54
+    path: patches/webpack-sources@3.2.3.patch
+
 importers:
 
   .:
@@ -1449,7 +1454,7 @@ importers:
         version: /webpack-sources@1.4.3
       webpack-sources3:
         specifier: npm:webpack-sources@3.2.3
-        version: /webpack-sources@3.2.3
+        version: /webpack-sources@3.2.3(patch_hash=exarmjd4pnde4auoeobjrmju54)
       ws:
         specifier: 8.2.3
         version: 8.2.3
@@ -25273,9 +25278,10 @@ packages:
       source-map: 0.6.1
     dev: true
 
-  /webpack-sources@3.2.3:
+  /webpack-sources@3.2.3(patch_hash=exarmjd4pnde4auoeobjrmju54):
     resolution: {integrity: sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==}
     engines: {node: '>=10.13.0'}
+    patched: true
 
   /webpack-stats-plugin@1.1.0:
     resolution: {integrity: sha512-D0meHk1WYryUbuCnWJuomJFAYvqs0rxv/JFu1XJT1YYpczdgnP1/vz+u/5Z31jrTxT6dJSxCg+TuKTgjhoZS6g==}
@@ -25314,7 +25320,7 @@ packages:
       tapable: 2.2.0
       terser-webpack-plugin: 5.3.10(@swc/core@1.5.7)(webpack@5.90.0)
       watchpack: 2.4.0
-      webpack-sources: 3.2.3
+      webpack-sources: 3.2.3(patch_hash=exarmjd4pnde4auoeobjrmju54)
     transitivePeerDependencies:
       - '@swc/core'
       - esbuild