[Memory] Add option to reduce memory usage caused by duplicate strings in webpack-sources (#66003)
This PR adds a flag to Next.js to enable Webpack options to improve memory usage. See https://github.com/webpack/webpack-sources/pull/155 for a full description of the changes and impact on memory. This PR adds a patch to `webpack-sources` temporarily that contains the fixes as the real changes are iterated on to merge upstream in the `webpack/webpack-sources` repository. After that is done, the patch will be reverted and the latest `webpack-sources` version will be updated in Next.js.
This commit is contained in:
parent
b17ca02695
commit
8b360afb7b
8 changed files with 264 additions and 5 deletions
|
@ -262,5 +262,10 @@
|
|||
"node": ">=18.17.0",
|
||||
"pnpm": "8.15.7"
|
||||
},
|
||||
"packageManager": "pnpm@8.15.7"
|
||||
"packageManager": "pnpm@8.15.7",
|
||||
"pnpm": {
|
||||
"patchedDependencies": {
|
||||
"webpack-sources@3.2.3": "patches/webpack-sources@3.2.3.patch"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import type { webpack } from 'next/dist/compiled/webpack/webpack'
|
||||
import { stringBufferUtils } from 'next/dist/compiled/webpack-sources3'
|
||||
import { red } from '../../lib/picocolors'
|
||||
import formatWebpackMessages from '../../client/components/react-dev-overlay/internal/helpers/format-webpack-messages'
|
||||
import { nonNullable } from '../../lib/non-nullable'
|
||||
|
@ -185,6 +186,11 @@ export async function webpackBuildImpl(
|
|||
debug(`starting compiler`, compilerName)
|
||||
// We run client and server compilation separately to optimize for memory usage
|
||||
await runWebpackSpan.traceAsyncFn(async () => {
|
||||
if (config.experimental.webpackMemoryOptimizations) {
|
||||
stringBufferUtils.disableDualStringBufferCaching()
|
||||
stringBufferUtils.enableStringInterning()
|
||||
}
|
||||
|
||||
// Run the server compilers first and then the client
|
||||
// compiler to track the boundary of server/client components.
|
||||
let clientResult: SingleCompilerResult | null = null
|
||||
|
@ -254,6 +260,9 @@ export async function webpackBuildImpl(
|
|||
}
|
||||
}
|
||||
|
||||
if (config.experimental.webpackMemoryOptimizations) {
|
||||
stringBufferUtils.disableStringInterning()
|
||||
}
|
||||
inputFileSystem?.purge?.()
|
||||
|
||||
result = {
|
||||
|
|
File diff suppressed because one or more lines are too long
|
@ -365,6 +365,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
|
|||
.optional(),
|
||||
typedRoutes: z.boolean().optional(),
|
||||
webpackBuildWorker: z.boolean().optional(),
|
||||
webpackMemoryOptimizations: z.boolean().optional(),
|
||||
turbo: z
|
||||
.object({
|
||||
loaders: z.record(z.string(), z.array(zTurboLoaderItem)).optional(),
|
||||
|
|
|
@ -372,6 +372,15 @@ export interface ExperimentalConfig {
|
|||
*/
|
||||
webpackBuildWorker?: boolean
|
||||
|
||||
/**
|
||||
* Enables optimizations to reduce memory usage in Webpack. This reduces the max size of the heap
|
||||
* but may increase compile times slightly.
|
||||
* Valid values are:
|
||||
* - `false`: Disable Webpack memory optimizations (default).
|
||||
* - `true`: Enables Webpack memory optimizations.
|
||||
*/
|
||||
webpackMemoryOptimizations?: boolean
|
||||
|
||||
/**
|
||||
*
|
||||
*/
|
||||
|
@ -961,6 +970,7 @@ export const defaultConfig: NextConfig = {
|
|||
process.env.__NEXT_EXPERIMENTAL_PPR === 'true'
|
||||
),
|
||||
webpackBuildWorker: undefined,
|
||||
webpackMemoryOptimizations: false,
|
||||
optimizeServerReact: true,
|
||||
useEarlyImport: false,
|
||||
staleTimes: {
|
||||
|
|
10
packages/next/types/$$compiled.internal.d.ts
vendored
10
packages/next/types/$$compiled.internal.d.ts
vendored
|
@ -450,6 +450,16 @@ declare module 'next/dist/compiled/zod' {
|
|||
declare module 'mini-css-extract-plugin'
|
||||
declare module 'next/dist/compiled/loader-utils3'
|
||||
|
||||
declare module 'next/dist/compiled/webpack-sources3' {
|
||||
interface StringBufferUtils {
|
||||
disableDualStringBufferCaching: () => boolean
|
||||
disableStringInterning: () => boolean
|
||||
enableDualStringBufferCaching: () => boolean
|
||||
enableStringInterning: () => boolean
|
||||
}
|
||||
export let stringBufferUtils: StringBufferUtils
|
||||
}
|
||||
|
||||
declare module 'next/dist/compiled/webpack/webpack' {
|
||||
import type webpackSources from 'webpack-sources1'
|
||||
export function init(): void
|
||||
|
|
218
patches/webpack-sources@3.2.3.patch
Normal file
218
patches/webpack-sources@3.2.3.patch
Normal file
|
@ -0,0 +1,218 @@
|
|||
diff --git a/lib/RawSource.js b/lib/RawSource.js
|
||||
index 098d317..06e6b8f 100644
|
||||
--- a/lib/RawSource.js
|
||||
+++ b/lib/RawSource.js
|
||||
@@ -6,6 +6,10 @@
|
||||
"use strict";
|
||||
|
||||
const streamChunksOfRawSource = require("./helpers/streamChunksOfRawSource");
|
||||
+const {
|
||||
+ internString,
|
||||
+ isDualStringBufferCachingEnabled
|
||||
+} = require("./helpers/stringBufferUtils");
|
||||
const Source = require("./Source");
|
||||
|
||||
class RawSource extends Source {
|
||||
@@ -17,8 +21,13 @@ class RawSource extends Source {
|
||||
}
|
||||
this._valueIsBuffer = !convertToString && isBuffer;
|
||||
- this._value = convertToString && isBuffer ? undefined : value;
|
||||
+ this._value =
|
||||
+ convertToString && isBuffer
|
||||
+ ? undefined
|
||||
+ : typeof value === "string"
|
||||
+ ? internString(value)
|
||||
+ : value;
|
||||
this._valueAsBuffer = isBuffer ? value : undefined;
|
||||
- this._valueAsString = isBuffer ? undefined : value;
|
||||
+ this._valueAsString = isBuffer ? undefined : internString(value);
|
||||
}
|
||||
|
||||
isBuffer() {
|
||||
@@ -27,14 +36,22 @@ class RawSource extends Source {
|
||||
|
||||
source() {
|
||||
if (this._value === undefined) {
|
||||
- this._value = this._valueAsBuffer.toString("utf-8");
|
||||
+ const value = internString(this._valueAsBuffer.toString("utf-8"));
|
||||
+ if (isDualStringBufferCachingEnabled()) {
|
||||
+ this._value = value;
|
||||
+ }
|
||||
+ return value;
|
||||
}
|
||||
return this._value;
|
||||
}
|
||||
|
||||
buffer() {
|
||||
if (this._valueAsBuffer === undefined) {
|
||||
- this._valueAsBuffer = Buffer.from(this._value, "utf-8");
|
||||
+ const value = Buffer.from(this._value, "utf-8");
|
||||
+ if (isDualStringBufferCachingEnabled()) {
|
||||
+ this._valueAsBuffer = value;
|
||||
+ }
|
||||
+ return value;
|
||||
}
|
||||
return this._valueAsBuffer;
|
||||
}
|
||||
@@ -51,17 +68,21 @@ class RawSource extends Source {
|
||||
* @returns {void}
|
||||
*/
|
||||
streamChunks(options, onChunk, onSource, onName) {
|
||||
- if (this._value === undefined) {
|
||||
+ if (this._value === undefined && isDualStringBufferCachingEnabled()) {
|
||||
this._value = Buffer.from(this._valueAsBuffer, "utf-8");
|
||||
}
|
||||
- if (this._valueAsString === undefined) {
|
||||
- this._valueAsString =
|
||||
+ let strValue = this._valueAsString;
|
||||
+ if (strValue === undefined) {
|
||||
+ strValue =
|
||||
typeof this._value === "string"
|
||||
? this._value
|
||||
- : this._value.toString("utf-8");
|
||||
+ : internString(this._value.toString("utf-8"));
|
||||
+ if (isDualStringBufferCachingEnabled()) {
|
||||
+ this._valueAsString = strValue;
|
||||
+ }
|
||||
}
|
||||
return streamChunksOfRawSource(
|
||||
- this._valueAsString,
|
||||
+ strValue,
|
||||
onChunk,
|
||||
onSource,
|
||||
onName,
|
||||
@@ -70,11 +91,8 @@ class RawSource extends Source {
|
||||
}
|
||||
|
||||
updateHash(hash) {
|
||||
- if (this._valueAsBuffer === undefined) {
|
||||
- this._valueAsBuffer = Buffer.from(this._value, "utf-8");
|
||||
- }
|
||||
hash.update("RawSource");
|
||||
- hash.update(this._valueAsBuffer);
|
||||
+ hash.update(this.buffer());
|
||||
}
|
||||
}
|
||||
|
||||
diff --git a/lib/helpers/stringBufferUtils.js b/lib/helpers/stringBufferUtils.js
|
||||
new file mode 100644
|
||||
index 0000000..3b210f1
|
||||
--- /dev/null
|
||||
+++ b/lib/helpers/stringBufferUtils.js
|
||||
@@ -0,0 +1,107 @@
|
||||
+/*
|
||||
+ MIT License http://www.opensource.org/licenses/mit-license.php
|
||||
+ Author Mark Knichel @mknichel
|
||||
+*/
|
||||
+
|
||||
+"use strict";
|
||||
+
|
||||
+let dualStringBufferCaching = true;
|
||||
+
|
||||
+/**
|
||||
+ * @returns {boolean} Whether the optimization to cache copies of both the
|
||||
+ * string and buffer version of source content is enabled. This is enabled by
|
||||
+ * default to improve performance but can consume more memory since values are
|
||||
+ * stored twice.
|
||||
+ */
|
||||
+function isDualStringBufferCachingEnabled() {
|
||||
+ return dualStringBufferCaching;
|
||||
+}
|
||||
+
|
||||
+/**
|
||||
+ * Enables an optimization to save both string and buffer in memory to avoid
|
||||
+ * repeat conversions between the two formats when they are requested. This
|
||||
+ * is enabled by default. This option can improve performance but can consume
|
||||
+ * additional memory since values are stored twice.
|
||||
+ *
|
||||
+ * @returns {void}
|
||||
+ */
|
||||
+function enableDualStringBufferCaching() {
|
||||
+ dualStringBufferCaching = true;
|
||||
+}
|
||||
+
|
||||
+/**
|
||||
+ * Disables the optimization to save both string and buffer in memory. This
|
||||
+ * may increase performance but should reduce memory usage in the Webpack
|
||||
+ * compiler.
|
||||
+ *
|
||||
+ * @returns {void}
|
||||
+ */
|
||||
+function disableDualStringBufferCaching() {
|
||||
+ dualStringBufferCaching = false;
|
||||
+}
|
||||
+
|
||||
+const interningStringMap = new Map();
|
||||
+
|
||||
+/**
|
||||
+ * Saves the string in a map to ensure that only one copy of the string exists
|
||||
+ * in memory at a given time. This is controlled by {@link enableStringInterning}
|
||||
+ * and {@link disableStringInterning}. Callers are expect to manage the memory
|
||||
+ * of the interned strings by calling {@link disableStringInterning} after the
|
||||
+ * compiler no longer needs to save the interned memory.
|
||||
+ *
|
||||
+ * @param {string} str A string to be interned.
|
||||
+ * @returns {string} The original string or a reference to an existing string
|
||||
+ * of the same value if it has already been interned.
|
||||
+ */
|
||||
+function internString(str) {
|
||||
+ if (!isStringInterningEnabled() || !str || typeof str !== "string") {
|
||||
+ return str;
|
||||
+ }
|
||||
+ let internedString = interningStringMap.get(str);
|
||||
+ if (internedString === undefined) {
|
||||
+ internedString = str;
|
||||
+ interningStringMap.set(str, internedString);
|
||||
+ }
|
||||
+ return internedString;
|
||||
+}
|
||||
+
|
||||
+let enableStringInterningRefCount = 0;
|
||||
+
|
||||
+function isStringInterningEnabled() {
|
||||
+ return enableStringInterningRefCount > 0;
|
||||
+}
|
||||
+
|
||||
+/**
|
||||
+ * Enables a memory optimization to avoid repeat copies of the same string in
|
||||
+ * memory by caching a single reference to the string. This can reduce memory
|
||||
+ * usage if the same string is repeated many times in the compiler, such as
|
||||
+ * when Webpack layers are used with the same files.
|
||||
+ *
|
||||
+ * @returns {void}
|
||||
+ */
|
||||
+function enableStringInterning() {
|
||||
+ enableStringInterningRefCount++;
|
||||
+}
|
||||
+
|
||||
+/**
|
||||
+ * Disables string interning. This should be called to free the memory used by
|
||||
+ * the interned strings after the compiler no longer needs to reuse the
|
||||
+ * interned strings such as at the end of the compilation.
|
||||
+ *
|
||||
+ * @returns {void}
|
||||
+ */
|
||||
+function disableStringInterning() {
|
||||
+ if (--enableStringInterningRefCount <= 0) {
|
||||
+ interningStringMap.clear();
|
||||
+ enableStringInterningRefCount = 0;
|
||||
+ }
|
||||
+}
|
||||
+
|
||||
+module.exports = {
|
||||
+ disableDualStringBufferCaching,
|
||||
+ disableStringInterning,
|
||||
+ enableDualStringBufferCaching,
|
||||
+ enableStringInterning,
|
||||
+ internString,
|
||||
+ isDualStringBufferCachingEnabled
|
||||
+};
|
||||
diff --git a/lib/index.js b/lib/index.js
|
||||
index 0c11c2f..86a7234 100644
|
||||
--- a/lib/index.js
|
||||
+++ b/lib/index.js
|
||||
@@ -28,3 +28,4 @@ defineExport("ReplaceSource", () => require("./ReplaceSource"));
|
||||
defineExport("PrefixSource", () => require("./PrefixSource"));
|
||||
defineExport("SizeOnlySource", () => require("./SizeOnlySource"));
|
||||
defineExport("CompatSource", () => require("./CompatSource"));
|
||||
+defineExport("stringBufferUtils", () => require("./helpers/stringBufferUtils"));
|
|
@ -20,6 +20,11 @@ overrides:
|
|||
react-is: 19.0.0-rc-81c5ff2e04-20240521
|
||||
scheduler: 0.25.0-rc-81c5ff2e04-20240521
|
||||
|
||||
patchedDependencies:
|
||||
webpack-sources@3.2.3:
|
||||
hash: exarmjd4pnde4auoeobjrmju54
|
||||
path: patches/webpack-sources@3.2.3.patch
|
||||
|
||||
importers:
|
||||
|
||||
.:
|
||||
|
@ -1449,7 +1454,7 @@ importers:
|
|||
version: /webpack-sources@1.4.3
|
||||
webpack-sources3:
|
||||
specifier: npm:webpack-sources@3.2.3
|
||||
version: /webpack-sources@3.2.3
|
||||
version: /webpack-sources@3.2.3(patch_hash=exarmjd4pnde4auoeobjrmju54)
|
||||
ws:
|
||||
specifier: 8.2.3
|
||||
version: 8.2.3
|
||||
|
@ -25273,9 +25278,10 @@ packages:
|
|||
source-map: 0.6.1
|
||||
dev: true
|
||||
|
||||
/webpack-sources@3.2.3:
|
||||
/webpack-sources@3.2.3(patch_hash=exarmjd4pnde4auoeobjrmju54):
|
||||
resolution: {integrity: sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==}
|
||||
engines: {node: '>=10.13.0'}
|
||||
patched: true
|
||||
|
||||
/webpack-stats-plugin@1.1.0:
|
||||
resolution: {integrity: sha512-D0meHk1WYryUbuCnWJuomJFAYvqs0rxv/JFu1XJT1YYpczdgnP1/vz+u/5Z31jrTxT6dJSxCg+TuKTgjhoZS6g==}
|
||||
|
@ -25314,7 +25320,7 @@ packages:
|
|||
tapable: 2.2.0
|
||||
terser-webpack-plugin: 5.3.10(@swc/core@1.5.7)(webpack@5.90.0)
|
||||
watchpack: 2.4.0
|
||||
webpack-sources: 3.2.3
|
||||
webpack-sources: 3.2.3(patch_hash=exarmjd4pnde4auoeobjrmju54)
|
||||
transitivePeerDependencies:
|
||||
- '@swc/core'
|
||||
- esbuild
|
||||
|
|
Loading…
Reference in a new issue