[Memory] Add option to reduce memory usage caused by duplicate strings in webpack-sources (#66003)
This PR adds a flag to Next.js to enable Webpack options to improve memory usage. See https://github.com/webpack/webpack-sources/pull/155 for a full description of the changes and impact on memory. This PR adds a patch to `webpack-sources` temporarily that contains the fixes as the real changes are iterated on to merge upstream in the `webpack/webpack-sources` repository. After that is done, the patch will be reverted and the latest `webpack-sources` version will be updated in Next.js.
This commit is contained in:
parent
b17ca02695
commit
8b360afb7b
8 changed files with 264 additions and 5 deletions
|
@ -262,5 +262,10 @@
|
||||||
"node": ">=18.17.0",
|
"node": ">=18.17.0",
|
||||||
"pnpm": "8.15.7"
|
"pnpm": "8.15.7"
|
||||||
},
|
},
|
||||||
"packageManager": "pnpm@8.15.7"
|
"packageManager": "pnpm@8.15.7",
|
||||||
|
"pnpm": {
|
||||||
|
"patchedDependencies": {
|
||||||
|
"webpack-sources@3.2.3": "patches/webpack-sources@3.2.3.patch"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
import type { webpack } from 'next/dist/compiled/webpack/webpack'
|
import type { webpack } from 'next/dist/compiled/webpack/webpack'
|
||||||
|
import { stringBufferUtils } from 'next/dist/compiled/webpack-sources3'
|
||||||
import { red } from '../../lib/picocolors'
|
import { red } from '../../lib/picocolors'
|
||||||
import formatWebpackMessages from '../../client/components/react-dev-overlay/internal/helpers/format-webpack-messages'
|
import formatWebpackMessages from '../../client/components/react-dev-overlay/internal/helpers/format-webpack-messages'
|
||||||
import { nonNullable } from '../../lib/non-nullable'
|
import { nonNullable } from '../../lib/non-nullable'
|
||||||
|
@ -185,6 +186,11 @@ export async function webpackBuildImpl(
|
||||||
debug(`starting compiler`, compilerName)
|
debug(`starting compiler`, compilerName)
|
||||||
// We run client and server compilation separately to optimize for memory usage
|
// We run client and server compilation separately to optimize for memory usage
|
||||||
await runWebpackSpan.traceAsyncFn(async () => {
|
await runWebpackSpan.traceAsyncFn(async () => {
|
||||||
|
if (config.experimental.webpackMemoryOptimizations) {
|
||||||
|
stringBufferUtils.disableDualStringBufferCaching()
|
||||||
|
stringBufferUtils.enableStringInterning()
|
||||||
|
}
|
||||||
|
|
||||||
// Run the server compilers first and then the client
|
// Run the server compilers first and then the client
|
||||||
// compiler to track the boundary of server/client components.
|
// compiler to track the boundary of server/client components.
|
||||||
let clientResult: SingleCompilerResult | null = null
|
let clientResult: SingleCompilerResult | null = null
|
||||||
|
@ -254,6 +260,9 @@ export async function webpackBuildImpl(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (config.experimental.webpackMemoryOptimizations) {
|
||||||
|
stringBufferUtils.disableStringInterning()
|
||||||
|
}
|
||||||
inputFileSystem?.purge?.()
|
inputFileSystem?.purge?.()
|
||||||
|
|
||||||
result = {
|
result = {
|
||||||
|
|
File diff suppressed because one or more lines are too long
|
@ -365,6 +365,7 @@ export const configSchema: zod.ZodType<NextConfig> = z.lazy(() =>
|
||||||
.optional(),
|
.optional(),
|
||||||
typedRoutes: z.boolean().optional(),
|
typedRoutes: z.boolean().optional(),
|
||||||
webpackBuildWorker: z.boolean().optional(),
|
webpackBuildWorker: z.boolean().optional(),
|
||||||
|
webpackMemoryOptimizations: z.boolean().optional(),
|
||||||
turbo: z
|
turbo: z
|
||||||
.object({
|
.object({
|
||||||
loaders: z.record(z.string(), z.array(zTurboLoaderItem)).optional(),
|
loaders: z.record(z.string(), z.array(zTurboLoaderItem)).optional(),
|
||||||
|
|
|
@ -372,6 +372,15 @@ export interface ExperimentalConfig {
|
||||||
*/
|
*/
|
||||||
webpackBuildWorker?: boolean
|
webpackBuildWorker?: boolean
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Enables optimizations to reduce memory usage in Webpack. This reduces the max size of the heap
|
||||||
|
* but may increase compile times slightly.
|
||||||
|
* Valid values are:
|
||||||
|
* - `false`: Disable Webpack memory optimizations (default).
|
||||||
|
* - `true`: Enables Webpack memory optimizations.
|
||||||
|
*/
|
||||||
|
webpackMemoryOptimizations?: boolean
|
||||||
|
|
||||||
/**
|
/**
|
||||||
*
|
*
|
||||||
*/
|
*/
|
||||||
|
@ -961,6 +970,7 @@ export const defaultConfig: NextConfig = {
|
||||||
process.env.__NEXT_EXPERIMENTAL_PPR === 'true'
|
process.env.__NEXT_EXPERIMENTAL_PPR === 'true'
|
||||||
),
|
),
|
||||||
webpackBuildWorker: undefined,
|
webpackBuildWorker: undefined,
|
||||||
|
webpackMemoryOptimizations: false,
|
||||||
optimizeServerReact: true,
|
optimizeServerReact: true,
|
||||||
useEarlyImport: false,
|
useEarlyImport: false,
|
||||||
staleTimes: {
|
staleTimes: {
|
||||||
|
|
10
packages/next/types/$$compiled.internal.d.ts
vendored
10
packages/next/types/$$compiled.internal.d.ts
vendored
|
@ -450,6 +450,16 @@ declare module 'next/dist/compiled/zod' {
|
||||||
declare module 'mini-css-extract-plugin'
|
declare module 'mini-css-extract-plugin'
|
||||||
declare module 'next/dist/compiled/loader-utils3'
|
declare module 'next/dist/compiled/loader-utils3'
|
||||||
|
|
||||||
|
declare module 'next/dist/compiled/webpack-sources3' {
|
||||||
|
interface StringBufferUtils {
|
||||||
|
disableDualStringBufferCaching: () => boolean
|
||||||
|
disableStringInterning: () => boolean
|
||||||
|
enableDualStringBufferCaching: () => boolean
|
||||||
|
enableStringInterning: () => boolean
|
||||||
|
}
|
||||||
|
export let stringBufferUtils: StringBufferUtils
|
||||||
|
}
|
||||||
|
|
||||||
declare module 'next/dist/compiled/webpack/webpack' {
|
declare module 'next/dist/compiled/webpack/webpack' {
|
||||||
import type webpackSources from 'webpack-sources1'
|
import type webpackSources from 'webpack-sources1'
|
||||||
export function init(): void
|
export function init(): void
|
||||||
|
|
218
patches/webpack-sources@3.2.3.patch
Normal file
218
patches/webpack-sources@3.2.3.patch
Normal file
|
@ -0,0 +1,218 @@
|
||||||
|
diff --git a/lib/RawSource.js b/lib/RawSource.js
|
||||||
|
index 098d317..06e6b8f 100644
|
||||||
|
--- a/lib/RawSource.js
|
||||||
|
+++ b/lib/RawSource.js
|
||||||
|
@@ -6,6 +6,10 @@
|
||||||
|
"use strict";
|
||||||
|
|
||||||
|
const streamChunksOfRawSource = require("./helpers/streamChunksOfRawSource");
|
||||||
|
+const {
|
||||||
|
+ internString,
|
||||||
|
+ isDualStringBufferCachingEnabled
|
||||||
|
+} = require("./helpers/stringBufferUtils");
|
||||||
|
const Source = require("./Source");
|
||||||
|
|
||||||
|
class RawSource extends Source {
|
||||||
|
@@ -17,8 +21,13 @@ class RawSource extends Source {
|
||||||
|
}
|
||||||
|
this._valueIsBuffer = !convertToString && isBuffer;
|
||||||
|
- this._value = convertToString && isBuffer ? undefined : value;
|
||||||
|
+ this._value =
|
||||||
|
+ convertToString && isBuffer
|
||||||
|
+ ? undefined
|
||||||
|
+ : typeof value === "string"
|
||||||
|
+ ? internString(value)
|
||||||
|
+ : value;
|
||||||
|
this._valueAsBuffer = isBuffer ? value : undefined;
|
||||||
|
- this._valueAsString = isBuffer ? undefined : value;
|
||||||
|
+ this._valueAsString = isBuffer ? undefined : internString(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
isBuffer() {
|
||||||
|
@@ -27,14 +36,22 @@ class RawSource extends Source {
|
||||||
|
|
||||||
|
source() {
|
||||||
|
if (this._value === undefined) {
|
||||||
|
- this._value = this._valueAsBuffer.toString("utf-8");
|
||||||
|
+ const value = internString(this._valueAsBuffer.toString("utf-8"));
|
||||||
|
+ if (isDualStringBufferCachingEnabled()) {
|
||||||
|
+ this._value = value;
|
||||||
|
+ }
|
||||||
|
+ return value;
|
||||||
|
}
|
||||||
|
return this._value;
|
||||||
|
}
|
||||||
|
|
||||||
|
buffer() {
|
||||||
|
if (this._valueAsBuffer === undefined) {
|
||||||
|
- this._valueAsBuffer = Buffer.from(this._value, "utf-8");
|
||||||
|
+ const value = Buffer.from(this._value, "utf-8");
|
||||||
|
+ if (isDualStringBufferCachingEnabled()) {
|
||||||
|
+ this._valueAsBuffer = value;
|
||||||
|
+ }
|
||||||
|
+ return value;
|
||||||
|
}
|
||||||
|
return this._valueAsBuffer;
|
||||||
|
}
|
||||||
|
@@ -51,17 +68,21 @@ class RawSource extends Source {
|
||||||
|
* @returns {void}
|
||||||
|
*/
|
||||||
|
streamChunks(options, onChunk, onSource, onName) {
|
||||||
|
- if (this._value === undefined) {
|
||||||
|
+ if (this._value === undefined && isDualStringBufferCachingEnabled()) {
|
||||||
|
this._value = Buffer.from(this._valueAsBuffer, "utf-8");
|
||||||
|
}
|
||||||
|
- if (this._valueAsString === undefined) {
|
||||||
|
- this._valueAsString =
|
||||||
|
+ let strValue = this._valueAsString;
|
||||||
|
+ if (strValue === undefined) {
|
||||||
|
+ strValue =
|
||||||
|
typeof this._value === "string"
|
||||||
|
? this._value
|
||||||
|
- : this._value.toString("utf-8");
|
||||||
|
+ : internString(this._value.toString("utf-8"));
|
||||||
|
+ if (isDualStringBufferCachingEnabled()) {
|
||||||
|
+ this._valueAsString = strValue;
|
||||||
|
+ }
|
||||||
|
}
|
||||||
|
return streamChunksOfRawSource(
|
||||||
|
- this._valueAsString,
|
||||||
|
+ strValue,
|
||||||
|
onChunk,
|
||||||
|
onSource,
|
||||||
|
onName,
|
||||||
|
@@ -70,11 +91,8 @@ class RawSource extends Source {
|
||||||
|
}
|
||||||
|
|
||||||
|
updateHash(hash) {
|
||||||
|
- if (this._valueAsBuffer === undefined) {
|
||||||
|
- this._valueAsBuffer = Buffer.from(this._value, "utf-8");
|
||||||
|
- }
|
||||||
|
hash.update("RawSource");
|
||||||
|
- hash.update(this._valueAsBuffer);
|
||||||
|
+ hash.update(this.buffer());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
diff --git a/lib/helpers/stringBufferUtils.js b/lib/helpers/stringBufferUtils.js
|
||||||
|
new file mode 100644
|
||||||
|
index 0000000..3b210f1
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/lib/helpers/stringBufferUtils.js
|
||||||
|
@@ -0,0 +1,107 @@
|
||||||
|
+/*
|
||||||
|
+ MIT License http://www.opensource.org/licenses/mit-license.php
|
||||||
|
+ Author Mark Knichel @mknichel
|
||||||
|
+*/
|
||||||
|
+
|
||||||
|
+"use strict";
|
||||||
|
+
|
||||||
|
+let dualStringBufferCaching = true;
|
||||||
|
+
|
||||||
|
+/**
|
||||||
|
+ * @returns {boolean} Whether the optimization to cache copies of both the
|
||||||
|
+ * string and buffer version of source content is enabled. This is enabled by
|
||||||
|
+ * default to improve performance but can consume more memory since values are
|
||||||
|
+ * stored twice.
|
||||||
|
+ */
|
||||||
|
+function isDualStringBufferCachingEnabled() {
|
||||||
|
+ return dualStringBufferCaching;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+/**
|
||||||
|
+ * Enables an optimization to save both string and buffer in memory to avoid
|
||||||
|
+ * repeat conversions between the two formats when they are requested. This
|
||||||
|
+ * is enabled by default. This option can improve performance but can consume
|
||||||
|
+ * additional memory since values are stored twice.
|
||||||
|
+ *
|
||||||
|
+ * @returns {void}
|
||||||
|
+ */
|
||||||
|
+function enableDualStringBufferCaching() {
|
||||||
|
+ dualStringBufferCaching = true;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+/**
|
||||||
|
+ * Disables the optimization to save both string and buffer in memory. This
|
||||||
|
+ * may increase performance but should reduce memory usage in the Webpack
|
||||||
|
+ * compiler.
|
||||||
|
+ *
|
||||||
|
+ * @returns {void}
|
||||||
|
+ */
|
||||||
|
+function disableDualStringBufferCaching() {
|
||||||
|
+ dualStringBufferCaching = false;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+const interningStringMap = new Map();
|
||||||
|
+
|
||||||
|
+/**
|
||||||
|
+ * Saves the string in a map to ensure that only one copy of the string exists
|
||||||
|
+ * in memory at a given time. This is controlled by {@link enableStringInterning}
|
||||||
|
+ * and {@link disableStringInterning}. Callers are expect to manage the memory
|
||||||
|
+ * of the interned strings by calling {@link disableStringInterning} after the
|
||||||
|
+ * compiler no longer needs to save the interned memory.
|
||||||
|
+ *
|
||||||
|
+ * @param {string} str A string to be interned.
|
||||||
|
+ * @returns {string} The original string or a reference to an existing string
|
||||||
|
+ * of the same value if it has already been interned.
|
||||||
|
+ */
|
||||||
|
+function internString(str) {
|
||||||
|
+ if (!isStringInterningEnabled() || !str || typeof str !== "string") {
|
||||||
|
+ return str;
|
||||||
|
+ }
|
||||||
|
+ let internedString = interningStringMap.get(str);
|
||||||
|
+ if (internedString === undefined) {
|
||||||
|
+ internedString = str;
|
||||||
|
+ interningStringMap.set(str, internedString);
|
||||||
|
+ }
|
||||||
|
+ return internedString;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+let enableStringInterningRefCount = 0;
|
||||||
|
+
|
||||||
|
+function isStringInterningEnabled() {
|
||||||
|
+ return enableStringInterningRefCount > 0;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+/**
|
||||||
|
+ * Enables a memory optimization to avoid repeat copies of the same string in
|
||||||
|
+ * memory by caching a single reference to the string. This can reduce memory
|
||||||
|
+ * usage if the same string is repeated many times in the compiler, such as
|
||||||
|
+ * when Webpack layers are used with the same files.
|
||||||
|
+ *
|
||||||
|
+ * @returns {void}
|
||||||
|
+ */
|
||||||
|
+function enableStringInterning() {
|
||||||
|
+ enableStringInterningRefCount++;
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+/**
|
||||||
|
+ * Disables string interning. This should be called to free the memory used by
|
||||||
|
+ * the interned strings after the compiler no longer needs to reuse the
|
||||||
|
+ * interned strings such as at the end of the compilation.
|
||||||
|
+ *
|
||||||
|
+ * @returns {void}
|
||||||
|
+ */
|
||||||
|
+function disableStringInterning() {
|
||||||
|
+ if (--enableStringInterningRefCount <= 0) {
|
||||||
|
+ interningStringMap.clear();
|
||||||
|
+ enableStringInterningRefCount = 0;
|
||||||
|
+ }
|
||||||
|
+}
|
||||||
|
+
|
||||||
|
+module.exports = {
|
||||||
|
+ disableDualStringBufferCaching,
|
||||||
|
+ disableStringInterning,
|
||||||
|
+ enableDualStringBufferCaching,
|
||||||
|
+ enableStringInterning,
|
||||||
|
+ internString,
|
||||||
|
+ isDualStringBufferCachingEnabled
|
||||||
|
+};
|
||||||
|
diff --git a/lib/index.js b/lib/index.js
|
||||||
|
index 0c11c2f..86a7234 100644
|
||||||
|
--- a/lib/index.js
|
||||||
|
+++ b/lib/index.js
|
||||||
|
@@ -28,3 +28,4 @@ defineExport("ReplaceSource", () => require("./ReplaceSource"));
|
||||||
|
defineExport("PrefixSource", () => require("./PrefixSource"));
|
||||||
|
defineExport("SizeOnlySource", () => require("./SizeOnlySource"));
|
||||||
|
defineExport("CompatSource", () => require("./CompatSource"));
|
||||||
|
+defineExport("stringBufferUtils", () => require("./helpers/stringBufferUtils"));
|
|
@ -20,6 +20,11 @@ overrides:
|
||||||
react-is: 19.0.0-rc-81c5ff2e04-20240521
|
react-is: 19.0.0-rc-81c5ff2e04-20240521
|
||||||
scheduler: 0.25.0-rc-81c5ff2e04-20240521
|
scheduler: 0.25.0-rc-81c5ff2e04-20240521
|
||||||
|
|
||||||
|
patchedDependencies:
|
||||||
|
webpack-sources@3.2.3:
|
||||||
|
hash: exarmjd4pnde4auoeobjrmju54
|
||||||
|
path: patches/webpack-sources@3.2.3.patch
|
||||||
|
|
||||||
importers:
|
importers:
|
||||||
|
|
||||||
.:
|
.:
|
||||||
|
@ -1449,7 +1454,7 @@ importers:
|
||||||
version: /webpack-sources@1.4.3
|
version: /webpack-sources@1.4.3
|
||||||
webpack-sources3:
|
webpack-sources3:
|
||||||
specifier: npm:webpack-sources@3.2.3
|
specifier: npm:webpack-sources@3.2.3
|
||||||
version: /webpack-sources@3.2.3
|
version: /webpack-sources@3.2.3(patch_hash=exarmjd4pnde4auoeobjrmju54)
|
||||||
ws:
|
ws:
|
||||||
specifier: 8.2.3
|
specifier: 8.2.3
|
||||||
version: 8.2.3
|
version: 8.2.3
|
||||||
|
@ -25273,9 +25278,10 @@ packages:
|
||||||
source-map: 0.6.1
|
source-map: 0.6.1
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
/webpack-sources@3.2.3:
|
/webpack-sources@3.2.3(patch_hash=exarmjd4pnde4auoeobjrmju54):
|
||||||
resolution: {integrity: sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==}
|
resolution: {integrity: sha512-/DyMEOrDgLKKIG0fmvtz+4dUX/3Ghozwgm6iPp8KRhvn+eQf9+Q7GWxVNMk3+uCPWfdXYC4ExGBckIXdFEfH1w==}
|
||||||
engines: {node: '>=10.13.0'}
|
engines: {node: '>=10.13.0'}
|
||||||
|
patched: true
|
||||||
|
|
||||||
/webpack-stats-plugin@1.1.0:
|
/webpack-stats-plugin@1.1.0:
|
||||||
resolution: {integrity: sha512-D0meHk1WYryUbuCnWJuomJFAYvqs0rxv/JFu1XJT1YYpczdgnP1/vz+u/5Z31jrTxT6dJSxCg+TuKTgjhoZS6g==}
|
resolution: {integrity: sha512-D0meHk1WYryUbuCnWJuomJFAYvqs0rxv/JFu1XJT1YYpczdgnP1/vz+u/5Z31jrTxT6dJSxCg+TuKTgjhoZS6g==}
|
||||||
|
@ -25314,7 +25320,7 @@ packages:
|
||||||
tapable: 2.2.0
|
tapable: 2.2.0
|
||||||
terser-webpack-plugin: 5.3.10(@swc/core@1.5.7)(webpack@5.90.0)
|
terser-webpack-plugin: 5.3.10(@swc/core@1.5.7)(webpack@5.90.0)
|
||||||
watchpack: 2.4.0
|
watchpack: 2.4.0
|
||||||
webpack-sources: 3.2.3
|
webpack-sources: 3.2.3(patch_hash=exarmjd4pnde4auoeobjrmju54)
|
||||||
transitivePeerDependencies:
|
transitivePeerDependencies:
|
||||||
- '@swc/core'
|
- '@swc/core'
|
||||||
- esbuild
|
- esbuild
|
||||||
|
|
Loading…
Reference in a new issue