feat(next-swc): add compile-time heap profiler flag (#47729)
### What? This PR adds an internal compile time flags for the `next-swc`, which exposes two runtime apis into next.js to initialize heap-profiling enabled memory allocator and teardown those once next.js exits. While there are newly added 2 js interfaces (`initHeapProfiler`, `teardownHeapProfiler`) underlying napi binary itself have compile time flags to actually enable those feature: any user who runs npm-published next.js cannot enable this features with any kind of runtime configuration. Only manually built next-swc binary with specific flag can enable this. Since this is primarily for the CI testing workflow only, those flag / configs are not visibily exposed as well. ### Why? It is for some experiments on the CI to see if it can observe some of memory pressure issues (WEB-593, WEB-804) while it is not easily reproducible on the local machines.
This commit is contained in:
parent
a52ebf4089
commit
a34f02b3d2
8 changed files with 165 additions and 6 deletions
43
packages/next-swc/Cargo.lock
generated
43
packages/next-swc/Cargo.lock
generated
|
@ -1418,6 +1418,22 @@ dependencies = [
|
|||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dhat"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4f2aaf837aaf456f6706cb46386ba8dffd4013a757e36f4ea05c20dd46b209a3"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"lazy_static",
|
||||
"mintex",
|
||||
"parking_lot",
|
||||
"rustc-hash",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thousands",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "diff"
|
||||
version = "0.1.13"
|
||||
|
@ -2903,6 +2919,16 @@ dependencies = [
|
|||
"adler",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mintex"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd7c5ba1c3b5a23418d7bbf98c71c3d4946a0125002129231da8d6b723d559cb"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"sys-info",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "0.6.23"
|
||||
|
@ -3208,6 +3234,7 @@ version = "0.0.0"
|
|||
dependencies = [
|
||||
"anyhow",
|
||||
"backtrace",
|
||||
"dhat",
|
||||
"fxhash",
|
||||
"napi",
|
||||
"napi-build",
|
||||
|
@ -6105,6 +6132,16 @@ version = "0.1.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
|
||||
|
||||
[[package]]
|
||||
name = "sys-info"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b3a0d0aba8bf96a0e1ddfdc352fc53b3df7f39318c71854910c3c4b024ae52c"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "target-lexicon"
|
||||
version = "0.12.6"
|
||||
|
@ -6216,6 +6253,12 @@ dependencies = [
|
|||
"syn 2.0.8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thousands"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3bf63baf9f5039dadc247375c29eb13706706cfde997d0330d05aa63a77d8820"
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.1.7"
|
||||
|
|
|
@ -126,3 +126,4 @@ tracing = "0.1.37"
|
|||
url = "2.2.2"
|
||||
urlencoding = "2.1.2"
|
||||
webbrowser = "0.8.7"
|
||||
dhat = { version = "0.3.2" }
|
||||
|
|
|
@ -26,10 +26,18 @@ __internal_nextjs_integration_test = [
|
|||
"next-dev/serializable"
|
||||
]
|
||||
|
||||
# Enable dhat profiling allocator for heap profiling.
|
||||
__internal_dhat-heap = ["dhat"]
|
||||
# Enable dhat profiling allocator for ad hoc profiling.
|
||||
# [Note]: we do not have any ad hoc event in the codebase yet, so enabling this
|
||||
# effectively does nothing.
|
||||
__internal_dhat-ad-hoc = ["dhat"]
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1.0.66"
|
||||
backtrace = "0.3"
|
||||
fxhash = "0.2.1"
|
||||
dhat = { workspace = true, optional = true }
|
||||
napi = { version = "2", default-features = false, features = [
|
||||
"napi3",
|
||||
"serde-json",
|
||||
|
|
|
@ -52,10 +52,18 @@ pub mod util;
|
|||
|
||||
// don't use turbo malloc (`mimalloc`) on linux-musl-aarch64 because of the
|
||||
// compile error
|
||||
#[cfg(not(all(target_os = "linux", target_env = "musl", target_arch = "aarch64")))]
|
||||
#[cfg(not(any(
|
||||
all(target_os = "linux", target_env = "musl", target_arch = "aarch64"),
|
||||
feature = "__internal_dhat-heap",
|
||||
feature = "__internal_dhat-ad-hoc"
|
||||
)))]
|
||||
#[global_allocator]
|
||||
static ALLOC: turbo_binding::turbo::malloc::TurboMalloc = turbo_binding::turbo::malloc::TurboMalloc;
|
||||
|
||||
#[cfg(feature = "__internal_dhat-heap")]
|
||||
#[global_allocator]
|
||||
static ALLOC: dhat::Alloc = dhat::Alloc;
|
||||
|
||||
static COMPILER: Lazy<Arc<Compiler>> = Lazy::new(|| {
|
||||
let cm = Arc::new(SourceMap::new(FilePathMapping::empty()));
|
||||
|
||||
|
|
|
@ -53,6 +53,45 @@ pub trait MapErr<T>: Into<Result<T, anyhow::Error>> {
|
|||
|
||||
impl<T> MapErr<T> for Result<T, anyhow::Error> {}
|
||||
|
||||
#[cfg(any(feature = "__internal_dhat-heap", feature = "__internal_dhat-ad-hoc"))]
|
||||
#[napi]
|
||||
pub fn init_heap_profiler() -> napi::Result<External<RefCell<Option<dhat::Profiler>>>> {
|
||||
#[cfg(feature = "__internal_dhat-heap")]
|
||||
{
|
||||
println!("[dhat-heap]: Initializing heap profiler");
|
||||
let _profiler = dhat::Profiler::new_heap();
|
||||
return Ok(External::new(RefCell::new(Some(_profiler))));
|
||||
}
|
||||
|
||||
#[cfg(feature = "__internal_dhat-ad-hoc")]
|
||||
{
|
||||
println!("[dhat-ad-hoc]: Initializing ad-hoc profiler");
|
||||
let _profiler = dhat::Profiler::new_ad_hoc();
|
||||
return Ok(External::new(RefCell::new(Some(_profiler))));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(feature = "__internal_dhat-heap", feature = "__internal_dhat-ad-hoc"))]
|
||||
#[napi]
|
||||
pub fn teardown_heap_profiler(guard_external: External<RefCell<Option<dhat::Profiler>>>) {
|
||||
let guard_cell = &*guard_external;
|
||||
|
||||
if let Some(guard) = guard_cell.take() {
|
||||
println!("[dhat]: Teardown profiler");
|
||||
drop(guard);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(feature = "__internal_dhat-heap", feature = "__internal_dhat-ad-hoc")))]
|
||||
#[napi]
|
||||
pub fn init_heap_profiler() -> napi::Result<External<RefCell<Option<u32>>>> {
|
||||
Ok(External::new(RefCell::new(Some(0))))
|
||||
}
|
||||
|
||||
#[cfg(not(any(feature = "__internal_dhat-heap", feature = "__internal_dhat-ad-hoc")))]
|
||||
#[napi]
|
||||
pub fn teardown_heap_profiler(_guard_external: External<RefCell<Option<u32>>>) {}
|
||||
|
||||
/// Initialize tracing subscriber to emit traces. This configures subscribers
|
||||
/// for Trace Event Format (https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview).
|
||||
#[napi]
|
||||
|
|
|
@ -119,6 +119,7 @@ import {
|
|||
teardownTraceSubscriber,
|
||||
teardownCrashReporter,
|
||||
loadBindings,
|
||||
teardownHeapProfiler,
|
||||
} from './swc'
|
||||
import { getNamedRouteRegex } from '../shared/lib/router/utils/route-regex'
|
||||
import { flatReaddir } from '../lib/flat-readdir'
|
||||
|
@ -3144,6 +3145,7 @@ export default async function build(
|
|||
// Ensure all traces are flushed before finishing the command
|
||||
await flushAllTraces()
|
||||
teardownTraceSubscriber()
|
||||
teardownHeapProfiler()
|
||||
teardownCrashReporter()
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,7 +1,11 @@
|
|||
import createStore from 'next/dist/compiled/unistore'
|
||||
import stripAnsi from 'next/dist/compiled/strip-ansi'
|
||||
import { flushAllTraces } from '../../trace'
|
||||
import { teardownCrashReporter, teardownTraceSubscriber } from '../swc'
|
||||
import {
|
||||
teardownCrashReporter,
|
||||
teardownHeapProfiler,
|
||||
teardownTraceSubscriber,
|
||||
} from '../swc'
|
||||
import * as Log from './log'
|
||||
|
||||
export type OutputState =
|
||||
|
@ -92,6 +96,7 @@ store.subscribe((state) => {
|
|||
// Ensure traces are flushed after each compile in development mode
|
||||
flushAllTraces()
|
||||
teardownTraceSubscriber()
|
||||
teardownHeapProfiler()
|
||||
teardownCrashReporter()
|
||||
return
|
||||
}
|
||||
|
@ -120,6 +125,7 @@ store.subscribe((state) => {
|
|||
// Ensure traces are flushed after each compile in development mode
|
||||
flushAllTraces()
|
||||
teardownTraceSubscriber()
|
||||
teardownHeapProfiler()
|
||||
teardownCrashReporter()
|
||||
return
|
||||
}
|
||||
|
@ -137,5 +143,6 @@ store.subscribe((state) => {
|
|||
// Ensure traces are flushed after each compile in development mode
|
||||
flushAllTraces()
|
||||
teardownTraceSubscriber()
|
||||
teardownHeapProfiler()
|
||||
teardownCrashReporter()
|
||||
})
|
||||
|
|
|
@ -76,6 +76,7 @@ let wasmBindings: any
|
|||
let downloadWasmPromise: any
|
||||
let pendingBindings: any
|
||||
let swcTraceFlushGuard: any
|
||||
let swcHeapProfilerFlushGuard: any
|
||||
let swcCrashReporterFlushGuard: any
|
||||
export const lockfilePatchPromise: { cur?: Promise<void> } = {}
|
||||
|
||||
|
@ -425,9 +426,13 @@ function loadNative(isCustomTurbopack = false) {
|
|||
getTargetTriple: bindings.getTargetTriple,
|
||||
initCustomTraceSubscriber: bindings.initCustomTraceSubscriber,
|
||||
teardownTraceSubscriber: bindings.teardownTraceSubscriber,
|
||||
initHeapProfiler: bindings.initHeapProfiler,
|
||||
teardownHeapProfiler: bindings.teardownHeapProfiler,
|
||||
teardownCrashReporter: bindings.teardownCrashReporter,
|
||||
turbo: {
|
||||
startDev: (options: any) => {
|
||||
initHeapProfiler()
|
||||
|
||||
const devOptions = {
|
||||
...options,
|
||||
noOpen: options.noOpen ?? true,
|
||||
|
@ -504,13 +509,19 @@ function loadNative(isCustomTurbopack = false) {
|
|||
}
|
||||
},
|
||||
nextBuild: (options: unknown) => {
|
||||
return bindings.nextBuild(options)
|
||||
initHeapProfiler()
|
||||
const ret = bindings.nextBuild(options)
|
||||
|
||||
return ret
|
||||
},
|
||||
startTrace: (options = {}, turboTasks: unknown) =>
|
||||
bindings.runTurboTracing(
|
||||
startTrace: (options = {}, turboTasks: unknown) => {
|
||||
initHeapProfiler()
|
||||
const ret = bindings.runTurboTracing(
|
||||
toBuffer({ exact: true, ...options }),
|
||||
turboTasks
|
||||
),
|
||||
)
|
||||
return ret
|
||||
},
|
||||
createTurboTasks: (memoryLimit?: number): unknown =>
|
||||
bindings.createTurboTasks(memoryLimit),
|
||||
},
|
||||
|
@ -589,6 +600,46 @@ export const initCustomTraceSubscriber = (traceFileName?: string): void => {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize heap profiler, if possible.
|
||||
* Note this is not available in release build of next-swc by default,
|
||||
* only available by manually building next-swc with specific flags.
|
||||
* Calling in release build will not do anything.
|
||||
*/
|
||||
export const initHeapProfiler = () => {
|
||||
try {
|
||||
if (!swcHeapProfilerFlushGuard) {
|
||||
let bindings = loadNative()
|
||||
swcHeapProfilerFlushGuard = bindings.initHeapProfiler()
|
||||
}
|
||||
} catch (_) {
|
||||
// Suppress exceptions, this fn allows to fail to load native bindings
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Teardown heap profiler, if possible.
|
||||
*
|
||||
* Same as initialization, this is not available in release build of next-swc by default
|
||||
* and calling it will not do anything.
|
||||
*/
|
||||
export const teardownHeapProfiler = (() => {
|
||||
let flushed = false
|
||||
return (): void => {
|
||||
if (!flushed) {
|
||||
flushed = true
|
||||
try {
|
||||
let bindings = loadNative()
|
||||
if (swcHeapProfilerFlushGuard) {
|
||||
bindings.teardownHeapProfiler(swcHeapProfilerFlushGuard)
|
||||
}
|
||||
} catch (e) {
|
||||
// Suppress exceptions, this fn allows to fail to load native bindings
|
||||
}
|
||||
}
|
||||
}
|
||||
})()
|
||||
|
||||
/**
|
||||
* Teardown swc's trace subscriber if there's an initialized flush guard exists.
|
||||
*
|
||||
|
|
Loading…
Reference in a new issue