[Turbopack] improve memory measurement suite (#66748)
### What? * adds next-build-test to the workspace * use multiple turbo-tasks root tasks to be more realistic * add tracing support * run pages in order * add development mode with HMR * updates for RcStr ### Why? ### How?
This commit is contained in:
parent
0b1209edfa
commit
70df7cfb01
5 changed files with 214 additions and 80 deletions
23
Cargo.lock
generated
23
Cargo.lock
generated
|
@ -3000,6 +3000,25 @@ dependencies = [
|
|||
"vergen 7.5.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "next-build-test"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"futures-util",
|
||||
"next-api",
|
||||
"next-core",
|
||||
"num_cpus",
|
||||
"rand",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"turbo-tasks",
|
||||
"turbopack-binding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "next-core"
|
||||
version = "0.1.0"
|
||||
|
@ -6725,9 +6744,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "tokio-stream"
|
||||
version = "0.1.14"
|
||||
version = "0.1.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
|
||||
checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"pin-project-lite",
|
||||
|
|
|
@ -6,6 +6,7 @@ members = [
|
|||
"packages/next-swc/crates/napi",
|
||||
"packages/next-swc/crates/wasm",
|
||||
"packages/next-swc/crates/next-api",
|
||||
"packages/next-swc/crates/next-build-test",
|
||||
"packages/next-swc/crates/next-build",
|
||||
"packages/next-swc/crates/next-core",
|
||||
"packages/next-swc/crates/next-custom-transforms",
|
||||
|
|
|
@ -10,18 +10,17 @@ autobenches = false
|
|||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
next-core = { workspace = true, features = ["rustls-tls"] }
|
||||
next-api = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
anyhow = { workspace = true }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
turbo-tasks-malloc = { workspace = true, default-features = false }
|
||||
turbopack-binding = { workspace = true, features = [
|
||||
"__turbo_tasks",
|
||||
"__turbo_tasks_memory",
|
||||
"__turbo_tasks_env",
|
||||
"__turbo_tasks_fs",
|
||||
"__turbo_tasks_memory",
|
||||
"__turbo_tasks_malloc",
|
||||
"__turbopack",
|
||||
"__turbopack_nodejs",
|
||||
"__turbopack_core",
|
||||
|
@ -39,9 +38,12 @@ tracing = "0.1"
|
|||
tracing-subscriber = "0.3"
|
||||
futures-util = "0.3.30"
|
||||
|
||||
# Enable specific tls features per-target.
|
||||
[target.'cfg(all(target_os = "windows", target_arch = "aarch64"))'.dependencies]
|
||||
next-core = { workspace = true, features = ["native-tls"] }
|
||||
|
||||
[target.'cfg(not(any(all(target_os = "windows", target_arch = "aarch64"), target_arch="wasm32")))'.dependencies]
|
||||
next-core = { workspace = true, features = ["rustls-tls"] }
|
||||
|
||||
[build-dependencies]
|
||||
turbopack-binding = { workspace = true, features = ["__turbo_tasks_build"] }
|
||||
|
||||
[dev-dependencies]
|
||||
iai-callgrind = "0.10.2"
|
||||
tempdir = "0.3.7"
|
||||
|
|
|
@ -2,7 +2,7 @@
|
|||
#![feature(min_specialization)]
|
||||
#![feature(arbitrary_self_types)]
|
||||
|
||||
use std::str::FromStr;
|
||||
use std::{str::FromStr, time::Instant};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use futures_util::{StreamExt, TryStreamExt};
|
||||
|
@ -10,8 +10,11 @@ use next_api::{
|
|||
project::{ProjectContainer, ProjectOptions},
|
||||
route::{Endpoint, Route},
|
||||
};
|
||||
use turbo_tasks::{RcStr, TransientInstance, TurboTasks, Vc};
|
||||
use turbopack_binding::turbo::tasks_memory::MemoryBackend;
|
||||
|
||||
pub async fn main_inner(
|
||||
tt: &TurboTasks<MemoryBackend>,
|
||||
strat: Strategy,
|
||||
factor: usize,
|
||||
limit: usize,
|
||||
|
@ -19,18 +22,28 @@ pub async fn main_inner(
|
|||
) -> Result<()> {
|
||||
register();
|
||||
|
||||
let mut file = std::fs::File::open("project_options.json").with_context(|| {
|
||||
let path = std::env::current_dir()
|
||||
.unwrap()
|
||||
.join("project_options.json");
|
||||
format!("loading file at {}", path.display())
|
||||
})?;
|
||||
let path = std::env::current_dir()?.join("project_options.json");
|
||||
let mut file = std::fs::File::open(&path)
|
||||
.with_context(|| format!("loading file at {}", path.display()))?;
|
||||
|
||||
let options: ProjectOptions = serde_json::from_reader(&mut file).unwrap();
|
||||
let project = ProjectContainer::new(options);
|
||||
let mut options: ProjectOptions = serde_json::from_reader(&mut file)?;
|
||||
|
||||
if matches!(strat, Strategy::Development) {
|
||||
options.dev = true;
|
||||
options.watch = true;
|
||||
} else {
|
||||
options.dev = false;
|
||||
options.watch = false;
|
||||
}
|
||||
|
||||
let project = tt
|
||||
.run_once(async { Ok(ProjectContainer::new(options)) })
|
||||
.await?;
|
||||
|
||||
tracing::info!("collecting endpoints");
|
||||
let entrypoints = project.entrypoints().await?;
|
||||
let entrypoints = tt
|
||||
.run_once(async move { project.entrypoints().await })
|
||||
.await?;
|
||||
|
||||
let routes = if let Some(files) = files {
|
||||
tracing::info!("builing only the files:");
|
||||
|
@ -40,20 +53,24 @@ pub async fn main_inner(
|
|||
|
||||
// filter out the files that are not in the list
|
||||
// we expect this to be small so linear search OK
|
||||
Box::new(
|
||||
Box::new(files.into_iter().filter_map(|f| {
|
||||
entrypoints
|
||||
.routes
|
||||
.clone()
|
||||
.into_iter()
|
||||
.filter(move |(name, _)| files.iter().any(|f| f == name)),
|
||||
) as Box<dyn Iterator<Item = _> + Send + Sync>
|
||||
.iter()
|
||||
.find(|(name, _)| f.as_str() == name.as_str())
|
||||
.map(|(name, route)| (name.clone(), route.clone()))
|
||||
})) as Box<dyn Iterator<Item = _> + Send + Sync>
|
||||
} else {
|
||||
Box::new(shuffle(entrypoints.routes.clone().into_iter()))
|
||||
};
|
||||
|
||||
let count = render_routes(routes, strat, factor, limit).await;
|
||||
let count = render_routes(tt, routes, strat, factor, limit).await?;
|
||||
tracing::info!("rendered {} pages", count);
|
||||
|
||||
if matches!(strat, Strategy::Development) {
|
||||
hmr(tt, project).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
@ -67,6 +84,7 @@ pub enum Strategy {
|
|||
Sequential,
|
||||
Concurrent,
|
||||
Parallel,
|
||||
Development,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Strategy {
|
||||
|
@ -75,6 +93,7 @@ impl std::fmt::Display for Strategy {
|
|||
Strategy::Sequential => write!(f, "sequential"),
|
||||
Strategy::Concurrent => write!(f, "concurrent"),
|
||||
Strategy::Parallel => write!(f, "parallel"),
|
||||
Strategy::Development => write!(f, "development"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -87,6 +106,7 @@ impl FromStr for Strategy {
|
|||
"sequential" => Ok(Strategy::Sequential),
|
||||
"concurrent" => Ok(Strategy::Concurrent),
|
||||
"parallel" => Ok(Strategy::Parallel),
|
||||
"development" => Ok(Strategy::Development),
|
||||
_ => Err(anyhow::anyhow!("invalid strategy")),
|
||||
}
|
||||
}
|
||||
|
@ -101,11 +121,12 @@ pub fn shuffle<'a, T: 'a>(items: impl Iterator<Item = T>) -> impl Iterator<Item
|
|||
}
|
||||
|
||||
pub async fn render_routes(
|
||||
routes: impl Iterator<Item = (String, Route)>,
|
||||
tt: &TurboTasks<MemoryBackend>,
|
||||
routes: impl Iterator<Item = (RcStr, Route)>,
|
||||
strategy: Strategy,
|
||||
factor: usize,
|
||||
limit: usize,
|
||||
) -> usize {
|
||||
) -> Result<usize> {
|
||||
tracing::info!(
|
||||
"rendering routes with {} parallel and strat {}",
|
||||
factor,
|
||||
|
@ -113,51 +134,88 @@ pub async fn render_routes(
|
|||
);
|
||||
|
||||
let stream = tokio_stream::iter(routes)
|
||||
.map(move |(name, route)| {
|
||||
let fut = async move {
|
||||
tracing::info!("{name}");
|
||||
.map(move |(name, route)| async move {
|
||||
tracing::info!("{name}...");
|
||||
let start = Instant::now();
|
||||
|
||||
match route {
|
||||
Route::Page {
|
||||
html_endpoint,
|
||||
data_endpoint: _,
|
||||
} => {
|
||||
html_endpoint.write_to_disk().await?;
|
||||
}
|
||||
Route::PageApi { endpoint } => {
|
||||
endpoint.write_to_disk().await?;
|
||||
}
|
||||
Route::AppPage(routes) => {
|
||||
for route in routes {
|
||||
route.html_endpoint.write_to_disk().await?;
|
||||
tt.run_once({
|
||||
let name = name.clone();
|
||||
async move {
|
||||
match route {
|
||||
Route::Page {
|
||||
html_endpoint,
|
||||
data_endpoint: _,
|
||||
} => {
|
||||
html_endpoint.write_to_disk().await?;
|
||||
}
|
||||
Route::PageApi { endpoint } => {
|
||||
endpoint.write_to_disk().await?;
|
||||
}
|
||||
Route::AppPage(routes) => {
|
||||
for route in routes {
|
||||
route.html_endpoint.write_to_disk().await?;
|
||||
}
|
||||
}
|
||||
Route::AppRoute {
|
||||
original_name: _,
|
||||
endpoint,
|
||||
} => {
|
||||
endpoint.write_to_disk().await?;
|
||||
}
|
||||
Route::Conflict => {
|
||||
tracing::info!("WARN: conflict {}", name);
|
||||
}
|
||||
}
|
||||
Route::AppRoute {
|
||||
original_name: _,
|
||||
endpoint,
|
||||
} => {
|
||||
endpoint.write_to_disk().await?;
|
||||
}
|
||||
Route::Conflict => {
|
||||
tracing::info!("WARN: conflict {}", name);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
|
||||
Ok::<_, anyhow::Error>(())
|
||||
};
|
||||
tracing::info!("{name} {:?}", start.elapsed());
|
||||
|
||||
async move {
|
||||
match strategy {
|
||||
Strategy::Parallel => tokio::task::spawn(fut).await.unwrap(),
|
||||
_ => fut.await,
|
||||
}
|
||||
}
|
||||
Ok::<_, anyhow::Error>(())
|
||||
})
|
||||
.take(limit)
|
||||
.buffer_unordered(factor)
|
||||
.try_collect::<Vec<_>>()
|
||||
.await
|
||||
.unwrap();
|
||||
.await?;
|
||||
|
||||
stream.len()
|
||||
Ok(stream.len())
|
||||
}
|
||||
|
||||
async fn hmr(tt: &TurboTasks<MemoryBackend>, project: Vc<ProjectContainer>) -> Result<()> {
|
||||
tracing::info!("HMR...");
|
||||
let session = TransientInstance::new(());
|
||||
let idents = tt
|
||||
.run_once(async move { project.hmr_identifiers().await })
|
||||
.await?;
|
||||
let start = Instant::now();
|
||||
for ident in idents {
|
||||
if !ident.ends_with(".js") {
|
||||
continue;
|
||||
}
|
||||
let session = session.clone();
|
||||
let start = Instant::now();
|
||||
let task = tt.spawn_root_task(move || {
|
||||
let session = session.clone();
|
||||
async move {
|
||||
let project = project.project();
|
||||
project
|
||||
.hmr_update(
|
||||
ident.clone(),
|
||||
project.hmr_version_state(ident.clone(), session),
|
||||
)
|
||||
.await?;
|
||||
Ok(Vc::<()>::cell(()))
|
||||
}
|
||||
});
|
||||
tt.wait_task_completion(task, true).await?;
|
||||
let e = start.elapsed();
|
||||
if e.as_millis() > 10 {
|
||||
tracing::info!("HMR: {:?} {:?}", ident, e);
|
||||
}
|
||||
}
|
||||
tracing::info!("HMR {:?}", start.elapsed());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
@ -1,10 +1,22 @@
|
|||
use std::{convert::Infallible, str::FromStr};
|
||||
use std::{convert::Infallible, str::FromStr, time::Instant};
|
||||
|
||||
use next_api::project::{DefineEnv, ProjectOptions};
|
||||
use next_build_test::{main_inner, Strategy};
|
||||
use next_core::tracing_presets::{
|
||||
TRACING_NEXT_OVERVIEW_TARGETS, TRACING_NEXT_TARGETS, TRACING_NEXT_TURBOPACK_TARGETS,
|
||||
TRACING_NEXT_TURBO_TASKS_TARGETS,
|
||||
};
|
||||
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry};
|
||||
use turbo_tasks::TurboTasks;
|
||||
use turbo_tasks_malloc::TurboMalloc;
|
||||
use turbopack_binding::turbo::tasks_memory::MemoryBackend;
|
||||
use turbopack_binding::{
|
||||
turbo::{malloc::TurboMalloc, tasks_memory::MemoryBackend},
|
||||
turbopack::trace_utils::{
|
||||
exit::ExitGuard, raw_trace::RawTraceLayer, trace_writer::TraceWriter,
|
||||
},
|
||||
};
|
||||
|
||||
#[global_allocator]
|
||||
static ALLOC: TurboMalloc = TurboMalloc;
|
||||
|
||||
enum Cmd {
|
||||
Run,
|
||||
|
@ -23,8 +35,6 @@ impl FromStr for Cmd {
|
|||
}
|
||||
|
||||
fn main() {
|
||||
tracing_subscriber::fmt::init();
|
||||
|
||||
let cmd = std::env::args()
|
||||
.nth(1)
|
||||
.map(|s| Cmd::from_str(&s))
|
||||
|
@ -54,7 +64,7 @@ fn main() {
|
|||
.nth(5)
|
||||
.map(|f| f.split(',').map(ToOwned::to_owned).collect());
|
||||
|
||||
if strat == Strategy::Sequential {
|
||||
if matches!(strat, Strategy::Sequential | Strategy::Development) {
|
||||
factor = 1;
|
||||
}
|
||||
|
||||
|
@ -67,10 +77,54 @@ fn main() {
|
|||
.build()
|
||||
.unwrap()
|
||||
.block_on(async {
|
||||
let trace = std::env::var("NEXT_TURBOPACK_TRACING").ok();
|
||||
|
||||
let _guard = if let Some(mut trace) = trace {
|
||||
// Trace presets
|
||||
match trace.as_str() {
|
||||
"overview" | "1" => {
|
||||
trace = TRACING_NEXT_OVERVIEW_TARGETS.join(",");
|
||||
}
|
||||
"next" => {
|
||||
trace = TRACING_NEXT_TARGETS.join(",");
|
||||
}
|
||||
"turbopack" => {
|
||||
trace = TRACING_NEXT_TURBOPACK_TARGETS.join(",");
|
||||
}
|
||||
"turbo-tasks" => {
|
||||
trace = TRACING_NEXT_TURBO_TASKS_TARGETS.join(",");
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
let subscriber = Registry::default();
|
||||
|
||||
let subscriber =
|
||||
subscriber.with(EnvFilter::builder().parse(trace).unwrap());
|
||||
let trace_file = "trace.log";
|
||||
let trace_writer = std::fs::File::create(trace_file).unwrap();
|
||||
let (trace_writer, guard) = TraceWriter::new(trace_writer);
|
||||
let subscriber = subscriber.with(RawTraceLayer::new(trace_writer));
|
||||
|
||||
let guard = ExitGuard::new(guard).unwrap();
|
||||
|
||||
subscriber.init();
|
||||
|
||||
Some(guard)
|
||||
} else {
|
||||
tracing_subscriber::fmt::init();
|
||||
|
||||
None
|
||||
};
|
||||
|
||||
let tt = TurboTasks::new(MemoryBackend::new(usize::MAX));
|
||||
let x = tt.run_once(main_inner(strat, factor, limit, files)).await;
|
||||
tracing::debug!("done");
|
||||
x
|
||||
let result = main_inner(&tt, strat, factor, limit, files).await;
|
||||
let memory = TurboMalloc::memory_usage();
|
||||
tracing::info!("memory usage: {} MiB", memory / 1024 / 1024);
|
||||
let start = Instant::now();
|
||||
drop(tt);
|
||||
tracing::info!("drop {:?}", start.elapsed());
|
||||
result
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
@ -81,24 +135,24 @@ fn main() {
|
|||
let canonical_path = std::fs::canonicalize(absolute_dir).unwrap();
|
||||
|
||||
let options = ProjectOptions {
|
||||
build_id: "test".to_owned(),
|
||||
build_id: "test".into(),
|
||||
define_env: DefineEnv {
|
||||
client: vec![],
|
||||
edge: vec![],
|
||||
nodejs: vec![],
|
||||
},
|
||||
dev: true,
|
||||
encryption_key: "deadbeef".to_string(),
|
||||
encryption_key: "deadbeef".into(),
|
||||
env: vec![],
|
||||
js_config: include_str!("../jsConfig.json").to_string(),
|
||||
next_config: include_str!("../nextConfig.json").to_string(),
|
||||
js_config: include_str!("../jsConfig.json").into(),
|
||||
next_config: include_str!("../nextConfig.json").into(),
|
||||
preview_props: next_api::project::DraftModeOptions {
|
||||
preview_mode_encryption_key: "deadbeef".to_string(),
|
||||
preview_mode_id: "test".to_string(),
|
||||
preview_mode_signing_key: "deadbeef".to_string(),
|
||||
preview_mode_encryption_key: "deadbeef".into(),
|
||||
preview_mode_id: "test".into(),
|
||||
preview_mode_signing_key: "deadbeef".into(),
|
||||
},
|
||||
project_path: canonical_path.to_string_lossy().to_string(),
|
||||
root_path: "/".to_string(),
|
||||
project_path: canonical_path.to_string_lossy().into(),
|
||||
root_path: "/".into(),
|
||||
watch: false,
|
||||
};
|
||||
|
||||
|
|
Loading…
Reference in a new issue