[Turbopack] improve memory measurement suite (#66748)

### What?

* adds next-build-test to the workspace
* use multiple turbo-tasks root tasks to be more realistic
* add tracing support
* run pages in order
* add development mode with HMR
* updates for RcStr

### Why?

### How?

Cargo.lock

@@ -3000,6 +3000,25 @@ dependencies = [
  "vergen 7.5.1",
 ]
 
+[[package]]
+name = "next-build-test"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "futures-util",
+ "next-api",
+ "next-core",
+ "num_cpus",
+ "rand",
+ "serde_json",
+ "tokio",
+ "tokio-stream",
+ "tracing",
+ "tracing-subscriber",
+ "turbo-tasks",
+ "turbopack-binding",
+]
+
 [[package]]
 name = "next-core"
 version = "0.1.0"
@@ -6725,9 +6744,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.14"
+version = "0.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842"
+checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af"
 dependencies = [
  "futures-core",
  "pin-project-lite",

Cargo.toml

@@ -6,6 +6,7 @@ members = [
   "packages/next-swc/crates/napi",
   "packages/next-swc/crates/wasm",
   "packages/next-swc/crates/next-api",
+  "packages/next-swc/crates/next-build-test",
   "packages/next-swc/crates/next-build",
   "packages/next-swc/crates/next-core",
   "packages/next-swc/crates/next-custom-transforms",

packages/next-swc/crates/next-build-test/Cargo.toml

@@ -10,18 +10,17 @@ autobenches = false
 workspace = true
 
 [dependencies]
-next-core = { workspace = true, features = ["rustls-tls"] }
 next-api = { workspace = true }
 serde_json = { workspace = true }
 anyhow = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
-turbo-tasks-malloc = { workspace = true, default-features = false }
 turbopack-binding = { workspace = true, features = [
     "__turbo_tasks",
     "__turbo_tasks_memory",
     "__turbo_tasks_env",
     "__turbo_tasks_fs",
     "__turbo_tasks_memory",
+    "__turbo_tasks_malloc",
     "__turbopack",
     "__turbopack_nodejs",
     "__turbopack_core",
@@ -39,9 +38,12 @@ tracing = "0.1"
 tracing-subscriber = "0.3"
 futures-util = "0.3.30"
 
+# Enable specific tls features per-target.
+[target.'cfg(all(target_os = "windows", target_arch = "aarch64"))'.dependencies]
+next-core = { workspace = true, features = ["native-tls"] }
+
+[target.'cfg(not(any(all(target_os = "windows", target_arch = "aarch64"), target_arch="wasm32")))'.dependencies]
+next-core = { workspace = true, features = ["rustls-tls"] }
+
 [build-dependencies]
 turbopack-binding = { workspace = true, features = ["__turbo_tasks_build"] }
-
-[dev-dependencies]
-iai-callgrind = "0.10.2"
-tempdir = "0.3.7"

packages/next-swc/crates/next-build-test/src/lib.rs

@@ -2,7 +2,7 @@
 #![feature(min_specialization)]
 #![feature(arbitrary_self_types)]
 
-use std::str::FromStr;
+use std::{str::FromStr, time::Instant};
 
 use anyhow::{Context, Result};
 use futures_util::{StreamExt, TryStreamExt};
@@ -10,8 +10,11 @@ use next_api::{
     project::{ProjectContainer, ProjectOptions},
     route::{Endpoint, Route},
 };
+use turbo_tasks::{RcStr, TransientInstance, TurboTasks, Vc};
+use turbopack_binding::turbo::tasks_memory::MemoryBackend;
 
 pub async fn main_inner(
+    tt: &TurboTasks<MemoryBackend>,
     strat: Strategy,
     factor: usize,
     limit: usize,
@@ -19,18 +22,28 @@ pub async fn main_inner(
 ) -> Result<()> {
     register();
 
-    let mut file = std::fs::File::open("project_options.json").with_context(|| {
-        let path = std::env::current_dir()
-            .unwrap()
-            .join("project_options.json");
-        format!("loading file at {}", path.display())
-    })?;
+    let path = std::env::current_dir()?.join("project_options.json");
+    let mut file = std::fs::File::open(&path)
+        .with_context(|| format!("loading file at {}", path.display()))?;
 
-    let options: ProjectOptions = serde_json::from_reader(&mut file).unwrap();
-    let project = ProjectContainer::new(options);
+    let mut options: ProjectOptions = serde_json::from_reader(&mut file)?;
+
+    if matches!(strat, Strategy::Development) {
+        options.dev = true;
+        options.watch = true;
+    } else {
+        options.dev = false;
+        options.watch = false;
+    }
+
+    let project = tt
+        .run_once(async { Ok(ProjectContainer::new(options)) })
+        .await?;
 
     tracing::info!("collecting endpoints");
-    let entrypoints = project.entrypoints().await?;
+    let entrypoints = tt
+        .run_once(async move { project.entrypoints().await })
+        .await?;
 
     let routes = if let Some(files) = files {
         tracing::info!("builing only the files:");
@@ -40,20 +53,24 @@ pub async fn main_inner(
 
         // filter out the files that are not in the list
         // we expect this to be small so linear search OK
-        Box::new(
+        Box::new(files.into_iter().filter_map(|f| {
             entrypoints
                 .routes
-                .clone()
-                .into_iter()
-                .filter(move |(name, _)| files.iter().any(|f| f == name)),
-        ) as Box<dyn Iterator<Item = _> + Send + Sync>
+                .iter()
+                .find(|(name, _)| f.as_str() == name.as_str())
+                .map(|(name, route)| (name.clone(), route.clone()))
+        })) as Box<dyn Iterator<Item = _> + Send + Sync>
     } else {
         Box::new(shuffle(entrypoints.routes.clone().into_iter()))
     };
 
-    let count = render_routes(routes, strat, factor, limit).await;
+    let count = render_routes(tt, routes, strat, factor, limit).await?;
     tracing::info!("rendered {} pages", count);
 
+    if matches!(strat, Strategy::Development) {
+        hmr(tt, project).await?;
+    }
+
     Ok(())
 }
 
@@ -67,6 +84,7 @@ pub enum Strategy {
     Sequential,
     Concurrent,
     Parallel,
+    Development,
 }
 
 impl std::fmt::Display for Strategy {
@@ -75,6 +93,7 @@ impl std::fmt::Display for Strategy {
             Strategy::Sequential => write!(f, "sequential"),
             Strategy::Concurrent => write!(f, "concurrent"),
             Strategy::Parallel => write!(f, "parallel"),
+            Strategy::Development => write!(f, "development"),
         }
     }
 }
@@ -87,6 +106,7 @@ impl FromStr for Strategy {
             "sequential" => Ok(Strategy::Sequential),
             "concurrent" => Ok(Strategy::Concurrent),
             "parallel" => Ok(Strategy::Parallel),
+            "development" => Ok(Strategy::Development),
             _ => Err(anyhow::anyhow!("invalid strategy")),
         }
     }
@@ -101,11 +121,12 @@ pub fn shuffle<'a, T: 'a>(items: impl Iterator<Item = T>) -> impl Iterator<Item
 }
 
 pub async fn render_routes(
-    routes: impl Iterator<Item = (String, Route)>,
+    tt: &TurboTasks<MemoryBackend>,
+    routes: impl Iterator<Item = (RcStr, Route)>,
     strategy: Strategy,
     factor: usize,
     limit: usize,
-) -> usize {
+) -> Result<usize> {
     tracing::info!(
         "rendering routes with {} parallel and strat {}",
         factor,
@@ -113,10 +134,13 @@ pub async fn render_routes(
     );
 
     let stream = tokio_stream::iter(routes)
-        .map(move |(name, route)| {
-            let fut = async move {
-                tracing::info!("{name}");
+        .map(move |(name, route)| async move {
+            tracing::info!("{name}...");
+            let start = Instant::now();
 
+            tt.run_once({
+                let name = name.clone();
+                async move {
                     match route {
                         Route::Page {
                             html_endpoint,
@@ -142,22 +166,56 @@ pub async fn render_routes(
                             tracing::info!("WARN: conflict {}", name);
                         }
                     }
+                    Ok(())
+                }
+            })
+            .await?;
+
+            tracing::info!("{name} {:?}", start.elapsed());
 
             Ok::<_, anyhow::Error>(())
-            };
-
-            async move {
-                match strategy {
-                    Strategy::Parallel => tokio::task::spawn(fut).await.unwrap(),
-                    _ => fut.await,
-                }
-            }
         })
         .take(limit)
         .buffer_unordered(factor)
         .try_collect::<Vec<_>>()
-        .await
-        .unwrap();
+        .await?;
 
-    stream.len()
+    Ok(stream.len())
+}
+
+async fn hmr(tt: &TurboTasks<MemoryBackend>, project: Vc<ProjectContainer>) -> Result<()> {
+    tracing::info!("HMR...");
+    let session = TransientInstance::new(());
+    let idents = tt
+        .run_once(async move { project.hmr_identifiers().await })
+        .await?;
+    let start = Instant::now();
+    for ident in idents {
+        if !ident.ends_with(".js") {
+            continue;
+        }
+        let session = session.clone();
+        let start = Instant::now();
+        let task = tt.spawn_root_task(move || {
+            let session = session.clone();
+            async move {
+                let project = project.project();
+                project
+                    .hmr_update(
+                        ident.clone(),
+                        project.hmr_version_state(ident.clone(), session),
+                    )
+                    .await?;
+                Ok(Vc::<()>::cell(()))
+            }
+        });
+        tt.wait_task_completion(task, true).await?;
+        let e = start.elapsed();
+        if e.as_millis() > 10 {
+            tracing::info!("HMR: {:?} {:?}", ident, e);
+        }
+    }
+    tracing::info!("HMR {:?}", start.elapsed());
+
+    Ok(())
 }

packages/next-swc/crates/next-build-test/src/main.rs

@@ -1,10 +1,22 @@
-use std::{convert::Infallible, str::FromStr};
+use std::{convert::Infallible, str::FromStr, time::Instant};
 
 use next_api::project::{DefineEnv, ProjectOptions};
 use next_build_test::{main_inner, Strategy};
+use next_core::tracing_presets::{
+    TRACING_NEXT_OVERVIEW_TARGETS, TRACING_NEXT_TARGETS, TRACING_NEXT_TURBOPACK_TARGETS,
+    TRACING_NEXT_TURBO_TASKS_TARGETS,
+};
+use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Registry};
 use turbo_tasks::TurboTasks;
-use turbo_tasks_malloc::TurboMalloc;
-use turbopack_binding::turbo::tasks_memory::MemoryBackend;
+use turbopack_binding::{
+    turbo::{malloc::TurboMalloc, tasks_memory::MemoryBackend},
+    turbopack::trace_utils::{
+        exit::ExitGuard, raw_trace::RawTraceLayer, trace_writer::TraceWriter,
+    },
+};
+
+#[global_allocator]
+static ALLOC: TurboMalloc = TurboMalloc;
 
 enum Cmd {
     Run,
@@ -23,8 +35,6 @@ impl FromStr for Cmd {
 }
 
 fn main() {
-    tracing_subscriber::fmt::init();
-
     let cmd = std::env::args()
         .nth(1)
         .map(|s| Cmd::from_str(&s))
@@ -54,7 +64,7 @@ fn main() {
                 .nth(5)
                 .map(|f| f.split(',').map(ToOwned::to_owned).collect());
 
-            if strat == Strategy::Sequential {
+            if matches!(strat, Strategy::Sequential | Strategy::Development) {
                 factor = 1;
             }
 
@@ -67,10 +77,54 @@ fn main() {
                 .build()
                 .unwrap()
                 .block_on(async {
+                    let trace = std::env::var("NEXT_TURBOPACK_TRACING").ok();
+
+                    let _guard = if let Some(mut trace) = trace {
+                        // Trace presets
+                        match trace.as_str() {
+                            "overview" | "1" => {
+                                trace = TRACING_NEXT_OVERVIEW_TARGETS.join(",");
+                            }
+                            "next" => {
+                                trace = TRACING_NEXT_TARGETS.join(",");
+                            }
+                            "turbopack" => {
+                                trace = TRACING_NEXT_TURBOPACK_TARGETS.join(",");
+                            }
+                            "turbo-tasks" => {
+                                trace = TRACING_NEXT_TURBO_TASKS_TARGETS.join(",");
+                            }
+                            _ => {}
+                        }
+
+                        let subscriber = Registry::default();
+
+                        let subscriber =
+                            subscriber.with(EnvFilter::builder().parse(trace).unwrap());
+                        let trace_file = "trace.log";
+                        let trace_writer = std::fs::File::create(trace_file).unwrap();
+                        let (trace_writer, guard) = TraceWriter::new(trace_writer);
+                        let subscriber = subscriber.with(RawTraceLayer::new(trace_writer));
+
+                        let guard = ExitGuard::new(guard).unwrap();
+
+                        subscriber.init();
+
+                        Some(guard)
+                    } else {
+                        tracing_subscriber::fmt::init();
+
+                        None
+                    };
+
                     let tt = TurboTasks::new(MemoryBackend::new(usize::MAX));
-                    let x = tt.run_once(main_inner(strat, factor, limit, files)).await;
-                    tracing::debug!("done");
-                    x
+                    let result = main_inner(&tt, strat, factor, limit, files).await;
+                    let memory = TurboMalloc::memory_usage();
+                    tracing::info!("memory usage: {} MiB", memory / 1024 / 1024);
+                    let start = Instant::now();
+                    drop(tt);
+                    tracing::info!("drop {:?}", start.elapsed());
+                    result
                 })
                 .unwrap();
         }
@@ -81,24 +135,24 @@ fn main() {
             let canonical_path = std::fs::canonicalize(absolute_dir).unwrap();
 
             let options = ProjectOptions {
-                build_id: "test".to_owned(),
+                build_id: "test".into(),
                 define_env: DefineEnv {
                     client: vec![],
                     edge: vec![],
                     nodejs: vec![],
                 },
                 dev: true,
-                encryption_key: "deadbeef".to_string(),
+                encryption_key: "deadbeef".into(),
                 env: vec![],
-                js_config: include_str!("../jsConfig.json").to_string(),
-                next_config: include_str!("../nextConfig.json").to_string(),
+                js_config: include_str!("../jsConfig.json").into(),
+                next_config: include_str!("../nextConfig.json").into(),
                 preview_props: next_api::project::DraftModeOptions {
-                    preview_mode_encryption_key: "deadbeef".to_string(),
-                    preview_mode_id: "test".to_string(),
-                    preview_mode_signing_key: "deadbeef".to_string(),
+                    preview_mode_encryption_key: "deadbeef".into(),
+                    preview_mode_id: "test".into(),
+                    preview_mode_signing_key: "deadbeef".into(),
                 },
-                project_path: canonical_path.to_string_lossy().to_string(),
-                root_path: "/".to_string(),
+                project_path: canonical_path.to_string_lossy().into(),
+                root_path: "/".into(),
                 watch: false,
             };