Benchmark against Vite (vercel/turbo#299)

This adds a comparison against Vite to our benchmark suite, running the startup, change, and restart benchmarks.

Test Plan: `cargo bench`


Co-authored-by: Tobias Koppers <1365881+sokra@users.noreply.github.com>

packages/next-swc/crates/next-dev/Cargo.toml

@@ -2,6 +2,7 @@
 name = "next-dev"
 version = "0.1.0"
 edition = "2021"
+autobenches = false
 
 [[bin]]
 name = "next-dev"

packages/next-swc/crates/next-dev/benches/bundlers.rs

@@ -0,0 +1,147 @@
+use std::{
+    env::{self, VarError},
+    fs::File,
+    io::{self, BufRead, BufReader, Write},
+    path::Path,
+    process::{Child, ChildStdout, Command, Stdio},
+};
+
+use regex::Regex;
+use tempfile::TempDir;
+
+pub trait Bundler {
+    fn get_name(&self) -> &str;
+    fn start_server(&self, test_dir: &Path) -> (Child, String);
+}
+
+struct Turbopack;
+impl Bundler for Turbopack {
+    fn get_name(&self) -> &str {
+        "Turbopack"
+    }
+
+    fn start_server(&self, test_dir: &Path) -> (Child, String) {
+        let mut proc = Command::new(std::env!("CARGO_BIN_EXE_next-dev"))
+            .args([test_dir.to_str().unwrap(), "--no-open", "--port", "0"])
+            .stdout(Stdio::piped())
+            .spawn()
+            .unwrap();
+
+        // Wait for the devserver address to appear in stdout.
+        let addr = wait_for_match(
+            proc.stdout.as_mut().unwrap(),
+            Regex::new("server listening on: (.*)").unwrap(),
+        );
+
+        (proc, addr)
+    }
+}
+
+struct Vite {
+    install_dir: TempDir,
+}
+
+impl Vite {
+    fn new() -> Self {
+        // Manage our own installation and avoid `npm exec`, `npx`, etc. to avoid their
+        // overhead influencing benchmarks.
+        let install_dir = tempfile::tempdir().unwrap();
+
+        let package_json = json::object! {
+            private: true,
+            version: "0.0.0",
+            dependencies: json::object! {
+                "vite": "3.0.9",
+            }
+        };
+
+        File::create(install_dir.path().join("package.json"))
+            .unwrap()
+            .write_all(package_json.pretty(2).as_bytes())
+            .unwrap();
+
+        let npm = command("npm")
+            .args(["install", "--prefer-offline"])
+            .current_dir(&install_dir)
+            .output()
+            .unwrap();
+
+        if !npm.status.success() {
+            io::stdout().write_all(&npm.stdout).unwrap();
+            io::stderr().write_all(&npm.stderr).unwrap();
+            panic!("npm install failed. See above.");
+        }
+
+        Vite { install_dir }
+    }
+}
+
+impl Bundler for Vite {
+    fn get_name(&self) -> &str {
+        "Vite"
+    }
+
+    fn start_server(&self, test_dir: &Path) -> (Child, String) {
+        let mut proc = Command::new("node")
+            .args([
+                &self
+                    .install_dir
+                    .path()
+                    .join("node_modules")
+                    .join("vite")
+                    .join("bin")
+                    .join("vite.js")
+                    .to_str()
+                    .unwrap(),
+                "--port",
+                "0",
+            ])
+            .env("NO_COLOR", "1")
+            .current_dir(test_dir.to_str().unwrap())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::inherit())
+            .spawn()
+            .unwrap();
+
+        // Wait for the devserver address to appear in stdout.
+        let addr = wait_for_match(
+            proc.stdout.as_mut().unwrap(),
+            Regex::new("Local:\\s+(.*)").unwrap(),
+        );
+
+        (proc, addr)
+    }
+}
+
+pub fn get_bundlers() -> Vec<Box<dyn Bundler>> {
+    if let Err(VarError::NotPresent) = env::var("TURBOPACK_BENCH_ALL") {
+        vec![Box::new(Turbopack {})]
+    } else {
+        vec![Box::new(Turbopack {}), Box::new(Vite::new())]
+    }
+}
+
+fn wait_for_match(stdout: &mut ChildStdout, re: Regex) -> String {
+    // See https://docs.rs/async-process/latest/async_process/#examples
+    let mut line_reader = BufReader::new(stdout).lines();
+    // Read until the match appears in the buffer
+    let mut matched: Option<String> = None;
+    while let Some(Ok(line)) = line_reader.next() {
+        if let Some(cap) = re.captures(&line) {
+            matched = Some(cap.get(1).unwrap().as_str().into());
+            break;
+        }
+    }
+
+    matched.unwrap()
+}
+
+pub fn command(bin: &str) -> Command {
+    if cfg!(windows) {
+        let mut command = Command::new("cmd.exe");
+        command.args(["/C", bin]);
+        command
+    } else {
+        Command::new(bin)
+    }
+}

packages/next-swc/crates/next-dev/benches/mod.rs

@@ -1,12 +1,13 @@
 use std::{
     fs::{self, remove_dir_all},
     future::Future,
-    io::{self, BufRead, BufReader, Write},
-    path::{Path, PathBuf},
-    process::{Child, ChildStdout, Command, Stdio},
+    io::{self, Write},
+    path::PathBuf,
+    process::Child,
     time::Duration,
 };
 
+use bundlers::{command, get_bundlers, Bundler};
 use chromiumoxide::{
     browser::{Browser, BrowserConfig},
     cdp::js_protocol::runtime::EventExceptionThrown,
@@ -20,13 +21,14 @@ use criterion::{
     AsyncBencher, BenchmarkId, Criterion,
 };
 use futures::{FutureExt, StreamExt};
-use regex::Regex;
 use tokio::runtime::Runtime;
 use tungstenite::{error::ProtocolError::ResetWithoutClosingHandshake, Error::Protocol};
 use turbopack_create_test_app::test_app_builder::TestAppBuilder;
 
 static MODULE_COUNTS: &[usize] = &[100, 1_000];
 
+mod bundlers;
+
 fn bench_startup(c: &mut Criterion) {
     let mut g = c.benchmark_group("bench_startup");
     g.sample_size(10);
@@ -67,7 +69,7 @@ fn bench_startup(c: &mut Criterion) {
 fn bench_simple_file_change(c: &mut Criterion) {
     let mut g = c.benchmark_group("bench_simple_file_change");
     g.sample_size(10);
-    g.measurement_time(Duration::from_secs(30));
+    g.measurement_time(Duration::from_secs(60));
 
     let runtime = Runtime::new().unwrap();
     let browser = &runtime.block_on(create_browser());
@@ -183,56 +185,6 @@ fn bench_restart(c: &mut Criterion) {
     }
 }
 
-trait Bundler {
-    fn get_name(&self) -> &str;
-    fn start_server(&self, test_dir: &Path) -> (Child, String);
-}
-
-struct Turbopack;
-impl Bundler for Turbopack {
-    fn get_name(&self) -> &str {
-        "Turbopack"
-    }
-
-    fn start_server(&self, test_dir: &Path) -> (Child, String) {
-        let mut proc = Command::new(
-            std::env::var("CARGO_BIN_EXE_next-dev")
-                .unwrap_or_else(|_| std::env!("CARGO_BIN_EXE_next-dev").to_string()),
-        )
-        .args([test_dir.to_str().unwrap(), "--no-open", "--port", "0"])
-        .stdout(Stdio::piped())
-        .spawn()
-        .unwrap();
-
-        // Wait for the devserver address to appear in stdout.
-        let addr = wait_for_match(
-            proc.stdout.as_mut().unwrap(),
-            Regex::new("server listening on: (.*)").unwrap(),
-        );
-
-        (proc, addr)
-    }
-}
-
-fn get_bundlers() -> Vec<Box<dyn Bundler>> {
-    vec![Box::new(Turbopack {})]
-}
-
-fn wait_for_match(stdout: &mut ChildStdout, re: Regex) -> String {
-    // See https://docs.rs/async-process/latest/async_process/#examples
-    let mut line_reader = BufReader::new(stdout).lines();
-    // Read until the match appears in the buffer
-    let mut matched: Option<String> = None;
-    while let Some(Ok(line)) = line_reader.next() {
-        if let Some(cap) = re.captures(&line) {
-            matched = Some(cap.get(1).unwrap().as_str().into());
-            break;
-        }
-    }
-
-    matched.unwrap()
-}
-
 struct PreparedApp<'a> {
     bundler: &'a dyn Bundler,
     pages: Vec<Page>,
@@ -297,20 +249,11 @@ impl<'a> PreparedApp<'a> {
     fn stop_server(&mut self) -> Child {
         let mut proc = self.server.take().expect("Server never started").0;
         proc.kill().unwrap();
+        proc.wait().unwrap();
         proc
     }
 }
 
-fn command(bin: &str) -> Command {
-    if cfg!(windows) {
-        let mut command = Command::new("cmd.exe");
-        command.args(["/C", bin]);
-        command
-    } else {
-        Command::new(bin)
-    }
-}
-
 fn build_test(module_count: usize) -> PathBuf {
     let test_dir = TestAppBuilder {
         module_count,