Benchmark against Vite (vercel/turbo#299)
This adds a comparison against Vite to our benchmark suite, running the startup, change, and restart benchmarks. Test Plan: `cargo bench` Co-authored-by: Tobias Koppers <1365881+sokra@users.noreply.github.com>
This commit is contained in:
parent
b78ede6099
commit
12e5f19a84
3 changed files with 156 additions and 65 deletions
|
@ -2,6 +2,7 @@
|
||||||
name = "next-dev"
|
name = "next-dev"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
edition = "2021"
|
edition = "2021"
|
||||||
|
autobenches = false
|
||||||
|
|
||||||
[[bin]]
|
[[bin]]
|
||||||
name = "next-dev"
|
name = "next-dev"
|
||||||
|
|
147
packages/next-swc/crates/next-dev/benches/bundlers.rs
Normal file
147
packages/next-swc/crates/next-dev/benches/bundlers.rs
Normal file
|
@ -0,0 +1,147 @@
|
||||||
|
use std::{
|
||||||
|
env::{self, VarError},
|
||||||
|
fs::File,
|
||||||
|
io::{self, BufRead, BufReader, Write},
|
||||||
|
path::Path,
|
||||||
|
process::{Child, ChildStdout, Command, Stdio},
|
||||||
|
};
|
||||||
|
|
||||||
|
use regex::Regex;
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
|
pub trait Bundler {
|
||||||
|
fn get_name(&self) -> &str;
|
||||||
|
fn start_server(&self, test_dir: &Path) -> (Child, String);
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Turbopack;
|
||||||
|
impl Bundler for Turbopack {
|
||||||
|
fn get_name(&self) -> &str {
|
||||||
|
"Turbopack"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn start_server(&self, test_dir: &Path) -> (Child, String) {
|
||||||
|
let mut proc = Command::new(std::env!("CARGO_BIN_EXE_next-dev"))
|
||||||
|
.args([test_dir.to_str().unwrap(), "--no-open", "--port", "0"])
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.spawn()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Wait for the devserver address to appear in stdout.
|
||||||
|
let addr = wait_for_match(
|
||||||
|
proc.stdout.as_mut().unwrap(),
|
||||||
|
Regex::new("server listening on: (.*)").unwrap(),
|
||||||
|
);
|
||||||
|
|
||||||
|
(proc, addr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct Vite {
|
||||||
|
install_dir: TempDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Vite {
|
||||||
|
fn new() -> Self {
|
||||||
|
// Manage our own installation and avoid `npm exec`, `npx`, etc. to avoid their
|
||||||
|
// overhead influencing benchmarks.
|
||||||
|
let install_dir = tempfile::tempdir().unwrap();
|
||||||
|
|
||||||
|
let package_json = json::object! {
|
||||||
|
private: true,
|
||||||
|
version: "0.0.0",
|
||||||
|
dependencies: json::object! {
|
||||||
|
"vite": "3.0.9",
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
File::create(install_dir.path().join("package.json"))
|
||||||
|
.unwrap()
|
||||||
|
.write_all(package_json.pretty(2).as_bytes())
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let npm = command("npm")
|
||||||
|
.args(["install", "--prefer-offline"])
|
||||||
|
.current_dir(&install_dir)
|
||||||
|
.output()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
if !npm.status.success() {
|
||||||
|
io::stdout().write_all(&npm.stdout).unwrap();
|
||||||
|
io::stderr().write_all(&npm.stderr).unwrap();
|
||||||
|
panic!("npm install failed. See above.");
|
||||||
|
}
|
||||||
|
|
||||||
|
Vite { install_dir }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Bundler for Vite {
|
||||||
|
fn get_name(&self) -> &str {
|
||||||
|
"Vite"
|
||||||
|
}
|
||||||
|
|
||||||
|
fn start_server(&self, test_dir: &Path) -> (Child, String) {
|
||||||
|
let mut proc = Command::new("node")
|
||||||
|
.args([
|
||||||
|
&self
|
||||||
|
.install_dir
|
||||||
|
.path()
|
||||||
|
.join("node_modules")
|
||||||
|
.join("vite")
|
||||||
|
.join("bin")
|
||||||
|
.join("vite.js")
|
||||||
|
.to_str()
|
||||||
|
.unwrap(),
|
||||||
|
"--port",
|
||||||
|
"0",
|
||||||
|
])
|
||||||
|
.env("NO_COLOR", "1")
|
||||||
|
.current_dir(test_dir.to_str().unwrap())
|
||||||
|
.stdout(Stdio::piped())
|
||||||
|
.stderr(Stdio::inherit())
|
||||||
|
.spawn()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
// Wait for the devserver address to appear in stdout.
|
||||||
|
let addr = wait_for_match(
|
||||||
|
proc.stdout.as_mut().unwrap(),
|
||||||
|
Regex::new("Local:\\s+(.*)").unwrap(),
|
||||||
|
);
|
||||||
|
|
||||||
|
(proc, addr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn get_bundlers() -> Vec<Box<dyn Bundler>> {
|
||||||
|
if let Err(VarError::NotPresent) = env::var("TURBOPACK_BENCH_ALL") {
|
||||||
|
vec![Box::new(Turbopack {})]
|
||||||
|
} else {
|
||||||
|
vec![Box::new(Turbopack {}), Box::new(Vite::new())]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn wait_for_match(stdout: &mut ChildStdout, re: Regex) -> String {
|
||||||
|
// See https://docs.rs/async-process/latest/async_process/#examples
|
||||||
|
let mut line_reader = BufReader::new(stdout).lines();
|
||||||
|
// Read until the match appears in the buffer
|
||||||
|
let mut matched: Option<String> = None;
|
||||||
|
while let Some(Ok(line)) = line_reader.next() {
|
||||||
|
if let Some(cap) = re.captures(&line) {
|
||||||
|
matched = Some(cap.get(1).unwrap().as_str().into());
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
matched.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn command(bin: &str) -> Command {
|
||||||
|
if cfg!(windows) {
|
||||||
|
let mut command = Command::new("cmd.exe");
|
||||||
|
command.args(["/C", bin]);
|
||||||
|
command
|
||||||
|
} else {
|
||||||
|
Command::new(bin)
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,12 +1,13 @@
|
||||||
use std::{
|
use std::{
|
||||||
fs::{self, remove_dir_all},
|
fs::{self, remove_dir_all},
|
||||||
future::Future,
|
future::Future,
|
||||||
io::{self, BufRead, BufReader, Write},
|
io::{self, Write},
|
||||||
path::{Path, PathBuf},
|
path::PathBuf,
|
||||||
process::{Child, ChildStdout, Command, Stdio},
|
process::Child,
|
||||||
time::Duration,
|
time::Duration,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
use bundlers::{command, get_bundlers, Bundler};
|
||||||
use chromiumoxide::{
|
use chromiumoxide::{
|
||||||
browser::{Browser, BrowserConfig},
|
browser::{Browser, BrowserConfig},
|
||||||
cdp::js_protocol::runtime::EventExceptionThrown,
|
cdp::js_protocol::runtime::EventExceptionThrown,
|
||||||
|
@ -20,13 +21,14 @@ use criterion::{
|
||||||
AsyncBencher, BenchmarkId, Criterion,
|
AsyncBencher, BenchmarkId, Criterion,
|
||||||
};
|
};
|
||||||
use futures::{FutureExt, StreamExt};
|
use futures::{FutureExt, StreamExt};
|
||||||
use regex::Regex;
|
|
||||||
use tokio::runtime::Runtime;
|
use tokio::runtime::Runtime;
|
||||||
use tungstenite::{error::ProtocolError::ResetWithoutClosingHandshake, Error::Protocol};
|
use tungstenite::{error::ProtocolError::ResetWithoutClosingHandshake, Error::Protocol};
|
||||||
use turbopack_create_test_app::test_app_builder::TestAppBuilder;
|
use turbopack_create_test_app::test_app_builder::TestAppBuilder;
|
||||||
|
|
||||||
static MODULE_COUNTS: &[usize] = &[100, 1_000];
|
static MODULE_COUNTS: &[usize] = &[100, 1_000];
|
||||||
|
|
||||||
|
mod bundlers;
|
||||||
|
|
||||||
fn bench_startup(c: &mut Criterion) {
|
fn bench_startup(c: &mut Criterion) {
|
||||||
let mut g = c.benchmark_group("bench_startup");
|
let mut g = c.benchmark_group("bench_startup");
|
||||||
g.sample_size(10);
|
g.sample_size(10);
|
||||||
|
@ -67,7 +69,7 @@ fn bench_startup(c: &mut Criterion) {
|
||||||
fn bench_simple_file_change(c: &mut Criterion) {
|
fn bench_simple_file_change(c: &mut Criterion) {
|
||||||
let mut g = c.benchmark_group("bench_simple_file_change");
|
let mut g = c.benchmark_group("bench_simple_file_change");
|
||||||
g.sample_size(10);
|
g.sample_size(10);
|
||||||
g.measurement_time(Duration::from_secs(30));
|
g.measurement_time(Duration::from_secs(60));
|
||||||
|
|
||||||
let runtime = Runtime::new().unwrap();
|
let runtime = Runtime::new().unwrap();
|
||||||
let browser = &runtime.block_on(create_browser());
|
let browser = &runtime.block_on(create_browser());
|
||||||
|
@ -183,56 +185,6 @@ fn bench_restart(c: &mut Criterion) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
trait Bundler {
|
|
||||||
fn get_name(&self) -> &str;
|
|
||||||
fn start_server(&self, test_dir: &Path) -> (Child, String);
|
|
||||||
}
|
|
||||||
|
|
||||||
struct Turbopack;
|
|
||||||
impl Bundler for Turbopack {
|
|
||||||
fn get_name(&self) -> &str {
|
|
||||||
"Turbopack"
|
|
||||||
}
|
|
||||||
|
|
||||||
fn start_server(&self, test_dir: &Path) -> (Child, String) {
|
|
||||||
let mut proc = Command::new(
|
|
||||||
std::env::var("CARGO_BIN_EXE_next-dev")
|
|
||||||
.unwrap_or_else(|_| std::env!("CARGO_BIN_EXE_next-dev").to_string()),
|
|
||||||
)
|
|
||||||
.args([test_dir.to_str().unwrap(), "--no-open", "--port", "0"])
|
|
||||||
.stdout(Stdio::piped())
|
|
||||||
.spawn()
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
// Wait for the devserver address to appear in stdout.
|
|
||||||
let addr = wait_for_match(
|
|
||||||
proc.stdout.as_mut().unwrap(),
|
|
||||||
Regex::new("server listening on: (.*)").unwrap(),
|
|
||||||
);
|
|
||||||
|
|
||||||
(proc, addr)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn get_bundlers() -> Vec<Box<dyn Bundler>> {
|
|
||||||
vec![Box::new(Turbopack {})]
|
|
||||||
}
|
|
||||||
|
|
||||||
fn wait_for_match(stdout: &mut ChildStdout, re: Regex) -> String {
|
|
||||||
// See https://docs.rs/async-process/latest/async_process/#examples
|
|
||||||
let mut line_reader = BufReader::new(stdout).lines();
|
|
||||||
// Read until the match appears in the buffer
|
|
||||||
let mut matched: Option<String> = None;
|
|
||||||
while let Some(Ok(line)) = line_reader.next() {
|
|
||||||
if let Some(cap) = re.captures(&line) {
|
|
||||||
matched = Some(cap.get(1).unwrap().as_str().into());
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
matched.unwrap()
|
|
||||||
}
|
|
||||||
|
|
||||||
struct PreparedApp<'a> {
|
struct PreparedApp<'a> {
|
||||||
bundler: &'a dyn Bundler,
|
bundler: &'a dyn Bundler,
|
||||||
pages: Vec<Page>,
|
pages: Vec<Page>,
|
||||||
|
@ -297,20 +249,11 @@ impl<'a> PreparedApp<'a> {
|
||||||
fn stop_server(&mut self) -> Child {
|
fn stop_server(&mut self) -> Child {
|
||||||
let mut proc = self.server.take().expect("Server never started").0;
|
let mut proc = self.server.take().expect("Server never started").0;
|
||||||
proc.kill().unwrap();
|
proc.kill().unwrap();
|
||||||
|
proc.wait().unwrap();
|
||||||
proc
|
proc
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn command(bin: &str) -> Command {
|
|
||||||
if cfg!(windows) {
|
|
||||||
let mut command = Command::new("cmd.exe");
|
|
||||||
command.args(["/C", bin]);
|
|
||||||
command
|
|
||||||
} else {
|
|
||||||
Command::new(bin)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn build_test(module_count: usize) -> PathBuf {
|
fn build_test(module_count: usize) -> PathBuf {
|
||||||
let test_dir = TestAppBuilder {
|
let test_dir = TestAppBuilder {
|
||||||
module_count,
|
module_count,
|
||||||
|
|
Loading…
Reference in a new issue