From 509ed00fc163bc185221cd8d9b40c726ac36aadf Mon Sep 17 00:00:00 2001 From: LongYinan Date: Tue, 21 Mar 2023 18:25:08 +0800 Subject: [PATCH] Calling turbopack from the next build CLI (#46602) Close WEB-661 --- .github/workflows/build_test_deploy.yml | 2 + packages/next-swc/Cargo.lock | 14 +++ packages/next-swc/Cargo.toml | 1 + packages/next-swc/crates/napi/Cargo.toml | 1 + .../next-swc/crates/napi/src/turbopack.rs | 103 +++++++++++++++++- .../next-swc/crates/next-binding/Cargo.toml | 4 + .../next-swc/crates/next-binding/src/lib.rs | 2 + .../next-swc/crates/next-build/Cargo.toml | 27 +++++ packages/next-swc/crates/next-build/build.rs | 13 +++ .../next-swc/crates/next-build/src/lib.rs | 33 ++++++ packages/next/src/build/index.ts | 24 ++-- packages/next/src/build/swc/index.ts | 3 + packages/next/src/build/webpack-build.ts | 6 +- packages/next/src/cli/next-build.ts | 4 +- 14 files changed, 223 insertions(+), 14 deletions(-) create mode 100644 packages/next-swc/crates/next-build/Cargo.toml create mode 100644 packages/next-swc/crates/next-build/build.rs create mode 100644 packages/next-swc/crates/next-build/src/lib.rs diff --git a/.github/workflows/build_test_deploy.yml b/.github/workflows/build_test_deploy.yml index e8f52df18c..0ad4b7744b 100644 --- a/.github/workflows/build_test_deploy.yml +++ b/.github/workflows/build_test_deploy.yml @@ -1498,6 +1498,8 @@ jobs: with: envs: DEBUG RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS CARGO_PROFILE_RELEASE_LTO NAPI_CLI_VERSION RUST_TOOLCHAIN PNPM_VERSION VM_RELEASE usesh: true + sync: rsync + copyback: false mem: 6000 prepare: | pkg install -y -f curl node libnghttp2 diff --git a/packages/next-swc/Cargo.lock b/packages/next-swc/Cargo.lock index 05043a146b..c1b5fd0fa8 100644 --- a/packages/next-swc/Cargo.lock +++ b/packages/next-swc/Cargo.lock @@ -3008,6 +3008,7 @@ version = "0.1.0" dependencies = [ "mdxjs", "modularize_imports", + "next-build", "next-dev", "node-file-trace", "styled_components", @@ -3017,6 +3018,19 @@ dependencies = [ "testing", ] +[[package]] +name = "next-build" +version = "0.1.0" +dependencies = [ + "anyhow", + "next-core", + "turbo-malloc", + "turbo-tasks", + "turbo-tasks-build", + "turbo-tasks-memory", + "vergen", +] + [[package]] name = "next-core" version = "0.1.0" diff --git a/packages/next-swc/Cargo.toml b/packages/next-swc/Cargo.toml index 2172348cdd..5c12148820 100644 --- a/packages/next-swc/Cargo.toml +++ b/packages/next-swc/Cargo.toml @@ -5,6 +5,7 @@ members = [ "crates/napi", "crates/wasm", "crates/next-binding", + "crates/next-build", "crates/next-core", "crates/next-dev", "crates/next-dev-tests", diff --git a/packages/next-swc/crates/napi/Cargo.toml b/packages/next-swc/crates/napi/Cargo.toml index 657d2a41c8..bd45cc8ba7 100644 --- a/packages/next-swc/crates/napi/Cargo.toml +++ b/packages/next-swc/crates/napi/Cargo.toml @@ -44,6 +44,7 @@ turbo-tasks = { workspace = true } turbo-tasks-memory = { workspace = true } next-binding = { path = "../next-binding", features = [ "__swc_core_binding_napi", + "__turbo_next_build", "__turbo_next_dev_server", "__turbo_node_file_trace", "__feature_mdx_rs", diff --git a/packages/next-swc/crates/napi/src/turbopack.rs b/packages/next-swc/crates/napi/src/turbopack.rs index 61ea6f3689..4e2aa29d39 100644 --- a/packages/next-swc/crates/napi/src/turbopack.rs +++ b/packages/next-swc/crates/napi/src/turbopack.rs @@ -1,9 +1,110 @@ +use std::convert::TryFrom; + use crate::util::MapErr; use napi::bindgen_prelude::*; -use next_binding::turbo::next_dev::{devserver_options::DevServerOptions, start_server}; +use next_binding::turbo::{ + next_build::{next_build as turbo_next_build, NextBuildOptions}, + next_dev::{devserver_options::DevServerOptions, start_server}, +}; #[napi] pub async fn start_turbo_dev(options: Buffer) -> napi::Result<()> { let options: DevServerOptions = serde_json::from_slice(&options)?; start_server(&options).await.convert_err() } + +#[napi(object, object_to_js = false)] +#[derive(Debug)] +pub struct NextBuildContext { + pub dir: Option, + pub app_dir: Option, + pub pages_dir: Option, + pub rewrites: Option, + pub original_rewrites: Option, + pub original_redirects: Option>, +} + +#[napi(object, object_to_js = false)] +#[derive(Debug)] +pub struct Rewrites { + pub fallback: Vec, + pub after_files: Vec, + pub before_files: Vec, +} + +#[napi(object, object_to_js = false)] +#[derive(Debug)] +pub struct Rewrite { + pub source: String, + pub destination: String, +} + +#[napi(object, object_to_js = false)] +#[derive(Debug)] +pub struct Redirect { + pub source: String, + pub destination: String, + pub permanent: Option, + pub status_code: Option, + pub has: Option, + pub missing: Option, +} + +#[derive(Debug)] +pub struct RouteHas { + pub r#type: RouteType, + pub key: Option, + pub value: Option, +} + +#[derive(Debug)] +pub enum RouteType { + Header, + Query, + Cookie, + Host, +} + +impl TryFrom for RouteType { + type Error = napi::Error; + + fn try_from(value: String) -> Result { + match value.as_str() { + "header" => Ok(RouteType::Header), + "query" => Ok(RouteType::Query), + "cookie" => Ok(RouteType::Cookie), + "host" => Ok(RouteType::Host), + _ => Err(napi::Error::new( + napi::Status::InvalidArg, + "Invalid route type", + )), + } + } +} + +impl FromNapiValue for RouteHas { + unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> Result { + let object = Object::from_napi_value(env, napi_val)?; + let r#type = object.get_named_property::("type")?; + Ok(RouteHas { + r#type: RouteType::try_from(r#type)?, + key: object.get("key")?, + value: object.get("value")?, + }) + } +} + +impl From for NextBuildOptions { + fn from(value: NextBuildContext) -> Self { + Self { + dir: value.dir, + memory_limit: None, + full_stats: None, + } + } +} + +#[napi] +pub async fn next_build(ctx: NextBuildContext) -> napi::Result<()> { + turbo_next_build(ctx.into()).await.convert_err() +} diff --git a/packages/next-swc/crates/next-binding/Cargo.toml b/packages/next-swc/crates/next-binding/Cargo.toml index 2242f10c4e..668e4731d9 100644 --- a/packages/next-swc/crates/next-binding/Cargo.toml +++ b/packages/next-swc/crates/next-binding/Cargo.toml @@ -74,6 +74,7 @@ __swc_core_binding_wasm_plugin = ["swc_core/plugin_transform_host_js"] __swc_core_testing_transform = ["swc_core/testing_transform"] __turbo = [] +__turbo_next_build = ["__turbo", "next-build"] __turbo_next_dev_server = ["__turbo", "next-dev/serializable"] __turbo_node_file_trace = ["__turbo", "node-file-trace/node-api"] @@ -102,6 +103,9 @@ __swc_testing = ["__swc", "testing"] [dependencies] mdxjs = { optional = true, workspace = true } modularize_imports = { optional = true, workspace = true } +next-build = { optional = true, path = "../next-build", default-features = false, features = [ + "custom_allocator", +] } # TODO: Not sure what's going on, but using `workspace = true` noops `default-features = false`? next-dev = { optional = true, path = "../next-dev", default-features = false, features = [ "custom_allocator", diff --git a/packages/next-swc/crates/next-binding/src/lib.rs b/packages/next-swc/crates/next-binding/src/lib.rs index 03b0741db7..db7756d02e 100644 --- a/packages/next-swc/crates/next-binding/src/lib.rs +++ b/packages/next-swc/crates/next-binding/src/lib.rs @@ -21,6 +21,8 @@ pub mod swc { #[cfg(feature = "__turbo")] pub mod turbo { + #[cfg(feature = "__turbo_next_build")] + pub use next_build; #[cfg(feature = "__turbo_next_dev_server")] pub use next_dev; #[cfg(feature = "__turbo_node_file_trace")] diff --git a/packages/next-swc/crates/next-build/Cargo.toml b/packages/next-swc/crates/next-build/Cargo.toml new file mode 100644 index 0000000000..8d4ab64f93 --- /dev/null +++ b/packages/next-swc/crates/next-build/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "next-build" +version = "0.1.0" +description = "TBD" +license = "MPL-2.0" +edition = "2021" +autobenches = false + +[features] +next-font-local = ["next-core/next-font-local"] +native-tls = ["next-core/native-tls"] +rustls-tls = ["next-core/rustls-tls"] +custom_allocator = ["turbo-malloc/custom_allocator"] + +[dependencies] +anyhow = "1.0.47" +next-core = { workspace = true } +turbo-malloc = { workspace = true, default-features = false } +turbo-tasks = { workspace = true } +turbo-tasks-memory = { workspace = true } + +[build-dependencies] +turbo-tasks-build = { workspace = true } +vergen = { version = "7.3.2", default-features = false, features = [ + "cargo", + "build", +] } \ No newline at end of file diff --git a/packages/next-swc/crates/next-build/build.rs b/packages/next-swc/crates/next-build/build.rs new file mode 100644 index 0000000000..cc35b5e244 --- /dev/null +++ b/packages/next-swc/crates/next-build/build.rs @@ -0,0 +1,13 @@ +use turbo_tasks_build::{generate_register, rerun_if_glob}; + +use vergen::{vergen, Config}; + +fn main() { + generate_register(); + + rerun_if_glob("tests/integration/*/*", "tests/integration"); + + // Attempt to collect some build time env values but will skip if there are any + // errors. + let _ = vergen(Config::default()); +} diff --git a/packages/next-swc/crates/next-build/src/lib.rs b/packages/next-swc/crates/next-build/src/lib.rs new file mode 100644 index 0000000000..e9d04ee765 --- /dev/null +++ b/packages/next-swc/crates/next-build/src/lib.rs @@ -0,0 +1,33 @@ +use turbo_tasks::{NothingVc, StatsType, TurboTasks, TurboTasksBackendApi}; +use turbo_tasks_memory::MemoryBackend; + +pub fn register() { + turbo_tasks::register(); + include!(concat!(env!("OUT_DIR"), "/register.rs")); +} + +pub struct NextBuildOptions { + pub dir: Option, + pub memory_limit: Option, + pub full_stats: Option, +} + +pub async fn next_build(options: NextBuildOptions) -> anyhow::Result<()> { + register(); + let tt = TurboTasks::new(MemoryBackend::new( + options.memory_limit.map_or(usize::MAX, |l| l * 1024 * 1024), + )); + let stats_type = match options.full_stats { + Some(true) => StatsType::Full, + _ => StatsType::Essential, + }; + tt.set_stats_type(stats_type); + let task = tt.spawn_root_task(move || { + Box::pin(async move { + // run next build here + Ok(NothingVc::new().into()) + }) + }); + tt.wait_task_completion(task, true).await?; + Ok(()) +} diff --git a/packages/next/src/build/index.ts b/packages/next/src/build/index.ts index 4f62bd3aa2..2aa333e22c 100644 --- a/packages/next/src/build/index.ts +++ b/packages/next/src/build/index.ts @@ -255,7 +255,8 @@ export default async function build( debugOutput = false, runLint = true, noMangling = false, - appDirOnly = false + appDirOnly = false, + turboNextBuild = false ): Promise { try { const nextBuildSpan = trace('next-build', undefined, { @@ -1010,8 +1011,17 @@ export default async function build( ignore: [] as string[], })) + let binding = (await loadBindings()) as any + + async function turbopackBuild() { + const turboNextBuildStart = process.hrtime() + await binding.turbo.nextBuild(NextBuildContext) + const [duration] = process.hrtime(turboNextBuildStart) + return { duration, turbotraceContext: null } + } + const { duration: webpackBuildDuration, turbotraceContext } = - await webpackBuild() + turboNextBuild ? await turbopackBuild() : await webpackBuild() telemetry.record( eventBuildCompleted(pagesPaths, { @@ -1026,7 +1036,6 @@ export default async function build( if (!turbotraceContext) { return } - let binding = (await loadBindings()) as any if ( !binding?.isWasm && typeof binding.turbo.startTrace === 'function' @@ -1069,11 +1078,9 @@ export default async function build( if (filesTracedFromEntries.length) { // The turbo trace doesn't provide the traced file type and reason at present // let's write the traced files into the first [entry].nft.json - // @ts-expect-error types - const [[, entryName]] = Array.from(entryNameMap.entries()).filter( - // @ts-expect-error types - ([k]) => k.startsWith(turbotraceContextAppDir) - ) + const [[, entryName]] = Array.from<[string, string]>( + entryNameMap.entries() + ).filter(([k]) => k.startsWith(turbotraceContextAppDir)) const traceOutputPath = path.join( outputPath, `../${entryName}.js.nft.json` @@ -1797,7 +1804,6 @@ export default async function build( } else if (config.outputFileTracing) { let nodeFileTrace: any if (config.experimental.turbotrace) { - let binding = (await loadBindings()) as any if (!binding?.isWasm) { nodeFileTrace = binding.turbo.startTrace } diff --git a/packages/next/src/build/swc/index.ts b/packages/next/src/build/swc/index.ts index 36473fc618..ca33f8a337 100644 --- a/packages/next/src/build/swc/index.ts +++ b/packages/next/src/build/swc/index.ts @@ -471,6 +471,9 @@ function loadNative(isCustomTurbopack = false) { require(__INTERNAL_CUSTOM_TURBOPACK_BINDINGS).startDev(devOptions) } }, + nextBuild: (options: unknown) => { + return bindings.nextBuild(options) + }, startTrace: (options = {}, turboTasks: unknown) => bindings.runTurboTracing( toBuffer({ exact: true, ...options }), diff --git a/packages/next/src/build/webpack-build.ts b/packages/next/src/build/webpack-build.ts index 088bf4b67a..f2832bed8d 100644 --- a/packages/next/src/build/webpack-build.ts +++ b/packages/next/src/build/webpack-build.ts @@ -409,7 +409,7 @@ async function webpackBuildWithWorker() { const combinedResult = { duration: 0, - turbotraceContext: {} as any, + turbotraceContext: {} as TurbotraceContext, } // order matters here const ORDERED_COMPILER_NAMES = [ @@ -447,9 +447,9 @@ async function webpackBuildWithWorker() { if (curResult.turbotraceContext?.entriesTrace) { combinedResult.turbotraceContext = curResult.turbotraceContext - const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace + const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace! if (entryNameMap) { - combinedResult.turbotraceContext.entriesTrace.entryNameMap = new Map( + combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map( entryNameMap ) } diff --git a/packages/next/src/cli/next-build.ts b/packages/next/src/cli/next-build.ts index a2d5546ef2..fb2dc714b9 100755 --- a/packages/next/src/cli/next-build.ts +++ b/packages/next/src/cli/next-build.ts @@ -17,6 +17,7 @@ const nextBuild: CliCommand = (argv) => { '--no-lint': Boolean, '--no-mangling': Boolean, '--experimental-app-only': Boolean, + '--experimental-turbo': Boolean, // Aliases '-h': '--help', '-d': '--debug', @@ -76,7 +77,8 @@ const nextBuild: CliCommand = (argv) => { args['--debug'] || process.env.NEXT_DEBUG_BUILD, !args['--no-lint'], args['--no-mangling'], - args['--experimental-app-only'] + args['--experimental-app-only'], + args['--experimental-turbo'] ).catch((err) => { console.error('') if (