Calling turbopack from the next build CLI (#46602)

Close WEB-661

.github/workflows/build_test_deploy.yml

@@ -1498,6 +1498,8 @@ jobs:
         with:
           envs: DEBUG RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS CARGO_PROFILE_RELEASE_LTO NAPI_CLI_VERSION RUST_TOOLCHAIN PNPM_VERSION VM_RELEASE
           usesh: true
+          sync: rsync
+          copyback: false
           mem: 6000
           prepare: |
             pkg install -y -f curl node libnghttp2

packages/next-swc/Cargo.lock

@@ -3008,6 +3008,7 @@ version = "0.1.0"
 dependencies = [
  "mdxjs",
  "modularize_imports",
+ "next-build",
  "next-dev",
  "node-file-trace",
  "styled_components",
@@ -3017,6 +3018,19 @@ dependencies = [
  "testing",
 ]
 
+[[package]]
+name = "next-build"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "next-core",
+ "turbo-malloc",
+ "turbo-tasks",
+ "turbo-tasks-build",
+ "turbo-tasks-memory",
+ "vergen",
+]
+
 [[package]]
 name = "next-core"
 version = "0.1.0"

packages/next-swc/Cargo.toml

@@ -5,6 +5,7 @@ members = [
   "crates/napi",
   "crates/wasm",
   "crates/next-binding",
+  "crates/next-build",
   "crates/next-core",
   "crates/next-dev",
   "crates/next-dev-tests",

packages/next-swc/crates/napi/Cargo.toml

@@ -44,6 +44,7 @@ turbo-tasks = { workspace = true }
 turbo-tasks-memory = { workspace = true }
 next-binding = { path = "../next-binding", features = [
   "__swc_core_binding_napi",
+  "__turbo_next_build",
   "__turbo_next_dev_server",
   "__turbo_node_file_trace",
   "__feature_mdx_rs",

packages/next-swc/crates/napi/src/turbopack.rs

@@ -1,9 +1,110 @@
+use std::convert::TryFrom;
+
 use crate::util::MapErr;
 use napi::bindgen_prelude::*;
-use next_binding::turbo::next_dev::{devserver_options::DevServerOptions, start_server};
+use next_binding::turbo::{
+    next_build::{next_build as turbo_next_build, NextBuildOptions},
+    next_dev::{devserver_options::DevServerOptions, start_server},
+};
 
 #[napi]
 pub async fn start_turbo_dev(options: Buffer) -> napi::Result<()> {
     let options: DevServerOptions = serde_json::from_slice(&options)?;
     start_server(&options).await.convert_err()
 }
+
+#[napi(object, object_to_js = false)]
+#[derive(Debug)]
+pub struct NextBuildContext {
+    pub dir: Option<String>,
+    pub app_dir: Option<String>,
+    pub pages_dir: Option<String>,
+    pub rewrites: Option<Rewrites>,
+    pub original_rewrites: Option<Rewrites>,
+    pub original_redirects: Option<Vec<Redirect>>,
+}
+
+#[napi(object, object_to_js = false)]
+#[derive(Debug)]
+pub struct Rewrites {
+    pub fallback: Vec<Rewrite>,
+    pub after_files: Vec<Rewrite>,
+    pub before_files: Vec<Rewrite>,
+}
+
+#[napi(object, object_to_js = false)]
+#[derive(Debug)]
+pub struct Rewrite {
+    pub source: String,
+    pub destination: String,
+}
+
+#[napi(object, object_to_js = false)]
+#[derive(Debug)]
+pub struct Redirect {
+    pub source: String,
+    pub destination: String,
+    pub permanent: Option<bool>,
+    pub status_code: Option<u32>,
+    pub has: Option<RouteHas>,
+    pub missing: Option<RouteHas>,
+}
+
+#[derive(Debug)]
+pub struct RouteHas {
+    pub r#type: RouteType,
+    pub key: Option<String>,
+    pub value: Option<String>,
+}
+
+#[derive(Debug)]
+pub enum RouteType {
+    Header,
+    Query,
+    Cookie,
+    Host,
+}
+
+impl TryFrom<String> for RouteType {
+    type Error = napi::Error;
+
+    fn try_from(value: String) -> Result<Self> {
+        match value.as_str() {
+            "header" => Ok(RouteType::Header),
+            "query" => Ok(RouteType::Query),
+            "cookie" => Ok(RouteType::Cookie),
+            "host" => Ok(RouteType::Host),
+            _ => Err(napi::Error::new(
+                napi::Status::InvalidArg,
+                "Invalid route type",
+            )),
+        }
+    }
+}
+
+impl FromNapiValue for RouteHas {
+    unsafe fn from_napi_value(env: sys::napi_env, napi_val: sys::napi_value) -> Result<Self> {
+        let object = Object::from_napi_value(env, napi_val)?;
+        let r#type = object.get_named_property::<String>("type")?;
+        Ok(RouteHas {
+            r#type: RouteType::try_from(r#type)?,
+            key: object.get("key")?,
+            value: object.get("value")?,
+        })
+    }
+}
+
+impl From<NextBuildContext> for NextBuildOptions {
+    fn from(value: NextBuildContext) -> Self {
+        Self {
+            dir: value.dir,
+            memory_limit: None,
+            full_stats: None,
+        }
+    }
+}
+
+#[napi]
+pub async fn next_build(ctx: NextBuildContext) -> napi::Result<()> {
+    turbo_next_build(ctx.into()).await.convert_err()
+}

packages/next-swc/crates/next-binding/Cargo.toml

@@ -74,6 +74,7 @@ __swc_core_binding_wasm_plugin = ["swc_core/plugin_transform_host_js"]
 __swc_core_testing_transform = ["swc_core/testing_transform"]
 
 __turbo = []
+__turbo_next_build = ["__turbo", "next-build"]
 __turbo_next_dev_server = ["__turbo", "next-dev/serializable"]
 __turbo_node_file_trace = ["__turbo", "node-file-trace/node-api"]
 
@@ -102,6 +103,9 @@ __swc_testing = ["__swc", "testing"]
 [dependencies]
 mdxjs = { optional = true, workspace = true }
 modularize_imports = { optional = true, workspace = true }
+next-build = { optional = true, path = "../next-build", default-features = false, features = [
+  "custom_allocator",
+] }
 # TODO: Not sure what's going on, but using `workspace = true` noops `default-features = false`?
 next-dev = { optional = true, path = "../next-dev", default-features = false, features = [
   "custom_allocator",

packages/next-swc/crates/next-binding/src/lib.rs

@@ -21,6 +21,8 @@ pub mod swc {
 
 #[cfg(feature = "__turbo")]
 pub mod turbo {
+    #[cfg(feature = "__turbo_next_build")]
+    pub use next_build;
     #[cfg(feature = "__turbo_next_dev_server")]
     pub use next_dev;
     #[cfg(feature = "__turbo_node_file_trace")]

packages/next-swc/crates/next-build/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "next-build"
+version = "0.1.0"
+description = "TBD"
+license = "MPL-2.0"
+edition = "2021"
+autobenches = false
+
+[features]
+next-font-local = ["next-core/next-font-local"]
+native-tls = ["next-core/native-tls"]
+rustls-tls = ["next-core/rustls-tls"]
+custom_allocator = ["turbo-malloc/custom_allocator"]
+
+[dependencies]
+anyhow = "1.0.47"
+next-core = { workspace = true }
+turbo-malloc = { workspace = true, default-features = false }
+turbo-tasks = { workspace = true }
+turbo-tasks-memory = { workspace = true }
+
+[build-dependencies]
+turbo-tasks-build = { workspace = true }
+vergen = { version = "7.3.2", default-features = false, features = [
+  "cargo",
+  "build",
+] }

packages/next-swc/crates/next-build/build.rs

@@ -0,0 +1,13 @@
+use turbo_tasks_build::{generate_register, rerun_if_glob};
+
+use vergen::{vergen, Config};
+
+fn main() {
+    generate_register();
+
+    rerun_if_glob("tests/integration/*/*", "tests/integration");
+
+    // Attempt to collect some build time env values but will skip if there are any
+    // errors.
+    let _ = vergen(Config::default());
+}

packages/next-swc/crates/next-build/src/lib.rs

@@ -0,0 +1,33 @@
+use turbo_tasks::{NothingVc, StatsType, TurboTasks, TurboTasksBackendApi};
+use turbo_tasks_memory::MemoryBackend;
+
+pub fn register() {
+    turbo_tasks::register();
+    include!(concat!(env!("OUT_DIR"), "/register.rs"));
+}
+
+pub struct NextBuildOptions {
+    pub dir: Option<String>,
+    pub memory_limit: Option<usize>,
+    pub full_stats: Option<bool>,
+}
+
+pub async fn next_build(options: NextBuildOptions) -> anyhow::Result<()> {
+    register();
+    let tt = TurboTasks::new(MemoryBackend::new(
+        options.memory_limit.map_or(usize::MAX, |l| l * 1024 * 1024),
+    ));
+    let stats_type = match options.full_stats {
+        Some(true) => StatsType::Full,
+        _ => StatsType::Essential,
+    };
+    tt.set_stats_type(stats_type);
+    let task = tt.spawn_root_task(move || {
+        Box::pin(async move {
+            // run next build here
+            Ok(NothingVc::new().into())
+        })
+    });
+    tt.wait_task_completion(task, true).await?;
+    Ok(())
+}

packages/next/src/build/index.ts

@@ -255,7 +255,8 @@ export default async function build(
   debugOutput = false,
   runLint = true,
   noMangling = false,
-  appDirOnly = false
+  appDirOnly = false,
+  turboNextBuild = false
 ): Promise<void> {
   try {
     const nextBuildSpan = trace('next-build', undefined, {
@@ -1010,8 +1011,17 @@ export default async function build(
           ignore: [] as string[],
         }))
 
+      let binding = (await loadBindings()) as any
+
+      async function turbopackBuild() {
+        const turboNextBuildStart = process.hrtime()
+        await binding.turbo.nextBuild(NextBuildContext)
+        const [duration] = process.hrtime(turboNextBuildStart)
+        return { duration, turbotraceContext: null }
+      }
+
       const { duration: webpackBuildDuration, turbotraceContext } =
-        await webpackBuild()
+        turboNextBuild ? await turbopackBuild() : await webpackBuild()
 
       telemetry.record(
         eventBuildCompleted(pagesPaths, {
@@ -1026,7 +1036,6 @@ export default async function build(
         if (!turbotraceContext) {
           return
         }
-        let binding = (await loadBindings()) as any
         if (
           !binding?.isWasm &&
           typeof binding.turbo.startTrace === 'function'
@@ -1069,11 +1078,9 @@ export default async function build(
             if (filesTracedFromEntries.length) {
               // The turbo trace doesn't provide the traced file type and reason at present
               // let's write the traced files into the first [entry].nft.json
-              // @ts-expect-error types
-              const [[, entryName]] = Array.from(entryNameMap.entries()).filter(
-                // @ts-expect-error types
-                ([k]) => k.startsWith(turbotraceContextAppDir)
-              )
+              const [[, entryName]] = Array.from<[string, string]>(
+                entryNameMap.entries()
+              ).filter(([k]) => k.startsWith(turbotraceContextAppDir))
               const traceOutputPath = path.join(
                 outputPath,
                 `../${entryName}.js.nft.json`
@@ -1797,7 +1804,6 @@ export default async function build(
       } else if (config.outputFileTracing) {
         let nodeFileTrace: any
         if (config.experimental.turbotrace) {
-          let binding = (await loadBindings()) as any
           if (!binding?.isWasm) {
             nodeFileTrace = binding.turbo.startTrace
           }

packages/next/src/build/swc/index.ts

@@ -471,6 +471,9 @@ function loadNative(isCustomTurbopack = false) {
             require(__INTERNAL_CUSTOM_TURBOPACK_BINDINGS).startDev(devOptions)
           }
         },
+        nextBuild: (options: unknown) => {
+          return bindings.nextBuild(options)
+        },
         startTrace: (options = {}, turboTasks: unknown) =>
           bindings.runTurboTracing(
             toBuffer({ exact: true, ...options }),

packages/next/src/build/webpack-build.ts

@@ -409,7 +409,7 @@ async function webpackBuildWithWorker() {
 
   const combinedResult = {
     duration: 0,
-    turbotraceContext: {} as any,
+    turbotraceContext: {} as TurbotraceContext,
   }
   // order matters here
   const ORDERED_COMPILER_NAMES = [
@@ -447,9 +447,9 @@ async function webpackBuildWithWorker() {
     if (curResult.turbotraceContext?.entriesTrace) {
       combinedResult.turbotraceContext = curResult.turbotraceContext
 
-      const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace
+      const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace!
       if (entryNameMap) {
-        combinedResult.turbotraceContext.entriesTrace.entryNameMap = new Map(
+        combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map(
           entryNameMap
         )
       }

packages/next/src/cli/next-build.ts

@@ -17,6 +17,7 @@ const nextBuild: CliCommand = (argv) => {
     '--no-lint': Boolean,
     '--no-mangling': Boolean,
     '--experimental-app-only': Boolean,
+    '--experimental-turbo': Boolean,
     // Aliases
     '-h': '--help',
     '-d': '--debug',
@@ -76,7 +77,8 @@ const nextBuild: CliCommand = (argv) => {
     args['--debug'] || process.env.NEXT_DEBUG_BUILD,
     !args['--no-lint'],
     args['--no-mangling'],
-    args['--experimental-app-only']
+    args['--experimental-app-only'],
+    args['--experimental-turbo']
   ).catch((err) => {
     console.error('')
     if (