diff --git a/packages/next-swc/crates/next-core/src/lib.rs b/packages/next-swc/crates/next-core/src/lib.rs index 3041c1dc3d..9240bcd1f9 100644 --- a/packages/next-swc/crates/next-core/src/lib.rs +++ b/packages/next-swc/crates/next-core/src/lib.rs @@ -3,6 +3,7 @@ pub mod env; pub mod next_client; +mod nodejs; pub mod react_refresh; mod server_render; mod server_rendered_source; diff --git a/packages/next-swc/crates/next-core/src/server_render/nodejs_bootstrap.rs b/packages/next-swc/crates/next-core/src/nodejs/bootstrap.rs similarity index 94% rename from packages/next-swc/crates/next-core/src/server_render/nodejs_bootstrap.rs rename to packages/next-swc/crates/next-core/src/nodejs/bootstrap.rs index b291b5a57d..6f79defa15 100644 --- a/packages/next-swc/crates/next-core/src/server_render/nodejs_bootstrap.rs +++ b/packages/next-swc/crates/next-core/src/nodejs/bootstrap.rs @@ -28,7 +28,7 @@ impl Asset for NodeJsBootstrapAsset { // TODO(sokra) We need to have a chunk format for node.js // but until then this is a simple hack to make it work for now - let mut output = "global.self = global;\n".to_string(); + let mut output = "Error.stackTraceLimit = 100;\nglobal.self = global;\n".to_string(); for chunk in self.chunk_group.chunks().await?.iter() { let path = &*chunk.path().await?; diff --git a/packages/next-swc/crates/next-core/src/server_render/issue.rs b/packages/next-swc/crates/next-core/src/nodejs/issue.rs similarity index 100% rename from packages/next-swc/crates/next-core/src/server_render/issue.rs rename to packages/next-swc/crates/next-core/src/nodejs/issue.rs diff --git a/packages/next-swc/crates/next-core/src/nodejs/mod.rs b/packages/next-swc/crates/next-core/src/nodejs/mod.rs new file mode 100644 index 0000000000..704067c344 --- /dev/null +++ b/packages/next-swc/crates/next-core/src/nodejs/mod.rs @@ -0,0 +1,277 @@ +use std::{ + collections::{HashMap, HashSet}, + path::PathBuf, +}; + +use anyhow::{anyhow, Result}; +use futures::{stream::FuturesUnordered, TryStreamExt}; +use mime::TEXT_HTML_UTF_8; +use serde_json::Value as JsonValue; +use turbo_tasks::{ + primitives::{JsonValueVc, StringVc}, + spawn_blocking, CompletionVc, CompletionsVc, ValueToString, +}; +use turbo_tasks_fs::{DiskFileSystemVc, File, FileContent, FileSystemPathVc}; +use turbopack::ecmascript::EcmascriptModuleAssetVc; +use turbopack_core::{ + asset::{AssetContentVc, AssetVc, AssetsSetVc}, + chunk::{dev::DevChunkingContextVc, ChunkGroupVc}, +}; +use turbopack_ecmascript::chunk::EcmascriptChunkPlaceablesVc; + +use self::{ + bootstrap::NodeJsBootstrapAsset, + pool::{NodeJsPool, NodeJsPoolVc}, +}; +use crate::nodejs::issue::RenderingIssue; + +pub mod bootstrap; +pub(crate) mod issue; +pub mod pool; + +#[turbo_tasks::function] +async fn emit( + intermediate_asset: AssetVc, + intermediate_output_path: FileSystemPathVc, +) -> Result { + Ok(CompletionsVc::cell( + internal_assets(intermediate_asset, intermediate_output_path) + .await? + .iter() + .map(|a| a.content().write(a.path())) + .collect(), + ) + .all()) +} + +/// List of the all assets of the "internal" subgraph and a list of boundary +/// assets that are not considered "internal" ("external") +#[turbo_tasks::value] +pub struct SeparatedAssets { + pub internal_assets: AssetsSetVc, + pub external_asset_entrypoints: AssetsSetVc, +} + +/// Extracts the subgraph of "internal" assets (assets within the passes +/// directory). Also lists all boundary assets that are not part of the +/// "internal" subgraph. +#[turbo_tasks::function] +async fn internal_assets( + intermediate_asset: AssetVc, + intermediate_output_path: FileSystemPathVc, +) -> Result { + Ok( + separate_assets(intermediate_asset, intermediate_output_path) + .await? + .internal_assets, + ) +} + +/// Returns a set of "external" assets on the boundary of the "internal" +/// subgraph +#[turbo_tasks::function] +pub async fn external_asset_entrypoints( + module: EcmascriptModuleAssetVc, + runtime_entries: EcmascriptChunkPlaceablesVc, + chunking_context: DevChunkingContextVc, + intermediate_output_path: FileSystemPathVc, +) -> Result { + Ok(separate_assets( + get_intermediate_asset( + module, + runtime_entries, + chunking_context, + intermediate_output_path, + ), + intermediate_output_path, + ) + .await? + .external_asset_entrypoints) +} + +/// Splits the asset graph into "internal" assets and boundaries to "external" +/// assets. +#[turbo_tasks::function] +async fn separate_assets( + intermediate_asset: AssetVc, + intermediate_output_path: FileSystemPathVc, +) -> Result { + enum Type { + Internal(AssetVc, Vec), + External(AssetVc), + } + let intermediate_output_path = intermediate_output_path.await?; + let mut queue = FuturesUnordered::new(); + let process_asset = |asset: AssetVc| { + let intermediate_output_path = &intermediate_output_path; + async move { + // Assets within the output directory are considered as "internal" and all + // others as "external". We follow references on "internal" assets, but do not + // look into references of "external" assets, since there are no "internal" + // assets behind "externals" + if asset.path().await?.is_inside(intermediate_output_path) { + let mut assets = Vec::new(); + for reference in asset.references().await?.iter() { + for asset in reference.resolve_reference().primary_assets().await?.iter() { + assets.push(*asset); + } + } + Ok::<_, anyhow::Error>(Type::Internal(asset, assets)) + } else { + Ok(Type::External(asset)) + } + } + }; + queue.push(process_asset(intermediate_asset)); + let mut processed = HashSet::new(); + let mut internal_assets = HashSet::new(); + let mut external_asset_entrypoints = HashSet::new(); + while let Some(item) = queue.try_next().await? { + match item { + Type::Internal(asset, assets) => { + internal_assets.insert(asset); + for asset in assets { + if processed.insert(asset) { + queue.push(process_asset(asset)); + } + } + } + Type::External(asset) => { + external_asset_entrypoints.insert(asset); + } + } + } + Ok(SeparatedAssets { + internal_assets: AssetsSetVc::cell(internal_assets), + external_asset_entrypoints: AssetsSetVc::cell(external_asset_entrypoints), + } + .cell()) +} + +/// Creates a node.js renderer pool for an entrypoint. +#[turbo_tasks::function] +pub async fn get_renderer_pool( + intermediate_asset: AssetVc, + intermediate_output_path: FileSystemPathVc, +) -> Result { + emit(intermediate_asset, intermediate_output_path).await?; + let output = intermediate_output_path.await?; + if let Some(disk) = DiskFileSystemVc::resolve_from(output.fs).await? { + let dir = PathBuf::from(&disk.await?.root).join(&output.path); + let entrypoint = dir.join("index.js"); + let pool = NodeJsPool::new(dir, entrypoint, HashMap::new(), 4); + Ok(pool.cell()) + } else { + Err(anyhow!("can only render from a disk filesystem")) + } +} + +/// Converts a module graph into node.js executable assets +#[turbo_tasks::function] +async fn get_intermediate_asset( + entry_module: EcmascriptModuleAssetVc, + runtime_entries: EcmascriptChunkPlaceablesVc, + chunking_context: DevChunkingContextVc, + intermediate_output_path: FileSystemPathVc, +) -> Result { + let chunk = entry_module.as_evaluated_chunk(chunking_context.into(), Some(runtime_entries)); + let chunk_group = ChunkGroupVc::from_chunk(chunk); + Ok(NodeJsBootstrapAsset { + path: intermediate_output_path.join("index.js"), + chunk_group, + } + .cell() + .into()) +} + +/// Renders a module as static HTML in a node.js process. +#[turbo_tasks::function] +pub async fn render_static( + path: FileSystemPathVc, + module: EcmascriptModuleAssetVc, + runtime_entries: EcmascriptChunkPlaceablesVc, + chunking_context: DevChunkingContextVc, + intermediate_output_path: FileSystemPathVc, + data: JsonValueVc, +) -> Result { + fn into_result(content: String) -> Result { + Ok( + FileContent::Content(File::from_source(content).with_content_type(TEXT_HTML_UTF_8)) + .into(), + ) + } + let renderer_pool = get_renderer_pool( + get_intermediate_asset( + module, + runtime_entries, + chunking_context, + intermediate_output_path, + ), + intermediate_output_path, + ); + let pool = renderer_pool.await?; + let mut op = pool.run(data.to_string().await?.as_bytes()).await?; + let lines = spawn_blocking(move || { + let lines = op.read_lines()?; + drop(op); + Ok::<_, anyhow::Error>(lines) + }) + .await?; + let issue = if let Some(last_line) = lines.last() { + if let Some(data) = last_line.strip_prefix("RESULT=") { + let data: JsonValue = serde_json::from_str(data)?; + if let Some(s) = data.as_str() { + return into_result(s.to_string()); + } else { + RenderingIssue { + context: path, + message: StringVc::cell( + "Result provided by Node.js rendering process was not a string".to_string(), + ), + logging: StringVc::cell(lines.join("\n")), + } + } + } else if let Some(data) = last_line.strip_prefix("ERROR=") { + let data: JsonValue = serde_json::from_str(data)?; + if let Some(s) = data.as_str() { + RenderingIssue { + context: path, + message: StringVc::cell(s.to_string()), + logging: StringVc::cell(lines[..lines.len() - 1].join("\n")), + } + } else { + RenderingIssue { + context: path, + message: StringVc::cell(data.to_string()), + logging: StringVc::cell(lines[..lines.len() - 1].join("\n")), + } + } + } else { + RenderingIssue { + context: path, + message: StringVc::cell("No result provided by Node.js process".to_string()), + logging: StringVc::cell(lines.join("\n")), + } + } + } else { + RenderingIssue { + context: path, + message: StringVc::cell("No content received from Node.js process.".to_string()), + logging: StringVc::cell("".to_string()), + } + }; + + // Show error page + // TODO This need to include HMR handler to allow auto refresh + let result = into_result(format!( + "

Error during \ + rendering

\n

Message

\n
{}
\n

Logs

\n
{}
", + issue.message.await?, + issue.logging.await? + )); + + // Emit an issue for error reporting + issue.cell().as_issue().emit(); + + result +} diff --git a/packages/next-swc/crates/next-core/src/server_render/nodejs_pool.rs b/packages/next-swc/crates/next-core/src/nodejs/pool.rs similarity index 99% rename from packages/next-swc/crates/next-core/src/server_render/nodejs_pool.rs rename to packages/next-swc/crates/next-core/src/nodejs/pool.rs index ba26f44a9a..b5b5244b35 100644 --- a/packages/next-swc/crates/next-core/src/server_render/nodejs_pool.rs +++ b/packages/next-swc/crates/next-core/src/nodejs/pool.rs @@ -136,6 +136,7 @@ impl NodeJsPool { let static_input: &'static [u8] = unsafe { transmute(input) }; let child = spawn_blocking(move || { child.write(static_input)?; + child.write(b"\n")?; Ok::<_, anyhow::Error>(child) }) .await?; diff --git a/packages/next-swc/crates/next-core/src/server_render/asset.rs b/packages/next-swc/crates/next-core/src/server_render/asset.rs index a3c84af1da..ff89d2e0a1 100644 --- a/packages/next-swc/crates/next-core/src/server_render/asset.rs +++ b/packages/next-swc/crates/next-core/src/server_render/asset.rs @@ -1,23 +1,16 @@ -use std::{ - collections::{HashMap, HashSet}, - path::PathBuf, -}; - -use anyhow::{anyhow, Result}; -use futures::{stream::FuturesUnordered, TryStreamExt}; -use mime::TEXT_HTML_UTF_8; -use serde_json::Value as JsonValue; +use anyhow::Result; use turbo_tasks::{ - primitives::StringVc, spawn_blocking, CompletionVc, CompletionsVc, Value, ValueToString, - ValueToStringVc, + primitives::{JsonValueVc, StringVc}, + Value, ValueToString, ValueToStringVc, }; -use turbo_tasks_fs::{embed_file, DiskFileSystemVc, File, FileContent, FileSystemPathVc}; +use turbo_tasks_fs::{embed_file, FileSystemPathVc}; use turbopack::ecmascript::{ EcmascriptInputTransform, EcmascriptInputTransformsVc, EcmascriptModuleAssetVc, ModuleAssetType, }; use turbopack_core::{ + self, asset::{Asset, AssetContentVc, AssetVc}, - chunk::{dev::DevChunkingContextVc, ChunkGroupVc}, + chunk::dev::DevChunkingContextVc, context::AssetContextVc, reference::{AssetReference, AssetReferenceVc, AssetReferencesVc}, resolve::{ResolveResult, ResolveResultVc}, @@ -25,11 +18,7 @@ use turbopack_core::{ }; use turbopack_ecmascript::chunk::EcmascriptChunkPlaceablesVc; -use super::{ - nodejs_bootstrap::NodeJsBootstrapAsset, - nodejs_pool::{NodeJsPool, NodeJsPoolVc}, -}; -use crate::server_render::issue::RenderingIssue; +use crate::nodejs::{external_asset_entrypoints, render_static}; /// This is an asset which content is determined by running /// `React.renderToString` on the default export of [entry_asset] in a Node.js @@ -48,7 +37,7 @@ pub struct ServerRenderedAsset { chunking_context: DevChunkingContextVc, runtime_entries: EcmascriptChunkPlaceablesVc, intermediate_output_path: FileSystemPathVc, - request_data: String, + request_data: JsonValueVc, } #[turbo_tasks::value_impl] @@ -61,7 +50,7 @@ impl ServerRenderedAssetVc { runtime_entries: EcmascriptChunkPlaceablesVc, chunking_context: DevChunkingContextVc, intermediate_output_path: FileSystemPathVc, - request_data: String, + request_data: JsonValueVc, ) -> Self { ServerRenderedAsset { path, @@ -85,37 +74,26 @@ impl Asset for ServerRenderedAsset { #[turbo_tasks::function] fn content(&self) -> AssetContentVc { - render( + render_static( self.path, - get_renderer_pool( - get_intermediate_asset( - self.context, - self.entry_asset, - self.runtime_entries, - self.chunking_context, - self.intermediate_output_path, - ), - self.intermediate_output_path, - ), - &self.request_data, + get_intermediate_module(self.context, self.entry_asset), + self.runtime_entries, + self.chunking_context, + self.intermediate_output_path, + self.request_data, ) } #[turbo_tasks::function] async fn references(&self) -> Result { Ok(AssetReferencesVc::cell( - separate_assets( - get_intermediate_asset( - self.context, - self.entry_asset, - self.runtime_entries, - self.chunking_context, - self.intermediate_output_path, - ), + external_asset_entrypoints( + get_intermediate_module(self.context, self.entry_asset), + self.runtime_entries, + self.chunking_context, self.intermediate_output_path, ) .await? - .external_asset_entrypoints .iter() .map(|a| { ServerRenderedClientAssetReference { asset: *a } @@ -152,202 +130,20 @@ impl ValueToString for ServerRenderedClientAssetReference { } #[turbo_tasks::function] -async fn get_intermediate_asset( +fn get_intermediate_module( context: AssetContextVc, entry_asset: AssetVc, - runtime_entries: EcmascriptChunkPlaceablesVc, - chunking_context: DevChunkingContextVc, - intermediate_output_path: FileSystemPathVc, -) -> Result { - let server_renderer = embed_file!("server_renderer.js").into(); - - let module = EcmascriptModuleAssetVc::new( - WrapperAssetVc::new(entry_asset, "server-renderer.js", server_renderer).into(), +) -> EcmascriptModuleAssetVc { + EcmascriptModuleAssetVc::new( + WrapperAssetVc::new( + entry_asset, + "server-renderer.js", + embed_file!("server_renderer.js").into(), + ) + .into(), context.with_context_path(entry_asset.path()), Value::new(ModuleAssetType::Ecmascript), EcmascriptInputTransformsVc::cell(vec![EcmascriptInputTransform::React { refresh: false }]), context.environment(), - ); - - let chunk = module.as_evaluated_chunk(chunking_context.into(), Some(runtime_entries)); - - let chunk_group = ChunkGroupVc::from_chunk(chunk); - Ok(NodeJsBootstrapAsset { - path: intermediate_output_path.join("index.js"), - chunk_group, - } - .cell() - .into()) -} - -#[turbo_tasks::function] -async fn emit( - intermediate_asset: AssetVc, - intermediate_output_path: FileSystemPathVc, -) -> Result { - Ok(CompletionsVc::cell( - separate_assets(intermediate_asset, intermediate_output_path) - .await? - .internal_assets - .iter() - .map(|a| a.content().write(a.path())) - .collect(), ) - .all()) -} - -#[turbo_tasks::value] -struct SeparatedAssets { - internal_assets: HashSet, - external_asset_entrypoints: HashSet, -} - -#[turbo_tasks::function] -async fn separate_assets( - intermediate_asset: AssetVc, - intermediate_output_path: FileSystemPathVc, -) -> Result { - enum Type { - Internal(AssetVc, Vec), - External(AssetVc), - } - let intermediate_output_path = intermediate_output_path.await?; - let mut queue = FuturesUnordered::new(); - let process_asset = |asset: AssetVc| { - let intermediate_output_path = &intermediate_output_path; - async move { - if asset.path().await?.is_inside(intermediate_output_path) { - let mut assets = Vec::new(); - for reference in asset.references().await?.iter() { - for asset in reference.resolve_reference().primary_assets().await?.iter() { - assets.push(*asset); - } - } - Ok::<_, anyhow::Error>(Type::Internal(asset, assets)) - } else { - Ok(Type::External(asset)) - } - } - }; - queue.push(process_asset(intermediate_asset)); - let mut processed = HashSet::new(); - let mut internal_assets = HashSet::new(); - let mut external_asset_entrypoints = HashSet::new(); - while let Some(item) = queue.try_next().await? { - match item { - Type::Internal(asset, assets) => { - internal_assets.insert(asset); - for asset in assets { - if processed.insert(asset) { - queue.push(process_asset(asset)); - } - } - } - Type::External(asset) => { - // external - external_asset_entrypoints.insert(asset); - } - } - } - Ok(SeparatedAssets { - internal_assets, - external_asset_entrypoints, - } - .cell()) -} - -#[turbo_tasks::function] -async fn get_renderer_pool( - intermediate_asset: AssetVc, - intermediate_output_path: FileSystemPathVc, -) -> Result { - emit(intermediate_asset, intermediate_output_path).await?; - let output = intermediate_output_path.await?; - if let Some(disk) = DiskFileSystemVc::resolve_from(output.fs).await? { - let dir = PathBuf::from(&disk.await?.root).join(&output.path); - let entrypoint = dir.join("index.js"); - let pool = NodeJsPool::new(dir, entrypoint, HashMap::new(), 4); - Ok(pool.cell()) - } else { - Err(anyhow!("can only render from a disk filesystem")) - } -} - -#[turbo_tasks::function] -async fn render( - path: FileSystemPathVc, - renderer_pool: NodeJsPoolVc, - request_data: &str, -) -> Result { - fn into_result(content: String) -> Result { - Ok( - FileContent::Content(File::from_source(content).with_content_type(TEXT_HTML_UTF_8)) - .into(), - ) - } - let pool = renderer_pool.await?; - let mut op = pool.run(request_data.as_bytes()).await?; - let lines = spawn_blocking(move || { - let lines = op.read_lines()?; - drop(op); - Ok::<_, anyhow::Error>(lines) - }) - .await?; - let issue = if let Some(last_line) = lines.last() { - if let Some(data) = last_line.strip_prefix("RESULT=") { - let data: JsonValue = serde_json::from_str(data)?; - if let Some(s) = data.as_str() { - return into_result(s.to_string()); - } else { - RenderingIssue { - context: path, - message: StringVc::cell( - "Result provided by Node.js rendering process was not a string".to_string(), - ), - logging: StringVc::cell(lines.join("\n")), - } - } - } else if let Some(data) = last_line.strip_prefix("ERROR=") { - let data: JsonValue = serde_json::from_str(data)?; - if let Some(s) = data.as_str() { - RenderingIssue { - context: path, - message: StringVc::cell(s.to_string()), - logging: StringVc::cell(lines[..lines.len() - 1].join("\n")), - } - } else { - RenderingIssue { - context: path, - message: StringVc::cell(data.to_string()), - logging: StringVc::cell(lines[..lines.len() - 1].join("\n")), - } - } - } else { - RenderingIssue { - context: path, - message: StringVc::cell("No result provided by Node.js process".to_string()), - logging: StringVc::cell(lines.join("\n")), - } - } - } else { - RenderingIssue { - context: path, - message: StringVc::cell("No content received from Node.js process.".to_string()), - logging: StringVc::cell("".to_string()), - } - }; - - // Show error page - // TODO This need to include HMR handler to allow auto refresh - let result = into_result(format!( - "

Error during \ - rendering

\n

Message

\n
{}
\n

Logs

\n
{}
", - issue.message.await?, - issue.logging.await? - )); - - // Emit an issue for error reporting - issue.cell().as_issue().emit(); - - result } diff --git a/packages/next-swc/crates/next-core/src/server_render/mod.rs b/packages/next-swc/crates/next-core/src/server_render/mod.rs index d0d15de63b..64cdb62cc3 100644 --- a/packages/next-swc/crates/next-core/src/server_render/mod.rs +++ b/packages/next-swc/crates/next-core/src/server_render/mod.rs @@ -1,4 +1 @@ pub mod asset; -pub(crate) mod issue; -pub mod nodejs_bootstrap; -pub mod nodejs_pool; diff --git a/packages/next-swc/crates/next-core/src/server_rendered_source.rs b/packages/next-swc/crates/next-core/src/server_rendered_source.rs index 65e7fa200b..e2a59e2b8a 100644 --- a/packages/next-swc/crates/next-core/src/server_rendered_source.rs +++ b/packages/next-swc/crates/next-core/src/server_rendered_source.rs @@ -1,7 +1,11 @@ use std::collections::HashMap; use anyhow::Result; -use turbo_tasks::{primitives::StringVc, Value}; +use serde_json::json; +use turbo_tasks::{ + primitives::{JsonValueVc, StringVc}, + Value, +}; use turbo_tasks_fs::{DirectoryContent, DirectoryEntry, FileSystemEntryType, FileSystemPathVc}; use turbopack::{ module_options::ModuleOptionsContext, resolve_options_context::ResolveOptionsContext, @@ -37,7 +41,7 @@ use crate::{ }; /// Create a content source serving the `pages` or `src/pages` directory as -/// Node.js pages folder. +/// Next.js pages folder. #[turbo_tasks::function] pub async fn create_server_rendered_source( root_path: FileSystemPathVc, @@ -200,7 +204,7 @@ async fn create_server_rendered_source_for_file( runtime_entries, chunking_context, intermediate_output_path, - "{\"props\":{}}\n".to_string(), + JsonValueVc::cell(json!({ "props": {} })), ); Ok(AssetGraphContentSourceVc::new_lazy( target_root,