archive/rsnext
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactor nodejs rendering to be more isolated and reusable (vercel/turbo#428)

Browse source 
(Prerequirement for `app` support)
This commit is contained in:
Tobias Koppers 2022-09-30 10:13:37 +02:00 committed by GitHub
parent 9fd9163096
commit 797bbc1a2f
8 changed files with 315 additions and 239 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
packages/next-swc/crates/next-core/src/lib.rs
View file

@ -3,6 +3,7 @@
pub mod env;
pub mod next_client;
mod nodejs;
pub mod react_refresh;
mod server_render;
mod server_rendered_source;

2
packages/next-swc/crates/next-core/src/server_render/nodejs_bootstrap.rs → packages/next-swc/crates/next-core/src/nodejs/bootstrap.rs
View file

@ -28,7 +28,7 @@ impl Asset for NodeJsBootstrapAsset {
// TODO(sokra) We need to have a chunk format for node.js
// but until then this is a simple hack to make it work for now
let mut output = "global.self = global;\n".to_string();
let mut output = "Error.stackTraceLimit = 100;\nglobal.self = global;\n".to_string();
for chunk in self.chunk_group.chunks().await?.iter() {
let path = &*chunk.path().await?;

0
packages/next-swc/crates/next-core/src/server_render/issue.rs → packages/next-swc/crates/next-core/src/nodejs/issue.rs
View file

277
packages/next-swc/crates/next-core/src/nodejs/mod.rs Normal file
View file

@ -0,0 +1,277 @@
use std::{
collections::{HashMap, HashSet},
path::PathBuf,
};
use anyhow::{anyhow, Result};
use futures::{stream::FuturesUnordered, TryStreamExt};
use mime::TEXT_HTML_UTF_8;
use serde_json::Value as JsonValue;
use turbo_tasks::{
primitives::{JsonValueVc, StringVc},
spawn_blocking, CompletionVc, CompletionsVc, ValueToString,
};
use turbo_tasks_fs::{DiskFileSystemVc, File, FileContent, FileSystemPathVc};
use turbopack::ecmascript::EcmascriptModuleAssetVc;
use turbopack_core::{
asset::{AssetContentVc, AssetVc, AssetsSetVc},
chunk::{dev::DevChunkingContextVc, ChunkGroupVc},
};
use turbopack_ecmascript::chunk::EcmascriptChunkPlaceablesVc;
use self::{
bootstrap::NodeJsBootstrapAsset,
pool::{NodeJsPool, NodeJsPoolVc},
};
use crate::nodejs::issue::RenderingIssue;
pub mod bootstrap;
pub(crate) mod issue;
pub mod pool;
#[turbo_tasks::function]
async fn emit(
intermediate_asset: AssetVc,
intermediate_output_path: FileSystemPathVc,
) -> Result<CompletionVc> {
Ok(CompletionsVc::cell(
internal_assets(intermediate_asset, intermediate_output_path)
.await?
.iter()
.map(|a| a.content().write(a.path()))
.collect(),
)
.all())
}
/// List of the all assets of the "internal" subgraph and a list of boundary
/// assets that are not considered "internal" ("external")
#[turbo_tasks::value]
pub struct SeparatedAssets {
pub internal_assets: AssetsSetVc,
pub external_asset_entrypoints: AssetsSetVc,
}
/// Extracts the subgraph of "internal" assets (assets within the passes
/// directory). Also lists all boundary assets that are not part of the
/// "internal" subgraph.
#[turbo_tasks::function]
async fn internal_assets(
intermediate_asset: AssetVc,
intermediate_output_path: FileSystemPathVc,
) -> Result<AssetsSetVc> {
Ok(
separate_assets(intermediate_asset, intermediate_output_path)
.await?
.internal_assets,
)
}
/// Returns a set of "external" assets on the boundary of the "internal"
/// subgraph
#[turbo_tasks::function]
pub async fn external_asset_entrypoints(
module: EcmascriptModuleAssetVc,
runtime_entries: EcmascriptChunkPlaceablesVc,
chunking_context: DevChunkingContextVc,
intermediate_output_path: FileSystemPathVc,
) -> Result<AssetsSetVc> {
Ok(separate_assets(
get_intermediate_asset(
module,
runtime_entries,
chunking_context,
intermediate_output_path,
),
intermediate_output_path,
)
.await?
.external_asset_entrypoints)
}
/// Splits the asset graph into "internal" assets and boundaries to "external"
/// assets.
#[turbo_tasks::function]
async fn separate_assets(
intermediate_asset: AssetVc,
intermediate_output_path: FileSystemPathVc,
) -> Result<SeparatedAssetsVc> {
enum Type {
Internal(AssetVc, Vec<AssetVc>),
External(AssetVc),
}
let intermediate_output_path = intermediate_output_path.await?;
let mut queue = FuturesUnordered::new();
let process_asset = |asset: AssetVc| {
let intermediate_output_path = &intermediate_output_path;
async move {
// Assets within the output directory are considered as "internal" and all
// others as "external". We follow references on "internal" assets, but do not
// look into references of "external" assets, since there are no "internal"
// assets behind "externals"
if asset.path().await?.is_inside(intermediate_output_path) {
let mut assets = Vec::new();
for reference in asset.references().await?.iter() {
for asset in reference.resolve_reference().primary_assets().await?.iter() {
assets.push(*asset);
}
}
Ok::<_, anyhow::Error>(Type::Internal(asset, assets))
} else {
Ok(Type::External(asset))
}
}
};
queue.push(process_asset(intermediate_asset));
let mut processed = HashSet::new();
let mut internal_assets = HashSet::new();
let mut external_asset_entrypoints = HashSet::new();
while let Some(item) = queue.try_next().await? {
match item {
Type::Internal(asset, assets) => {
internal_assets.insert(asset);
for asset in assets {
if processed.insert(asset) {
queue.push(process_asset(asset));
}
}
}
Type::External(asset) => {
external_asset_entrypoints.insert(asset);
}
}
}
Ok(SeparatedAssets {
internal_assets: AssetsSetVc::cell(internal_assets),
external_asset_entrypoints: AssetsSetVc::cell(external_asset_entrypoints),
}
.cell())
}
/// Creates a node.js renderer pool for an entrypoint.
#[turbo_tasks::function]
pub async fn get_renderer_pool(
intermediate_asset: AssetVc,
intermediate_output_path: FileSystemPathVc,
) -> Result<NodeJsPoolVc> {
emit(intermediate_asset, intermediate_output_path).await?;
let output = intermediate_output_path.await?;
if let Some(disk) = DiskFileSystemVc::resolve_from(output.fs).await? {
let dir = PathBuf::from(&disk.await?.root).join(&output.path);
let entrypoint = dir.join("index.js");
let pool = NodeJsPool::new(dir, entrypoint, HashMap::new(), 4);
Ok(pool.cell())
} else {
Err(anyhow!("can only render from a disk filesystem"))
}
}
/// Converts a module graph into node.js executable assets
#[turbo_tasks::function]
async fn get_intermediate_asset(
entry_module: EcmascriptModuleAssetVc,
runtime_entries: EcmascriptChunkPlaceablesVc,
chunking_context: DevChunkingContextVc,
intermediate_output_path: FileSystemPathVc,
) -> Result<AssetVc> {
let chunk = entry_module.as_evaluated_chunk(chunking_context.into(), Some(runtime_entries));
let chunk_group = ChunkGroupVc::from_chunk(chunk);
Ok(NodeJsBootstrapAsset {
path: intermediate_output_path.join("index.js"),
chunk_group,
}
.cell()
.into())
}
/// Renders a module as static HTML in a node.js process.
#[turbo_tasks::function]
pub async fn render_static(
path: FileSystemPathVc,
module: EcmascriptModuleAssetVc,
runtime_entries: EcmascriptChunkPlaceablesVc,
chunking_context: DevChunkingContextVc,
intermediate_output_path: FileSystemPathVc,
data: JsonValueVc,
) -> Result<AssetContentVc> {
fn into_result(content: String) -> Result<AssetContentVc> {
Ok(
FileContent::Content(File::from_source(content).with_content_type(TEXT_HTML_UTF_8))
.into(),
)
}
let renderer_pool = get_renderer_pool(
get_intermediate_asset(
module,
runtime_entries,
chunking_context,
intermediate_output_path,
),
intermediate_output_path,
);
let pool = renderer_pool.await?;
let mut op = pool.run(data.to_string().await?.as_bytes()).await?;
let lines = spawn_blocking(move || {
let lines = op.read_lines()?;
drop(op);
Ok::<_, anyhow::Error>(lines)
})
.await?;
let issue = if let Some(last_line) = lines.last() {
if let Some(data) = last_line.strip_prefix("RESULT=") {
let data: JsonValue = serde_json::from_str(data)?;
if let Some(s) = data.as_str() {
return into_result(s.to_string());
} else {
RenderingIssue {
context: path,
message: StringVc::cell(
"Result provided by Node.js rendering process was not a string".to_string(),
),
logging: StringVc::cell(lines.join("\n")),
}
}
} else if let Some(data) = last_line.strip_prefix("ERROR=") {
let data: JsonValue = serde_json::from_str(data)?;
if let Some(s) = data.as_str() {
RenderingIssue {
context: path,
message: StringVc::cell(s.to_string()),
logging: StringVc::cell(lines[..lines.len() - 1].join("\n")),
}
} else {
RenderingIssue {
context: path,
message: StringVc::cell(data.to_string()),
logging: StringVc::cell(lines[..lines.len() - 1].join("\n")),
}
}
} else {
RenderingIssue {
context: path,
message: StringVc::cell("No result provided by Node.js process".to_string()),
logging: StringVc::cell(lines.join("\n")),
}
}
} else {
RenderingIssue {
context: path,
message: StringVc::cell("No content received from Node.js process.".to_string()),
logging: StringVc::cell("".to_string()),
}
};
// Show error page
// TODO This need to include HMR handler to allow auto refresh
let result = into_result(format!(
"<h1>Error during \
rendering</h1>\n<h2>Message</h2>\n<pre>{}</pre>\n<h2>Logs</h2>\n<pre>{}</pre>",
issue.message.await?,
issue.logging.await?
));
// Emit an issue for error reporting
issue.cell().as_issue().emit();
result
}

1
packages/next-swc/crates/next-core/src/server_render/nodejs_pool.rs → packages/next-swc/crates/next-core/src/nodejs/pool.rs
View file

@ -136,6 +136,7 @@ impl NodeJsPool {
let static_input: &'static [u8] = unsafe { transmute(input) };
let child = spawn_blocking(move || {
child.write(static_input)?;
child.write(b"\n")?;
Ok::<_, anyhow::Error>(child)
})
.await?;

260
packages/next-swc/crates/next-core/src/server_render/asset.rs
View file

@ -1,23 +1,16 @@
use std::{
collections::{HashMap, HashSet},
path::PathBuf,
};
use anyhow::{anyhow, Result};
use futures::{stream::FuturesUnordered, TryStreamExt};
use mime::TEXT_HTML_UTF_8;
use serde_json::Value as JsonValue;
use anyhow::Result;
use turbo_tasks::{
primitives::StringVc, spawn_blocking, CompletionVc, CompletionsVc, Value, ValueToString,
ValueToStringVc,
primitives::{JsonValueVc, StringVc},
Value, ValueToString, ValueToStringVc,
};
use turbo_tasks_fs::{embed_file, DiskFileSystemVc, File, FileContent, FileSystemPathVc};
use turbo_tasks_fs::{embed_file, FileSystemPathVc};
use turbopack::ecmascript::{
EcmascriptInputTransform, EcmascriptInputTransformsVc, EcmascriptModuleAssetVc, ModuleAssetType,
};
use turbopack_core::{
self,
asset::{Asset, AssetContentVc, AssetVc},
chunk::{dev::DevChunkingContextVc, ChunkGroupVc},
chunk::dev::DevChunkingContextVc,
context::AssetContextVc,
reference::{AssetReference, AssetReferenceVc, AssetReferencesVc},
resolve::{ResolveResult, ResolveResultVc},
@ -25,11 +18,7 @@ use turbopack_core::{
};
use turbopack_ecmascript::chunk::EcmascriptChunkPlaceablesVc;
use super::{
nodejs_bootstrap::NodeJsBootstrapAsset,
nodejs_pool::{NodeJsPool, NodeJsPoolVc},
};
use crate::server_render::issue::RenderingIssue;
use crate::nodejs::{external_asset_entrypoints, render_static};
/// This is an asset which content is determined by running
/// `React.renderToString` on the default export of [entry_asset] in a Node.js
@ -48,7 +37,7 @@ pub struct ServerRenderedAsset {
chunking_context: DevChunkingContextVc,
runtime_entries: EcmascriptChunkPlaceablesVc,
intermediate_output_path: FileSystemPathVc,
request_data: String,
request_data: JsonValueVc,
}
#[turbo_tasks::value_impl]
@ -61,7 +50,7 @@ impl ServerRenderedAssetVc {
runtime_entries: EcmascriptChunkPlaceablesVc,
chunking_context: DevChunkingContextVc,
intermediate_output_path: FileSystemPathVc,
request_data: String,
request_data: JsonValueVc,
) -> Self {
ServerRenderedAsset {
path,
@ -85,37 +74,26 @@ impl Asset for ServerRenderedAsset {
#[turbo_tasks::function]
fn content(&self) -> AssetContentVc {
render(
render_static(
self.path,
get_renderer_pool(
get_intermediate_asset(
self.context,
self.entry_asset,
self.runtime_entries,
self.chunking_context,
self.intermediate_output_path,
),
self.intermediate_output_path,
),
&self.request_data,
get_intermediate_module(self.context, self.entry_asset),
self.runtime_entries,
self.chunking_context,
self.intermediate_output_path,
self.request_data,
)
}
#[turbo_tasks::function]
async fn references(&self) -> Result<AssetReferencesVc> {
Ok(AssetReferencesVc::cell(
separate_assets(
get_intermediate_asset(
self.context,
self.entry_asset,
self.runtime_entries,
self.chunking_context,
self.intermediate_output_path,
),
external_asset_entrypoints(
get_intermediate_module(self.context, self.entry_asset),
self.runtime_entries,
self.chunking_context,
self.intermediate_output_path,
)
.await?
.external_asset_entrypoints
.iter()
.map(|a| {
ServerRenderedClientAssetReference { asset: *a }
@ -152,202 +130,20 @@ impl ValueToString for ServerRenderedClientAssetReference {
}
#[turbo_tasks::function]
async fn get_intermediate_asset(
fn get_intermediate_module(
context: AssetContextVc,
entry_asset: AssetVc,
runtime_entries: EcmascriptChunkPlaceablesVc,
chunking_context: DevChunkingContextVc,
intermediate_output_path: FileSystemPathVc,
) -> Result<AssetVc> {
let server_renderer = embed_file!("server_renderer.js").into();
let module = EcmascriptModuleAssetVc::new(
WrapperAssetVc::new(entry_asset, "server-renderer.js", server_renderer).into(),
) -> EcmascriptModuleAssetVc {
EcmascriptModuleAssetVc::new(
WrapperAssetVc::new(
entry_asset,
"server-renderer.js",
embed_file!("server_renderer.js").into(),
)
.into(),
context.with_context_path(entry_asset.path()),
Value::new(ModuleAssetType::Ecmascript),
EcmascriptInputTransformsVc::cell(vec![EcmascriptInputTransform::React { refresh: false }]),
context.environment(),
);
let chunk = module.as_evaluated_chunk(chunking_context.into(), Some(runtime_entries));
let chunk_group = ChunkGroupVc::from_chunk(chunk);
Ok(NodeJsBootstrapAsset {
path: intermediate_output_path.join("index.js"),
chunk_group,
}
.cell()
.into())
}
#[turbo_tasks::function]
async fn emit(
intermediate_asset: AssetVc,
intermediate_output_path: FileSystemPathVc,
) -> Result<CompletionVc> {
Ok(CompletionsVc::cell(
separate_assets(intermediate_asset, intermediate_output_path)
.await?
.internal_assets
.iter()
.map(|a| a.content().write(a.path()))
.collect(),
)
.all())
}
#[turbo_tasks::value]
struct SeparatedAssets {
internal_assets: HashSet<AssetVc>,
external_asset_entrypoints: HashSet<AssetVc>,
}
#[turbo_tasks::function]
async fn separate_assets(
intermediate_asset: AssetVc,
intermediate_output_path: FileSystemPathVc,
) -> Result<SeparatedAssetsVc> {
enum Type {
Internal(AssetVc, Vec<AssetVc>),
External(AssetVc),
}
let intermediate_output_path = intermediate_output_path.await?;
let mut queue = FuturesUnordered::new();
let process_asset = |asset: AssetVc| {
let intermediate_output_path = &intermediate_output_path;
async move {
if asset.path().await?.is_inside(intermediate_output_path) {
let mut assets = Vec::new();
for reference in asset.references().await?.iter() {
for asset in reference.resolve_reference().primary_assets().await?.iter() {
assets.push(*asset);
}
}
Ok::<_, anyhow::Error>(Type::Internal(asset, assets))
} else {
Ok(Type::External(asset))
}
}
};
queue.push(process_asset(intermediate_asset));
let mut processed = HashSet::new();
let mut internal_assets = HashSet::new();
let mut external_asset_entrypoints = HashSet::new();
while let Some(item) = queue.try_next().await? {
match item {
Type::Internal(asset, assets) => {
internal_assets.insert(asset);
for asset in assets {
if processed.insert(asset) {
queue.push(process_asset(asset));
}
}
}
Type::External(asset) => {
// external
external_asset_entrypoints.insert(asset);
}
}
}
Ok(SeparatedAssets {
internal_assets,
external_asset_entrypoints,
}
.cell())
}
#[turbo_tasks::function]
async fn get_renderer_pool(
intermediate_asset: AssetVc,
intermediate_output_path: FileSystemPathVc,
) -> Result<NodeJsPoolVc> {
emit(intermediate_asset, intermediate_output_path).await?;
let output = intermediate_output_path.await?;
if let Some(disk) = DiskFileSystemVc::resolve_from(output.fs).await? {
let dir = PathBuf::from(&disk.await?.root).join(&output.path);
let entrypoint = dir.join("index.js");
let pool = NodeJsPool::new(dir, entrypoint, HashMap::new(), 4);
Ok(pool.cell())
} else {
Err(anyhow!("can only render from a disk filesystem"))
}
}
#[turbo_tasks::function]
async fn render(
path: FileSystemPathVc,
renderer_pool: NodeJsPoolVc,
request_data: &str,
) -> Result<AssetContentVc> {
fn into_result(content: String) -> Result<AssetContentVc> {
Ok(
FileContent::Content(File::from_source(content).with_content_type(TEXT_HTML_UTF_8))
.into(),
)
}
let pool = renderer_pool.await?;
let mut op = pool.run(request_data.as_bytes()).await?;
let lines = spawn_blocking(move || {
let lines = op.read_lines()?;
drop(op);
Ok::<_, anyhow::Error>(lines)
})
.await?;
let issue = if let Some(last_line) = lines.last() {
if let Some(data) = last_line.strip_prefix("RESULT=") {
let data: JsonValue = serde_json::from_str(data)?;
if let Some(s) = data.as_str() {
return into_result(s.to_string());
} else {
RenderingIssue {
context: path,
message: StringVc::cell(
"Result provided by Node.js rendering process was not a string".to_string(),
),
logging: StringVc::cell(lines.join("\n")),
}
}
} else if let Some(data) = last_line.strip_prefix("ERROR=") {
let data: JsonValue = serde_json::from_str(data)?;
if let Some(s) = data.as_str() {
RenderingIssue {
context: path,
message: StringVc::cell(s.to_string()),
logging: StringVc::cell(lines[..lines.len() - 1].join("\n")),
}
} else {
RenderingIssue {
context: path,
message: StringVc::cell(data.to_string()),
logging: StringVc::cell(lines[..lines.len() - 1].join("\n")),
}
}
} else {
RenderingIssue {
context: path,
message: StringVc::cell("No result provided by Node.js process".to_string()),
logging: StringVc::cell(lines.join("\n")),
}
}
} else {
RenderingIssue {
context: path,
message: StringVc::cell("No content received from Node.js process.".to_string()),
logging: StringVc::cell("".to_string()),
}
};
// Show error page
// TODO This need to include HMR handler to allow auto refresh
let result = into_result(format!(
"<h1>Error during \
rendering</h1>\n<h2>Message</h2>\n<pre>{}</pre>\n<h2>Logs</h2>\n<pre>{}</pre>",
issue.message.await?,
issue.logging.await?
));
// Emit an issue for error reporting
issue.cell().as_issue().emit();
result
}

3
packages/next-swc/crates/next-core/src/server_render/mod.rs
View file

@ -1,4 +1 @@
pub mod asset;
pub(crate) mod issue;
pub mod nodejs_bootstrap;
pub mod nodejs_pool;

10
packages/next-swc/crates/next-core/src/server_rendered_source.rs
View file

@ -1,7 +1,11 @@
use std::collections::HashMap;
use anyhow::Result;
use turbo_tasks::{primitives::StringVc, Value};
use serde_json::json;
use turbo_tasks::{
primitives::{JsonValueVc, StringVc},
Value,
};
use turbo_tasks_fs::{DirectoryContent, DirectoryEntry, FileSystemEntryType, FileSystemPathVc};
use turbopack::{
module_options::ModuleOptionsContext, resolve_options_context::ResolveOptionsContext,
@ -37,7 +41,7 @@ use crate::{
};
/// Create a content source serving the `pages` or `src/pages` directory as
/// Node.js pages folder.
/// Next.js pages folder.
#[turbo_tasks::function]
pub async fn create_server_rendered_source(
root_path: FileSystemPathVc,
@ -200,7 +204,7 @@ async fn create_server_rendered_source_for_file(
runtime_entries,
chunking_context,
intermediate_output_path,
"{\"props\":{}}\n".to_string(),
JsonValueVc::cell(json!({ "props": {} })),
);
Ok(AssetGraphContentSourceVc::new_lazy(
target_root,