archive/rsnext
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(turbopack): port next.js template loading logic (#56425)

Browse source 
Closes WEB-1706
This commit is contained in:
Leah 2023-10-05 20:38:46 +02:00 committed by GitHub
parent a44b4f85b5
commit 11c1d07b89
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 312 additions and 179 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
packages/next-swc/crates/next-core/src/middleware.rs
View file

@ -1,13 +1,12 @@
use anyhow::Result;
use indexmap::indexmap;
use turbo_tasks::{Value, Vc};
use turbo_tasks_fs::{File, FileSystemPath};
use turbo_tasks_fs::FileSystemPath;
use turbopack_binding::turbopack::core::{
asset::AssetContent, context::AssetContext, module::Module, reference_type::ReferenceType,
virtual_source::VirtualSource,
context::AssetContext, module::Module, reference_type::ReferenceType,
};
use crate::util::{load_next_js_template, virtual_next_js_template_path};
use crate::util::load_next_js_template;
#[turbo_tasks::function]
pub async fn middleware_files(page_extensions: Vc<Vec<String>>) -> Result<Vc<Vec<String>>> {
@ -29,23 +28,25 @@ pub async fn get_middleware_module(
project_root: Vc<FileSystemPath>,
userland_module: Vc<Box<dyn Module>>,
) -> Result<Vc<Box<dyn Module>>> {
let template_file = "middleware.js";
const INNER: &str = "INNER_MIDDLEWARE_MODULE";
// Load the file from the next.js codebase.
let file = load_next_js_template(project_root, template_file.to_string()).await?;
let file = File::from(file.clone_value());
let template_path = virtual_next_js_template_path(project_root, template_file.to_string());
let virtual_source = VirtualSource::new(template_path, AssetContent::file(file.into()));
let source = load_next_js_template(
"middleware.js",
project_root,
indexmap! {
"VAR_USERLAND" => INNER.to_string(),
},
indexmap! {},
)
.await?;
let inner_assets = indexmap! {
"VAR_USERLAND".to_string() => userland_module
INNER.to_string() => userland_module
};
let module = context.process(
Vc::upcast(virtual_source),
source,
Value::new(ReferenceType::Internal(Vc::cell(inner_assets))),
);

79
packages/next-swc/crates/next-core/src/next_app/app_page_entry.rs
View file

@ -1,12 +1,16 @@
use std::io::Write;
use anyhow::{bail, Result};
use indexmap::indexmap;
use turbo_tasks::{TryJoinIterExt, Value, ValueToString, Vc};
use turbopack_binding::{
turbo::tasks_fs::{rope::RopeBuilder, File, FileSystemPath},
turbopack::{
core::{
asset::AssetContent, context::AssetContext, reference_type::ReferenceType,
asset::{Asset, AssetContent},
context::AssetContext,
reference_type::ReferenceType,
source::Source,
virtual_source::VirtualSource,
},
ecmascript::{chunk::EcmascriptChunkPlaceable, utils::StringifyJs},
@ -22,7 +26,7 @@ use crate::{
next_app::{AppPage, AppPath},
next_server_component::NextServerComponentTransition,
parse_segment_config_from_loader_tree,
util::{load_next_js_template, virtual_next_js_template_path, NextRuntime},
util::{file_content_rope, load_next_js_template, NextRuntime},
};
/// Computes the entry for a Next.js app page.
@ -70,59 +74,32 @@ pub async fn get_app_page_entry(
let original_name = page.to_string();
let pathname = AppPath::from(page.clone()).to_string();
let template_file = "app-page.js";
// Load the file from the next.js codebase.
let file = load_next_js_template(project_root, template_file.to_string()).await?;
let source = load_next_js_template(
"app-page.js",
project_root,
indexmap! {
"VAR_DEFINITION_PAGE" => page.to_string(),
"VAR_DEFINITION_PATHNAME" => pathname.clone(),
"VAR_ORIGINAL_PATHNAME" => original_name.clone(),
// TODO(alexkirsz) Support custom global error.
"VAR_MODULE_GLOBAL_ERROR" => "next/dist/client/components/error-boundary".to_string(),
},
indexmap! {
"tree" => loader_tree_code,
"pages" => StringifyJs(&pages).to_string(),
"__next_app_require__" => "__turbopack_require__".to_string(),
"__next_app_load_chunk__" => " __turbopack_load__".to_string(),
},
)
.await?;
let mut file = file
.to_str()?
.replace(
"\"VAR_DEFINITION_PAGE\"",
&StringifyJs(&page.to_string()).to_string(),
)
.replace(
"\"VAR_DEFINITION_PATHNAME\"",
&StringifyJs(&pathname).to_string(),
)
.replace(
"\"VAR_ORIGINAL_PATHNAME\"",
&StringifyJs(&original_name).to_string(),
)
// TODO(alexkirsz) Support custom global error.
.replace(
"\"VAR_MODULE_GLOBAL_ERROR\"",
&StringifyJs("next/dist/client/components/error-boundary").to_string(),
)
.replace(
"// INJECT:tree",
format!("const tree = {};", loader_tree_code).as_str(),
)
.replace(
"// INJECT:pages",
format!("const pages = {};", StringifyJs(&pages)).as_str(),
)
.replace(
"// INJECT:__next_app_require__",
"const __next_app_require__ = __turbopack_require__",
)
.replace(
"// INJECT:__next_app_load_chunk__",
"const __next_app_load_chunk__ = __turbopack_load__",
);
let source_content = &*file_content_rope(source.content().file_content()).await?;
// Ensure that the last line is a newline.
if !file.ends_with('\n') {
file.push('\n');
}
result.push_bytes(file.as_bytes());
result.concat(source_content);
let file = File::from(result.build());
let template_path = virtual_next_js_template_path(project_root, template_file.to_string());
let source = VirtualSource::new(template_path, AssetContent::file(file.into()));
let source = VirtualSource::new(source.ident().path(), AssetContent::file(file.into()));
let rsc_entry = context.process(
Vc::upcast(source),
@ -140,7 +117,7 @@ pub async fn get_app_page_entry(
};
Ok(AppEntry {
pathname: pathname.to_string(),
pathname,
original_name,
rsc_entry,
config,

72
packages/next-swc/crates/next-core/src/next_app/app_route_entry.rs
View file

@ -23,7 +23,7 @@ use turbopack_binding::{
use crate::{
next_app::{AppEntry, AppPage, AppPath},
parse_segment_config_from_source,
util::{load_next_js_template, virtual_next_js_template_path, NextRuntime},
util::{load_next_js_template, NextRuntime},
};
/// Computes the entry for a Next.js app route.
@ -49,62 +49,32 @@ pub async fn get_app_route_entry(
nodejs_context
};
let mut result = RopeBuilder::default();
let original_name = page.to_string();
let pathname = AppPath::from(page.clone()).to_string();
let path = source.ident().path();
let template_file = "app-route.js";
const INNER: &str = "INNER_APP_ROUTE";
// Load the file from the next.js codebase.
let file = load_next_js_template(project_root, template_file.to_string()).await?;
let mut file = file
.to_str()?
.replace(
"\"VAR_DEFINITION_PAGE\"",
&StringifyJs(&original_name).to_string(),
)
.replace(
"\"VAR_DEFINITION_PATHNAME\"",
&StringifyJs(&pathname).to_string(),
)
.replace(
"\"VAR_DEFINITION_FILENAME\"",
&StringifyJs(&path.file_stem().await?.as_ref().unwrap().clone()).to_string(),
)
// TODO(alexkirsz) Is this necessary?
.replace(
"\"VAR_DEFINITION_BUNDLE_PATH\"",
&StringifyJs("").to_string(),
)
.replace(
"\"VAR_ORIGINAL_PATHNAME\"",
&StringifyJs(&original_name).to_string(),
)
.replace(
"\"VAR_RESOLVED_PAGE_PATH\"",
&StringifyJs(&path.to_string().await?).to_string(),
)
.replace(
"// INJECT:nextConfigOutput",
"const nextConfigOutput = \"\"",
);
// Ensure that the last line is a newline.
if !file.ends_with('\n') {
file.push('\n');
}
result.push_bytes(file.as_bytes());
let file = File::from(result.build());
let template_path = virtual_next_js_template_path(project_root, template_file.to_string());
let virtual_source = VirtualSource::new(template_path, AssetContent::file(file.into()));
let virtual_source = load_next_js_template(
"app-route.js",
project_root,
indexmap! {
"VAR_DEFINITION_PAGE" => page.to_string(),
"VAR_DEFINITION_PATHNAME" => pathname.clone(),
"VAR_DEFINITION_FILENAME" => path.file_stem().await?.as_ref().unwrap().clone(),
// TODO(alexkirsz) Is this necessary?
"VAR_DEFINITION_BUNDLE_PATH" => "".to_string(),
"VAR_ORIGINAL_PATHNAME" => original_name.clone(),
"VAR_RESOLVED_PAGE_PATH" => path.to_string().await?.clone_value(),
"VAR_USERLAND" => INNER.to_string(),
},
indexmap! {
"nextConfigOutput" => "\"\"".to_string(),
},
)
.await?;
let userland_module = context.process(
source,
@ -112,7 +82,7 @@ pub async fn get_app_route_entry(
);
let inner_assets = indexmap! {
"VAR_USERLAND".to_string() => userland_module
INNER.to_string() => userland_module
};
let mut rsc_entry = context.process(

103
packages/next-swc/crates/next-core/src/next_pages/page_entry.rs
View file

@ -11,19 +11,19 @@ use turbopack_binding::{
},
turbopack::{
core::{
asset::AssetContent,
asset::{Asset, AssetContent},
context::AssetContext,
reference_type::{EntryReferenceSubType, ReferenceType},
source::Source,
virtual_source::VirtualSource,
},
ecmascript::{chunk::EcmascriptChunkPlaceable, utils::StringifyJs},
ecmascript::chunk::EcmascriptChunkPlaceable,
},
};
use crate::{
next_edge::entry::wrap_edge_entry,
util::{load_next_js_template, virtual_next_js_template_path, NextRuntime},
util::{file_content_rope, load_next_js_template, NextRuntime},
};
#[turbo_tasks::function]
@ -36,8 +36,8 @@ pub async fn create_page_ssr_entry_module(
next_original_name: Vc<String>,
runtime: NextRuntime,
) -> Result<Vc<Box<dyn EcmascriptChunkPlaceable>>> {
let definition_page = next_original_name.await?;
let definition_pathname = pathname.await?;
let definition_page = &*next_original_name.await?;
let definition_pathname = &*pathname.await?;
let ssr_module = ssr_module_context.process(source, reference_type.clone());
@ -59,64 +59,57 @@ pub async fn create_page_ssr_entry_module(
_ => bail!("Invalid path type"),
};
// Load the file from the next.js codebase.
let file = load_next_js_template(project_root, template_file.to_string()).await?;
const INNER: &str = "INNER_PAGE";
let mut file = file
.to_str()?
.replace(
"\"VAR_DEFINITION_PAGE\"",
&StringifyJs(&definition_page).to_string(),
)
.replace(
"\"VAR_DEFINITION_PATHNAME\"",
&StringifyJs(&definition_pathname).to_string(),
let mut replacements = indexmap! {
"VAR_DEFINITION_PAGE" => definition_page.clone(),
"VAR_DEFINITION_PATHNAME" => definition_pathname.clone(),
"VAR_USERLAND" => INNER.to_string(),
};
if reference_type == ReferenceType::Entry(EntryReferenceSubType::Page) {
replacements.insert(
"VAR_MODULE_DOCUMENT",
"@vercel/turbopack-next/pages/_document".to_string(),
);
replacements.insert(
"VAR_MODULE_APP",
"@vercel/turbopack-next/pages/_app".to_string(),
);
if reference_type == ReferenceType::Entry(EntryReferenceSubType::Page) {
file = file
.replace(
"\"VAR_MODULE_DOCUMENT\"",
&StringifyJs("@vercel/turbopack-next/pages/_document").to_string(),
)
.replace(
"\"VAR_MODULE_APP\"",
&StringifyJs("@vercel/turbopack-next/pages/_app").to_string(),
);
}
// Ensure that the last line is a newline.
if !file.ends_with('\n') {
file.push('\n');
// Load the file from the next.js codebase.
let mut source =
load_next_js_template(template_file, project_root, replacements, indexmap! {}).await?;
// When we're building the instrumentation page (only when the
// instrumentation file conflicts with a page also labeled
// /instrumentation) hoist the `register` method.
if reference_type == ReferenceType::Entry(EntryReferenceSubType::Page)
&& (*definition_page == "/instrumentation" || *definition_page == "/src/instrumentation")
{
let file = &*file_content_rope(source.content().file_content()).await?;
let mut result = RopeBuilder::default();
result += file;
writeln!(
result,
r#"export const register = hoist(userland, "register")"#
)?;
let file = File::from(result.build());
source = Vc::upcast(VirtualSource::new(
source.ident().path(),
AssetContent::file(file.into()),
));
}
let mut result = RopeBuilder::default();
result.push_bytes(file.as_bytes());
if reference_type == ReferenceType::Entry(EntryReferenceSubType::Page) {
// When we're building the instrumentation page (only when the
// instrumentation file conflicts with a page also labeled
// /instrumentation) hoist the `register` method.
if definition_page.to_string() == "/instrumentation"
|| definition_page.to_string() == "/src/instrumentation"
{
writeln!(
result,
r#"export const register = hoist(userland, "register")"#
)?;
}
}
let file = File::from(result.build());
let template_path = virtual_next_js_template_path(project_root, template_file.to_string());
let source = VirtualSource::new(template_path, AssetContent::file(file.into()));
let mut ssr_module = ssr_module_context.process(
Vc::upcast(source),
source,
Value::new(ReferenceType::Internal(Vc::cell(indexmap! {
"VAR_USERLAND".to_string() => ssr_module,
INNER.to_string() => ssr_module,
}))),
);

208
packages/next-swc/crates/next-core/src/util.rs
View file

@ -1,21 +1,26 @@
use anyhow::{bail, Result};
use anyhow::{bail, Context, Result};
use indexmap::{IndexMap, IndexSet};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use serde_json::Value as JsonValue;
use swc_core::ecma::ast::Program;
use turbo_tasks::{trace::TraceRawVcs, TaskInput, ValueDefault, ValueToString, Vc};
use turbo_tasks_fs::rope::Rope;
use turbo_tasks_fs::{rope::Rope, util::join_path, File};
use turbopack_binding::{
turbo::tasks_fs::{json::parse_json_rope_with_source_context, FileContent, FileSystemPath},
turbopack::{
core::{
asset::AssetContent,
environment::{ServerAddr, ServerInfo},
ident::AssetIdent,
issue::{Issue, IssueExt, IssueSeverity},
module::Module,
source::Source,
virtual_source::VirtualSource,
},
ecmascript::{
analyzer::{JsValue, ObjectPart},
parse::ParseResult,
utils::StringifyJs,
EcmascriptModuleAsset,
},
turbopack::condition::ContextCondition,
@ -347,14 +352,202 @@ fn parse_config_from_js_value(module: Vc<Box<dyn Module>>, value: &JsValue) -> N
config
}
#[turbo_tasks::function]
/// Loads a next.js template, replaces `replacements` and `injections` and makes
/// sure there are none left over.
// TODO: should this be a turbo tasks function?
// #[turbo_tasks::function]
pub async fn load_next_js_template(
path: &str,
project_path: Vc<FileSystemPath>,
file: String,
) -> Result<Vc<Rope>> {
let file_path = virtual_next_js_template_path(project_path, file);
replacements: IndexMap<&'static str, String>,
injections: IndexMap<&'static str, String>,
) -> Result<Vc<Box<dyn Source>>> {
let path = virtual_next_js_template_path(project_path, path.to_string());
let content = &*file_path.read().await?;
let content = &*file_content_rope(path.read()).await?;
let content = content.to_str()?.to_string();
let parent_path = path.parent();
let parent_path_value = &*parent_path.await?;
let package_root = get_next_package(project_path).parent();
let package_root_value = &*package_root.await?;
/// See [regex::Regex::replace_all].
fn replace_all<E>(
re: &regex::Regex,
haystack: &str,
mut replacement: impl FnMut(&regex::Captures) -> Result<String, E>,
) -> Result<String, E> {
let mut new = String::with_capacity(haystack.len());
let mut last_match = 0;
for caps in re.captures_iter(haystack) {
let m = caps.get(0).unwrap();
new.push_str(&haystack[last_match..m.start()]);
new.push_str(&replacement(&caps)?);
last_match = m.end();
}
new.push_str(&haystack[last_match..]);
Ok(new)
}
// Update the relative imports to be absolute. This will update any relative
// imports to be relative to the root of the `next` package.
let regex = lazy_regex::regex!("(?:from \"(\\..*)\"|import \"(\\..*)\")");
let mut count = 0;
let mut content = replace_all(regex, &content, |caps| {
let from_request = caps.get(1).map_or("", |c| c.as_str());
let import_request = caps.get(2).map_or("", |c| c.as_str());
count += 1;
let is_from_request = !from_request.is_empty();
let imported = FileSystemPath {
fs: package_root_value.fs,
path: join_path(
&parent_path_value.path,
if is_from_request {
from_request
} else {
import_request
},
)
.context("path should not leave the fs")?,
};
let relative = package_root_value
.get_relative_path_to(&imported)
.context("path has to be relative to package root")?;
if !relative.starts_with("./next/") {
bail!(
"Invariant: Expected relative import to start with \"./next/\", found \"{}\"",
relative
)
}
let relative = relative
.strip_prefix("./")
.context("should be able to strip the prefix")?;
Ok(if is_from_request {
format!("from {}", StringifyJs(relative))
} else {
format!("import {}", StringifyJs(relative))
})
})
.context("replacing imports failed")?;
// Verify that at least one import was replaced. It's the case today where
// every template file has at least one import to update, so this ensures that
// we don't accidentally remove the import replacement code or use the wrong
// template file.
if count == 0 {
bail!("Invariant: Expected to replace at least one import")
}
// Replace all the template variables with the actual values. If a template
// variable is missing, throw an error.
let mut replaced = IndexSet::new();
for (key, replacement) in &replacements {
let full = format!("\"{}\"", key);
if content.contains(&full) {
replaced.insert(*key);
content = content.replace(&full, &StringifyJs(&replacement).to_string());
}
}
// Check to see if there's any remaining template variables.
let regex = lazy_regex::regex!("/VAR_[A-Z_]+");
let matches = regex
.find_iter(&content)
.map(|m| m.as_str().to_string())
.collect::<Vec<_>>();
if !matches.is_empty() {
bail!(
"Invariant: Expected to replace all template variables, found {}",
matches.join(", "),
)
}
// Check to see if any template variable was provided but not used.
if replaced.len() != replacements.len() {
// Find the difference between the provided replacements and the replaced
// template variables. This will let us notify the user of any template
// variables that were not used but were provided.
let difference = replacements
.keys()
.filter(|k| !replaced.contains(*k))
.cloned()
.collect::<Vec<_>>();
bail!(
"Invariant: Expected to replace all template variables, missing {} in template",
difference.join(", "),
)
}
// Replace the injections.
let mut injected = IndexSet::new();
for (key, injection) in &injections {
let full = format!("// INJECT:{}", key);
if content.contains(&full) {
// Track all the injections to ensure that we're not missing any.
injected.insert(*key);
content = content.replace(&full, &format!("const {} = {}", key, injection));
}
}
// Check to see if there's any remaining injections.
let regex = lazy_regex::regex!("// INJECT:[A-Za-z0-9_]+");
let matches = regex
.find_iter(&content)
.map(|m| m.as_str().to_string())
.collect::<Vec<_>>();
if !matches.is_empty() {
bail!(
"Invariant: Expected to inject all injections, found {}",
matches.join(", "),
)
}
// Check to see if any injection was provided but not used.
if injected.len() != injections.len() {
// Find the difference between the provided replacements and the replaced
// template variables. This will let us notify the user of any template
// variables that were not used but were provided.
let difference = injections
.keys()
.filter(|k| !injected.contains(*k))
.cloned()
.collect::<Vec<_>>();
bail!(
"Invariant: Expected to inject all injections, missing {} in template",
difference.join(", "),
)
}
// Ensure that the last line is a newline.
if !content.ends_with('\n') {
content.push('\n');
}
let file = File::from(content);
let source = VirtualSource::new(path, AssetContent::file(file.into()));
Ok(Vc::upcast(source))
}
#[turbo_tasks::function]
pub async fn file_content_rope(content: Vc<FileContent>) -> Result<Vc<Rope>> {
let content = &*content.await?;
let FileContent::Content(file) = content else {
bail!("Expected file content for file");
@ -363,7 +556,6 @@ pub async fn load_next_js_template(
Ok(file.content().to_owned().cell())
}
#[turbo_tasks::function]
pub fn virtual_next_js_template_path(
project_path: Vc<FileSystemPath>,
file: String,