From f45a15b621225a4ad3d401e4d3d114dd17647a90 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 11 Jan 2024 09:07:56 +0100 Subject: [PATCH] move custom allocator flag and add rustls-tls comment (#60128) ### What? enable the custom allocator flag to enable mialloc. allow to configure custom allocator on napi level. ### Why? It's faster and we had it enabled before. It was disable before as `next-core` is used with no default features in workspace Native Build: https://github.com/vercel/next.js/actions/runs/7388725004 Closes PACK-2185 --------- Co-authored-by: OJ Kwon <1210596+kwonoj@users.noreply.github.com> --- .github/workflows/build_and_deploy.yml | 2 +- .github/workflows/setup-nextjs-build.yml | 4 +-- Cargo.toml | 6 ++-- packages/next-swc/README.md | 18 ++++++++++++ packages/next-swc/crates/napi/Cargo.toml | 29 ++++++++++++++----- packages/next-swc/crates/napi/src/lib.rs | 6 ---- packages/next-swc/crates/next-api/Cargo.toml | 7 ----- .../next-swc/crates/next-build/Cargo.toml | 12 -------- packages/next-swc/crates/next-core/src/lib.rs | 6 ++++ packages/next-swc/package.json | 10 +++---- 10 files changed, 56 insertions(+), 44 deletions(-) diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 3076d69bc8..237886a936 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -202,7 +202,7 @@ jobs: rustup target add aarch64-unknown-linux-gnu && npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" && export CC_aarch64_unknown_linux_gnu=/usr/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc && - cd packages/next-swc && npm run build-native-release -- --target aarch64-unknown-linux-gnu --features plugin,rustls-tls,tracing/release_max_level_info && + cd packages/next-swc && npm run build-native-release -- --target aarch64-unknown-linux-gnu --features plugin,tracing/release_max_level_info && llvm-strip -x native/next-swc.*.node && objdump -T native/next-swc.*.node | grep GLIBC_ diff --git a/.github/workflows/setup-nextjs-build.yml b/.github/workflows/setup-nextjs-build.yml index f39da1cd1d..5aa1ce24f7 100644 --- a/.github/workflows/setup-nextjs-build.yml +++ b/.github/workflows/setup-nextjs-build.yml @@ -90,7 +90,7 @@ jobs: export TASKS_FS=$(printf 'patch.\\"%s\\".%s.git=\\"%s?rev=%s\\"' "$TURBOPACK_REMOTE" "turbo-tasks-fs" "$TURBOPACK_REMOTE" "$GITHUB_SHA") echo "Trying to build next-swc with turbopack $GITHUB_SHA" - hyperfine --min-runs 1 --show-output 'pnpm run --filter=@next/swc build-native --features plugin,rustls-tls --release --cargo-flags="--config $BINDING --config $TASKS --config $TASKS_FS"' + hyperfine --min-runs 1 --show-output 'pnpm run --filter=@next/swc build-native --features plugin --release --cargo-flags="--config $BINDING --config $TASKS --config $TASKS_FS"' echo "built=pass" >> $GITHUB_OUTPUT echo "Successfully built next-swc with turbopack $GITHUB_SHA" @@ -98,7 +98,7 @@ jobs: if: steps.build-next-swc-turbopack-patch.outputs.built != 'pass' run: | echo "Looks like we could not apply latest turbopack to next-swc. Trying to build next-swc with published turbopack. This might happen when there is a breaking changes in turbopack, and next.js is not yet updated." - hyperfine --min-runs 1 --show-output 'pnpm run --filter=@next/swc build-native --features plugin,rustls-tls --release' + hyperfine --min-runs 1 --show-output 'pnpm run --filter=@next/swc build-native --features plugin --release' echo "Successfully built next-swc with published turbopack" - name: Build next.js diff --git a/Cargo.toml b/Cargo.toml index b9c39784c0..d60946f797 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,9 +24,9 @@ opt-level = 3 [workspace.dependencies] # Workspace crates -next-api = { path = "packages/next-swc/crates/next-api", default-features = false } -next-build = { path = "packages/next-swc/crates/next-build", default-features = false } -next-core = { path = "packages/next-swc/crates/next-core", default-features = false } +next-api = { path = "packages/next-swc/crates/next-api" } +next-build = { path = "packages/next-swc/crates/next-build" } +next-core = { path = "packages/next-swc/crates/next-core" } next-custom-transforms = { path = "packages/next-swc/crates/next-custom-transforms" } # SWC crates diff --git a/packages/next-swc/README.md b/packages/next-swc/README.md index f3131c16d8..5e832e6f5a 100644 --- a/packages/next-swc/README.md +++ b/packages/next-swc/README.md @@ -31,6 +31,24 @@ Build wasm bindings to integrate with next.js pnpm build-wasm ``` +### napi bindings feature matrix + +Due to platform differences napi bindings selectively enables supported features. +See below tables for the currently enabled features. + +| arch\platform | Linux(gnu) | Linux(musl) | Darwin | Win32 | +| ------------- | ---------- | ----------- | --------- | --------- | +| ia32 | | | | a,b,d,e | +| x64 | a,b,d,e,f | a,b,d,e,f | a,b,d,e,f | a,b,d,e,f | +| aarch64 | a,d,e,f | a,d,e,f | a,b,d,e,f | a,b,c,e | + +- a: `turbo_tasks_malloc`, +- b: `turbo_tasks_malloc_custom_allocator`, +- c: `native-tls`, +- d: `rustls-tls`, +- e: `image-extended` (webp) +- f: `plugin` + ### Package hierarchies `@next/swc` consist of multiple rust packages to enable features. See below for the high level hierarchies. diff --git a/packages/next-swc/crates/napi/Cargo.toml b/packages/next-swc/crates/napi/Cargo.toml index abe29437ad..f314bdb753 100644 --- a/packages/next-swc/crates/napi/Cargo.toml +++ b/packages/next-swc/crates/napi/Cargo.toml @@ -8,7 +8,6 @@ publish = false crate-type = ["cdylib", "rlib"] [features] -default = ["rustls-tls"] # Instead of enabling all the plugin-related features by default, make it explicitly specified # when build (i.e napi --build --features plugin), same for the wasm as well. # this is due to some of transitive dependencies have features cannot be enabled at the same time @@ -20,11 +19,7 @@ plugin = [ "next-custom-transforms/plugin", "next-core/plugin", ] -sentry_native_tls = ["sentry", "sentry/native-tls", "native-tls"] -sentry_rustls = ["sentry", "sentry/rustls", "rustls-tls"] -native-tls = ["next-core/native-tls"] -rustls-tls = ["next-core/rustls-tls"] image-webp = ["next-core/image-webp"] image-avif = ["next-core/image-avif"] # Enable all the available image codec support. @@ -39,6 +34,27 @@ __internal_dhat-heap = ["dhat"] # effectively does nothing. __internal_dhat-ad-hoc = ["dhat"] +# Making custom_allocator as default feature will break some targets (i.e aarch64-linux), controlling it with +# build-time cfg instead. +# +# [NOTE] this is a workaround to enable downstream features for the own pkgs, since +# cargo does not support per-target features enablement. +[target.'cfg(all(target_os = "linux", target_arch = "aarch64"))'.dependencies] +turbopack-binding = { workspace = true, features = ["__turbo_tasks_malloc"] } + +[target.'cfg(all(target_os = "linux", not(target_arch = "aarch64")))'.dependencies] +turbopack-binding = { workspace = true, features = ["__turbo_tasks_malloc", "__turbo_tasks_malloc_custom_allocator"] } + +[target.'cfg(not(target_os = "linux"))'.dependencies] +turbopack-binding = { workspace = true, features = ["__turbo_tasks_malloc", "__turbo_tasks_malloc_custom_allocator"] } + +# Enable specific tls features per-target. +[target.'cfg(all(target_os = "windows", target_arch = "aarch64"))'.dependencies] +next-core = { workspace = true, features = ["native-tls"] } + +[target.'cfg(not(all(target_os = "windows", target_arch = "aarch64")))'.dependencies] +next-core = { workspace = true, features = ["rustls-tls"] } + [lints] workspace = true @@ -81,9 +97,6 @@ turbopack-binding = { workspace = true, features = [ url = {workspace = true} urlencoding = {workspace = true} -[target.'cfg(not(all(target_os = "linux", target_env = "musl", target_arch = "aarch64")))'.dependencies] -turbopack-binding = { workspace = true, features = ["__turbo_tasks_malloc"] } - # There are few build targets we can't use native-tls which default features rely on, # allow to specify alternative (rustls) instead via features. # Note to opt in rustls default-features should be disabled diff --git a/packages/next-swc/crates/napi/src/lib.rs b/packages/next-swc/crates/napi/src/lib.rs index 24dbf88cc0..fe60c24538 100644 --- a/packages/next-swc/crates/napi/src/lib.rs +++ b/packages/next-swc/crates/napi/src/lib.rs @@ -126,9 +126,3 @@ fn register() { include!(concat!(env!("OUT_DIR"), "/register.rs")); }); } - -#[cfg(all(feature = "native-tls", feature = "rustls-tls"))] -compile_error!("You can't enable both `native-tls` and `rustls-tls`"); - -#[cfg(all(not(feature = "native-tls"), not(feature = "rustls-tls")))] -compile_error!("You have to enable one of the TLS backends: `native-tls` or `rustls-tls`"); diff --git a/packages/next-swc/crates/next-api/Cargo.toml b/packages/next-swc/crates/next-api/Cargo.toml index 017a948557..a25704355d 100644 --- a/packages/next-swc/crates/next-api/Cargo.toml +++ b/packages/next-swc/crates/next-api/Cargo.toml @@ -9,13 +9,6 @@ autobenches = false [lib] bench = false -[features] -default = ["custom_allocator"] -custom_allocator = [ - "turbopack-binding/__turbo_tasks_malloc", - "turbopack-binding/__turbo_tasks_malloc_custom_allocator", -] - [lints] workspace = true diff --git a/packages/next-swc/crates/next-build/Cargo.toml b/packages/next-swc/crates/next-build/Cargo.toml index 4c96408a40..8cbad90621 100644 --- a/packages/next-swc/crates/next-build/Cargo.toml +++ b/packages/next-swc/crates/next-build/Cargo.toml @@ -10,23 +10,11 @@ autobenches = false bench = false [features] -# By default, we enable native-tls for reqwest via downstream transitive features. -# This is for the convenience of running daily dev workflows, i.e running -# `cargo xxx` without explicitly specifying features, not that we want to -# promote this as default backend. Actual configuration is done when building next-swc, -# and also turbopack standalone when we have it. -default = ["custom_allocator", "native-tls"] tokio_console = [ "dep:console-subscriber", "tokio/tracing", "turbo-tasks/tokio_tracing", ] -native-tls = ["next-core/native-tls"] -rustls-tls = ["next-core/rustls-tls"] -custom_allocator = [ - "turbopack-binding/__turbo_tasks_malloc", - "turbopack-binding/__turbo_tasks_malloc_custom_allocator", -] serializable = [] profile = [] diff --git a/packages/next-swc/crates/next-core/src/lib.rs b/packages/next-swc/crates/next-core/src/lib.rs index b789484cef..8991492346 100644 --- a/packages/next-swc/crates/next-core/src/lib.rs +++ b/packages/next-swc/crates/next-core/src/lib.rs @@ -65,3 +65,9 @@ pub fn register() { turbopack::ecmascript_plugin::register(); include!(concat!(env!("OUT_DIR"), "/register.rs")); } + +#[cfg(all(feature = "native-tls", feature = "rustls-tls"))] +compile_error!("You can't enable both `native-tls` and `rustls-tls`"); + +#[cfg(all(not(feature = "native-tls"), not(feature = "rustls-tls")))] +compile_error!("You have to enable one of the TLS backends: `native-tls` or `rustls-tls`"); diff --git a/packages/next-swc/package.json b/packages/next-swc/package.json index a1c37d2418..7c8cf9d02d 100644 --- a/packages/next-swc/package.json +++ b/packages/next-swc/package.json @@ -4,16 +4,16 @@ "private": true, "scripts": { "clean": "node ../../scripts/rm.mjs native", - "build-native": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --features plugin,rustls-tls,image-extended --js false native", - "build-native-release": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --release --features plugin,rustls-tls,image-extended,tracing/release_max_level_info --js false native", + "build-native": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --features plugin,image-extended --js false native", + "build-native-release": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --release --features plugin,image-extended,tracing/release_max_level_info --js false native", "build-native-no-plugin": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --features image-webp --js false native", - "build-native-no-plugin-woa": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --cargo-flags=--no-default-features --features native-tls,image-webp --js false native", - "build-native-no-plugin-woa-release": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --release --cargo-flags=--no-default-features --features native-tls,image-webp,tracing/release_max_level_info --js false native", + "build-native-no-plugin-woa": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --cargo-flags=--no-default-features --features image-webp --js false native", + "build-native-no-plugin-woa-release": "napi build --platform -p next-swc-napi --cargo-cwd ../../ --cargo-name next_swc_napi --release --cargo-flags=--no-default-features --features image-webp,tracing/release_max_level_info --js false native", "build-wasm": "wasm-pack build crates/wasm --scope=next", "cache-build-native": "echo $(ls native)", "rust-check-fmt": "cd ../..; cargo fmt -- --check", "rust-check-clippy": "cargo clippy --workspace --all-targets -- -D warnings -A deprecated", - "rust-check-napi-rustls": "cargo check -p next-swc-napi --features=rustls-tls", + "rust-check-napi-rustls": "cargo check -p next-swc-napi", "test-cargo-unit": "cargo nextest run --workspace --release --no-fail-fast" }, "napi": {