From f993ea3d04e260ed73e28e98f86c8b32ca74841d Mon Sep 17 00:00:00 2001 From: Alexander Lyon Date: Fri, 24 May 2024 15:14:39 +0200 Subject: [PATCH] add next-build-test binary / library (#64121) This adds a new target in the next-swc package designed as an entrypoint for collecting memory benches. Later PRs in the stack add a script to run the benches and finally a manual-triggered CI step --- Cargo.lock | 243 ++++++++++++++++-- Cargo.toml | 7 + .../crates/next-build-test/Cargo.toml | 47 ++++ .../next-swc/crates/next-build-test/README.md | 32 +++ .../next-swc/crates/next-build-test/build.rs | 3 + .../crates/next-build-test/jsConfig.json | 35 +++ .../crates/next-build-test/nextConfig.json | 147 +++++++++++ .../crates/next-build-test/src/lib.rs | 163 ++++++++++++ .../crates/next-build-test/src/main.rs | 109 ++++++++ 9 files changed, 771 insertions(+), 15 deletions(-) create mode 100644 packages/next-swc/crates/next-build-test/Cargo.toml create mode 100644 packages/next-swc/crates/next-build-test/README.md create mode 100644 packages/next-swc/crates/next-build-test/build.rs create mode 100644 packages/next-swc/crates/next-build-test/jsConfig.json create mode 100644 packages/next-swc/crates/next-build-test/nextConfig.json create mode 100644 packages/next-swc/crates/next-build-test/src/lib.rs create mode 100644 packages/next-swc/crates/next-build-test/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 40cc5f25ff..d68b200e50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -450,6 +450,29 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "bitflags 2.5.0", + "cexpr", + "clang-sys", + "itertools 0.12.0", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.58", + "which", +] + [[package]] name = "binding_macros" version = "0.66.0" @@ -678,6 +701,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-expr" version = "0.15.4" @@ -709,6 +741,17 @@ dependencies = [ "windows-targets 0.48.1", ] +[[package]] +name = "clang-sys" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.2" @@ -1648,6 +1691,12 @@ dependencies = [ "libc", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "funty" version = "2.0.0" @@ -1972,6 +2021,15 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "hstr" version = "0.2.8" @@ -2048,7 +2106,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.9", + "socket2 0.5.6", "tokio", "tower-service", "tracing", @@ -2094,6 +2152,41 @@ dependencies = [ "tungstenite 0.18.0", ] +[[package]] +name = "iai-callgrind" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e99bf26f496b13ac6273014f40afda46a233fbfb0289ce50fb4daaad2f2ffc80" +dependencies = [ + "bincode", + "bindgen", + "cc", + "iai-callgrind-macros", + "iai-callgrind-runner", + "regex", +] + +[[package]] +name = "iai-callgrind-macros" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2a4bb39225592c0a28cfca6f70af52ebd8da23f533c2cdd0a3329c1fa252d56" +dependencies = [ + "proc-macro-error", + "proc-macro2", + "quote", + "syn 2.0.58", +] + +[[package]] +name = "iai-callgrind-runner" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c23a951b9eccaa1e38556d27473d1462a9c247a27961812edcaac156af861282" +dependencies = [ + "serde", +] + [[package]] name = "iana-time-zone" version = "0.1.57" @@ -2410,6 +2503,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "leb128" version = "0.2.5" @@ -3007,6 +3106,28 @@ dependencies = [ "vergen 7.5.1", ] +[[package]] +name = "next-build-test" +version = "0.1.0" +dependencies = [ + "anyhow", + "futures-util", + "iai-callgrind", + "next-api", + "next-core", + "num_cpus", + "rand 0.8.5", + "serde_json", + "tempdir", + "tokio", + "tokio-stream", + "tracing", + "tracing-subscriber", + "turbo-tasks", + "turbo-tasks-malloc", + "turbopack-binding", +] + [[package]] name = "next-core" version = "0.1.0" @@ -3201,6 +3322,12 @@ dependencies = [ "serde", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-derive" version = "0.3.3" @@ -3599,7 +3726,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" dependencies = [ "phf_shared 0.10.0", - "rand", + "rand 0.8.5", ] [[package]] @@ -3609,7 +3736,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared 0.11.2", - "rand", + "rand 0.8.5", ] [[package]] @@ -3762,6 +3889,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7" +dependencies = [ + "proc-macro2", + "syn 2.0.58", +] + [[package]] name = "priority-queue" version = "1.3.1" @@ -3891,6 +4028,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce082a9940a7ace2ad4a8b7d0b1eac6aa378895f18be598230c5f2284ac05426" +[[package]] +name = "rand" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552840b97013b1a26992c11eac34bdd778e464601a4c2054b5f0bff7c6761293" +dependencies = [ + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "rdrand", + "winapi", +] + [[package]] name = "rand" version = "0.8.5" @@ -3899,7 +4049,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -3909,9 +4059,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.6.4" @@ -3956,7 +4121,7 @@ dependencies = [ "num-traits", "once_cell", "paste", - "rand", + "rand 0.8.5", "rand_chacha", "rust_hawktracer", "rustc_version 0.4.0", @@ -4007,6 +4172,15 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "rdrand" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" +dependencies = [ + "rand_core 0.3.1", +] + [[package]] name = "react_remove_properties" version = "0.24.12" @@ -4138,6 +4312,15 @@ dependencies = [ "swc_ecma_visit", ] +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + [[package]] name = "rend" version = "0.4.0" @@ -4719,6 +4902,12 @@ dependencies = [ "memmap2 0.6.2", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook" version = "0.3.17" @@ -6462,6 +6651,16 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" +[[package]] +name = "tempdir" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f2b5fb00ccdf689e0149d1b1b3c03fead81c2b37735d812fa8bddbbf41b6d8" +dependencies = [ + "rand 0.4.6", + "remove_dir_all", +] + [[package]] name = "tempfile" version = "3.8.0" @@ -6609,13 +6808,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.30" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4a34ab300f2dee6e562c10a046fc05e358b29f9bf92277f30c3c8d82275f6f5" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", "libc", + "num-conv", "num_threads", "powerfmt", "serde", @@ -6631,10 +6831,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.15" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ad70d68dba9e1f8aceda7aa6711965dfec1cac869f311a51bd08b3a2ccbce20" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ + "num-conv", "time-core", ] @@ -6916,7 +7117,7 @@ dependencies = [ "http 0.2.11", "httparse", "log", - "rand", + "rand 0.8.5", "sha1", "thiserror", "url", @@ -6935,7 +7136,7 @@ dependencies = [ "http 1.1.0", "httparse", "log", - "rand", + "rand 0.8.5", "sha1", "thiserror", "url", @@ -7646,7 +7847,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", - "rand", + "rand 0.8.5", "static_assertions", ] @@ -8370,7 +8571,7 @@ dependencies = [ "once_cell", "petgraph", "pin-project", - "rand", + "rand 0.8.5", "rusty_pool", "semver 1.0.23", "serde", @@ -8501,7 +8702,7 @@ dependencies = [ "lexical-sort", "once_cell", "path-clean 1.0.1", - "rand", + "rand 0.8.5", "semver 1.0.23", "serde", "serde_cbor", @@ -8542,6 +8743,18 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index d0e16eed05..eb6f566a7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "packages/next-swc/crates/napi", "packages/next-swc/crates/wasm", "packages/next-swc/crates/next-api", + "packages/next-swc/crates/next-build-test", "packages/next-swc/crates/next-build", "packages/next-swc/crates/next-core", "packages/next-swc/crates/next-custom-transforms", @@ -22,6 +23,11 @@ debug-assertions = false [profile.dev.build-override] opt-level = 3 +[profile.release-with-debug] +inherits = "release" +debug = true +lto = "thin" + [workspace.dependencies] # Workspace crates next-api = { path = "packages/next-swc/crates/next-api" } @@ -42,6 +48,7 @@ turbopack-binding = { git = "https://github.com/vercel/turbo.git", tag = "turbop turbo-tasks = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-240521.2" } # [TODO]: need to refactor embed_directory! macro usage in next-core turbo-tasks-fs = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-240521.2" } +turbo-tasks-malloc = { git = "https://github.com/vercel/turbo.git", tag = "turbopack-240521.2" } # General Deps diff --git a/packages/next-swc/crates/next-build-test/Cargo.toml b/packages/next-swc/crates/next-build-test/Cargo.toml new file mode 100644 index 0000000000..4d7a2667ef --- /dev/null +++ b/packages/next-swc/crates/next-build-test/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "next-build-test" +version = "0.1.0" +description = "TBD" +license = "MPL-2.0" +edition = "2021" +autobenches = false + +[lints] +workspace = true + +[dependencies] +next-core = { workspace = true, features = ["rustls-tls"] } +next-api = { workspace = true } +serde_json = { workspace = true } +anyhow = { workspace = true } +tokio = { workspace = true, features = ["full"] } +turbo-tasks-malloc = { workspace = true, default-features = false } +turbopack-binding = { workspace = true, features = [ + "__turbo_tasks", + "__turbo_tasks_memory", + "__turbo_tasks_env", + "__turbo_tasks_fs", + "__turbo_tasks_memory", + "__turbopack", + "__turbopack_nodejs", + "__turbopack_core", + "__turbopack_browser", + "__turbopack_ecmascript", + "__turbopack_ecmascript_runtime", + "__turbopack_env", + "__turbopack_node", +] } +turbo-tasks = { workspace = true } +rand = { workspace = true, features = ["small_rng"] } +num_cpus = "1.16.0" +tokio-stream = "0.1.15" +tracing = "0.1" +tracing-subscriber = "0.3" +futures-util = "0.3.30" + +[build-dependencies] +turbopack-binding = { workspace = true, features = ["__turbo_tasks_build"] } + +[dev-dependencies] +iai-callgrind = "0.10.2" +tempdir = "0.3.7" diff --git a/packages/next-swc/crates/next-build-test/README.md b/packages/next-swc/crates/next-build-test/README.md new file mode 100644 index 0000000000..ee665abfc1 --- /dev/null +++ b/packages/next-swc/crates/next-build-test/README.md @@ -0,0 +1,32 @@ +# next-build-test + +This binary lets you sidestep all of the node bundling and run a turbo build +against a raw rust binary. It does _not_ do everything nextjs does, but it +is an ok approximation. + +## Getting started + +You will need a project_options file that points to some nextjs repo that has +its dependencies installed. The easiest way to do that is to run a nextjs +build using a modified binary that produces one out for you or to run the +`generate` command and tweak it manually. We cannot bundle one in the repo, +since it needs fs-specific paths and env vars. + +You can run the binary with the `generate` flag to build one for you. + +```sh +cargo run -- generate /path/to/project > project_options.json +cargo run -- run +``` + +## Flags + +The `run` command supports 4 flags: + +- `strategy` can be one of sequential, concurrent, or parallel. defines how + work is structured +- `factor` defined how many pages should be built at once. defaults to num_cpus +- `limit` defines the highest number of pages to build. the pages are + shuffled deterministically. defaults to 1 page +- `pages` a comma separated list of routes to run. queues that precise set in + the order specified diff --git a/packages/next-swc/crates/next-build-test/build.rs b/packages/next-swc/crates/next-build-test/build.rs new file mode 100644 index 0000000000..f66b495c19 --- /dev/null +++ b/packages/next-swc/crates/next-build-test/build.rs @@ -0,0 +1,3 @@ +fn main() { + turbopack_binding::turbo::tasks_build::generate_register(); +} diff --git a/packages/next-swc/crates/next-build-test/jsConfig.json b/packages/next-swc/crates/next-build-test/jsConfig.json new file mode 100644 index 0000000000..c3768cef33 --- /dev/null +++ b/packages/next-swc/crates/next-build-test/jsConfig.json @@ -0,0 +1,35 @@ +{ + "compilerOptions": { + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": false, + "skipLibCheck": true, + "strict": true, + "lib": ["lib.dom.d.ts", "lib.dom.iterable.d.ts", "lib.esnext.d.ts"], + "module": 99, + "target": 8, + "moduleResolution": 2, + "incremental": true, + "noEmit": true, + "resolveJsonModule": true, + "isolatedModules": true, + "jsx": 1, + "plugins": [ + { + "name": "typescript-plugin-css-modules", + "options": { + "goToDefinition": true + } + }, + { + "name": "next" + } + ], + "allowJs": true, + "paths": {}, + "tsBuildInfoFile": "/tmp/ignore", + "strictNullChecks": true, + "pathsBasePath": "/tmp/ignore" + } +} diff --git a/packages/next-swc/crates/next-build-test/nextConfig.json b/packages/next-swc/crates/next-build-test/nextConfig.json new file mode 100644 index 0000000000..511fe69f1b --- /dev/null +++ b/packages/next-swc/crates/next-build-test/nextConfig.json @@ -0,0 +1,147 @@ +{ + "env": {}, + "webpack": {}, + "eslint": { + "ignoreDuringBuilds": false + }, + "typescript": { + "ignoreBuildErrors": false, + "tsconfigPath": "tsconfig.json" + }, + "distDir": ".next", + "cleanDistDir": true, + "assetPrefix": "", + "cacheMaxMemorySize": 52428800, + "configOrigin": "next.config.mjs", + "useFileSystemPublicRoutes": true, + "generateBuildId": null, + "generateEtags": true, + "pageExtensions": ["jsx", "js", "tsx", "ts", "mdx", "md"], + "poweredByHeader": true, + "compress": true, + "analyticsId": "", + "images": { + "deviceSizes": [640, 750, 828, 1080, 1200, 1920, 2048, 3840], + "imageSizes": [16, 32, 48, 64, 96, 128, 256, 384], + "path": "/_next/image", + "loader": "default", + "loaderFile": "", + "domains": [], + "disableStaticImages": false, + "minimumCacheTTL": 60, + "formats": ["image/avif", "image/webp"], + "dangerouslyAllowSVG": false, + "contentSecurityPolicy": "script-src 'none'; frame-src 'none'; sandbox;", + "contentDispositionType": "inline", + "remotePatterns": [], + "unoptimized": false + }, + "devIndicators": { + "buildActivity": true, + "buildActivityPosition": "bottom-right" + }, + "onDemandEntries": { + "maxInactiveAge": 60000, + "pagesBufferLength": 5 + }, + "amp": { + "canonicalBase": "" + }, + "basePath": "", + "sassOptions": {}, + "trailingSlash": false, + "i18n": null, + "productionBrowserSourceMaps": false, + "optimizeFonts": true, + "excludeDefaultMomentLocales": true, + "serverRuntimeConfig": {}, + "publicRuntimeConfig": {}, + "reactProductionProfiling": false, + "reactStrictMode": true, + "httpAgentOptions": { + "keepAlive": true + }, + "outputFileTracing": true, + "staticPageGenerationTimeout": 60, + "swcMinify": true, + "modularizeImports": {}, + "experimental": { + "prerenderEarlyExit": false, + "serverMinification": true, + "serverSourceMaps": false, + "linkNoTouchStart": false, + "caseSensitiveRoutes": false, + "clientRouterFilter": true, + "clientRouterFilterRedirects": false, + "fetchCacheKeyPrefix": "", + "middlewarePrefetch": "flexible", + "optimisticClientCache": true, + "manualClientBasePath": false, + "cpus": 2, + "memoryBasedWorkersCount": false, + "isrFlushToDisk": true, + "workerThreads": false, + "optimizeCss": false, + "nextScriptWorkers": false, + "scrollRestoration": false, + "externalDir": false, + "disableOptimizedLoading": false, + "gzipSize": true, + "craCompat": false, + "esmExternals": true, + "fullySpecified": false, + "outputFileTracingRoot": "/tmp/ignore", + "swcTraceProfiling": false, + "forceSwcTransforms": false, + "largePageDataBytes": 128000, + "adjustFontFallbacks": false, + "adjustFontFallbacksWithSizeAdjust": false, + "turbo": { + "rules": { + "*.mdx": { + "loaders": ["turbopack-mdx-loader"], + "as": "*.tsx" + } + }, + "resolveAlias": { + "fs": { + "browser": "./turbopack/empty.js" + }, + "cookie": { + "browser": "./turbopack/empty.js" + }, + "http": { + "browser": "./turbopack/empty.js" + }, + "https": { + "browser": "./turbopack/empty.js" + }, + "node-fetch": { + "browser": "./turbopack/empty.js" + } + } + }, + "typedRoutes": false, + "instrumentationHook": true, + "bundlePagesExternals": false, + "parallelServerCompiles": false, + "parallelServerBuildTraces": false, + "ppr": false, + "missingSuspenseWithCSRBailout": true, + "optimizeServerReact": true, + "useEarlyImport": false, + "serverComponentsExternalPackages": [], + "useLightningcss": true, + "optimizePackageImports": [] + }, + "configFile": "/tmp/ignore", + "configFileName": "next.config.mjs", + "transpilePackages": [], + "_originalRewrites": { + "beforeFiles": [], + "afterFiles": [], + "fallback": [] + }, + "_originalRedirects": [], + "exportPathMap": {} +} diff --git a/packages/next-swc/crates/next-build-test/src/lib.rs b/packages/next-swc/crates/next-build-test/src/lib.rs new file mode 100644 index 0000000000..d7622ed459 --- /dev/null +++ b/packages/next-swc/crates/next-build-test/src/lib.rs @@ -0,0 +1,163 @@ +#![feature(future_join)] +#![feature(min_specialization)] +#![feature(arbitrary_self_types)] + +use std::str::FromStr; + +use anyhow::{Context, Result}; +use futures_util::{StreamExt, TryStreamExt}; +use next_api::{ + project::{ProjectContainer, ProjectOptions}, + route::{Endpoint, Route}, +}; + +pub async fn main_inner( + strat: Strategy, + factor: usize, + limit: usize, + files: Option>, +) -> Result<()> { + register(); + + let mut file = std::fs::File::open("project_options.json").with_context(|| { + let path = std::env::current_dir() + .unwrap() + .join("project_options.json"); + format!("loading file at {}", path.display()) + })?; + + let options: ProjectOptions = serde_json::from_reader(&mut file).unwrap(); + let project = ProjectContainer::new(options); + + tracing::info!("collecting endpoints"); + let entrypoints = project.entrypoints().await?; + + let routes = if let Some(files) = files { + tracing::info!("builing only the files:"); + for file in &files { + tracing::info!(" {}", file); + } + + // filter out the files that are not in the list + // we expect this to be small so linear search OK + Box::new( + entrypoints + .routes + .clone() + .into_iter() + .filter(move |(name, _)| files.iter().any(|f| f == name)), + ) as Box + Send + Sync> + } else { + Box::new(shuffle(entrypoints.routes.clone().into_iter())) + }; + + let count = render_routes(routes, strat, factor, limit).await; + tracing::info!("rendered {} pages", count); + + Ok(()) +} + +pub fn register() { + next_api::register(); + include!(concat!(env!("OUT_DIR"), "/register.rs")); +} + +#[derive(PartialEq, Copy, Clone)] +pub enum Strategy { + Sequential, + Concurrent, + Parallel, +} + +impl std::fmt::Display for Strategy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Strategy::Sequential => write!(f, "sequential"), + Strategy::Concurrent => write!(f, "concurrent"), + Strategy::Parallel => write!(f, "parallel"), + } + } +} + +impl FromStr for Strategy { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "sequential" => Ok(Strategy::Sequential), + "concurrent" => Ok(Strategy::Concurrent), + "parallel" => Ok(Strategy::Parallel), + _ => Err(anyhow::anyhow!("invalid strategy")), + } + } +} + +pub fn shuffle<'a, T: 'a>(items: impl Iterator) -> impl Iterator { + use rand::{seq::SliceRandom, SeedableRng}; + let mut rng = rand::rngs::SmallRng::from_seed([0; 32]); + let mut input = items.collect::>(); + input.shuffle(&mut rng); + input.into_iter() +} + +pub async fn render_routes( + routes: impl Iterator, + strategy: Strategy, + factor: usize, + limit: usize, +) -> usize { + tracing::info!( + "rendering routes with {} parallel and strat {}", + factor, + strategy + ); + + let stream = tokio_stream::iter(routes) + .map(move |(name, route)| { + let fut = async move { + tracing::info!("{name}"); + + match route { + Route::Page { + html_endpoint, + data_endpoint: _, + } => { + html_endpoint.write_to_disk().await?; + } + Route::PageApi { endpoint } => { + endpoint.write_to_disk().await?; + } + Route::AppPage(routes) => { + for route in routes { + route.html_endpoint.write_to_disk().await?; + } + } + Route::AppRoute { + original_name: _, + endpoint, + } => { + endpoint.write_to_disk().await?; + } + Route::Conflict => { + tracing::info!("WARN: conflict {}", name); + } + } + + Ok::<_, anyhow::Error>(()) + }; + + async move { + match strategy { + Strategy::Parallel => tokio::task::spawn(fut).await.unwrap(), + _ => fut.await, + } + } + }) + .take(limit) + .buffer_unordered(factor) + .try_collect::>() + .await + .unwrap(); + + stream.len() +} diff --git a/packages/next-swc/crates/next-build-test/src/main.rs b/packages/next-swc/crates/next-build-test/src/main.rs new file mode 100644 index 0000000000..32aaa4c4f3 --- /dev/null +++ b/packages/next-swc/crates/next-build-test/src/main.rs @@ -0,0 +1,109 @@ +use std::{convert::Infallible, str::FromStr}; + +use next_api::project::{DefineEnv, ProjectOptions}; +use next_build_test::{main_inner, Strategy}; +use turbo_tasks::TurboTasks; +use turbo_tasks_malloc::TurboMalloc; +use turbopack_binding::turbo::tasks_memory::MemoryBackend; + +enum Cmd { + Run, + Generate, +} +impl FromStr for Cmd { + type Err = Infallible; + + fn from_str(s: &str) -> Result { + match s { + "run" => Ok(Cmd::Run), + "generate" => Ok(Cmd::Generate), + _ => panic!("invalid command, please use 'run' or 'generate'"), + } + } +} + +fn main() { + tracing_subscriber::fmt::init(); + + let cmd = std::env::args() + .nth(1) + .map(|s| Cmd::from_str(&s)) + .unwrap_or(Ok(Cmd::Run)) + .unwrap(); + + match cmd { + Cmd::Run => { + let strat = std::env::args() + .nth(2) + .map(|s| Strategy::from_str(&s)) + .transpose() + .unwrap() + .unwrap_or(Strategy::Sequential); + + let mut factor = std::env::args() + .nth(3) + .map(|s| s.parse().unwrap()) + .unwrap_or(num_cpus::get()); + + let limit = std::env::args() + .nth(4) + .map(|s| s.parse().unwrap()) + .unwrap_or(1); + + let files = std::env::args() + .nth(5) + .map(|f| f.split(',').map(ToOwned::to_owned).collect()); + + if strat == Strategy::Sequential { + factor = 1; + } + + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .on_thread_stop(|| { + TurboMalloc::thread_stop(); + tracing::debug!("threads stopped"); + }) + .build() + .unwrap() + .block_on(async { + let tt = TurboTasks::new(MemoryBackend::new(usize::MAX)); + let x = tt.run_once(main_inner(strat, factor, limit, files)).await; + tracing::debug!("done"); + x + }) + .unwrap(); + } + Cmd::Generate => { + let project_path = std::env::args().nth(2).unwrap_or(".".to_string()); + let current_dir = std::env::current_dir().unwrap(); + let absolute_dir = current_dir.join(project_path); + let canonical_path = std::fs::canonicalize(absolute_dir).unwrap(); + + let options = ProjectOptions { + build_id: "test".to_owned(), + define_env: DefineEnv { + client: vec![], + edge: vec![], + nodejs: vec![], + }, + dev: true, + encryption_key: "deadbeef".to_string(), + env: vec![], + js_config: include_str!("../jsConfig.json").to_string(), + next_config: include_str!("../nextConfig.json").to_string(), + preview_props: next_api::project::DraftModeOptions { + preview_mode_encryption_key: "deadbeef".to_string(), + preview_mode_id: "test".to_string(), + preview_mode_signing_key: "deadbeef".to_string(), + }, + project_path: canonical_path.to_string_lossy().to_string(), + root_path: "/".to_string(), + watch: false, + }; + + let json = serde_json::to_string_pretty(&options).unwrap(); + println!("{}", json); + } + } +}