archive/rsnext
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make turbotrace run after the webpack build (#45621)

Browse source 
- [x] Depends on https://github.com/vercel/next.js/pull/45776

Turbotrace occupies too many memories while running; this PR makes it
run after the webpack build is finished, it can reduce the memory
hogging by webpack and turbotrace, thus avoiding OOM

The `maxFiles` option in turbotrace is removed because there is
`memoryLimit` option takes over its role.

Close WEB-556
This commit is contained in:
LongYinan 2023-02-11 03:58:55 +08:00 committed by GitHub
parent e13ace5824
commit ac7f2b6fe7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 500 additions and 234 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/workflows/build_test_deploy.yml vendored
View file

@ -7,7 +7,7 @@ on:
name: Build, test, and deploy
env:
NAPI_CLI_VERSION: 2.13.3
NAPI_CLI_VERSION: 2.14.7
TURBO_VERSION: 1.6.3
RUST_TOOLCHAIN: nightly-2022-11-04
PNPM_VERSION: 7.24.3
@ -932,8 +932,10 @@ jobs:
with:
image: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-x64
options: -e RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} -e NAPI_CLI_VERSION=${{ env.NAPI_CLI_VERSION }} -e TURBO_VERSION=${{ env.TURBO_VERSION }} -v ${{ env.HOME }}/.cargo/git:/root/.cargo/git -v ${{ env.HOME }}/.cargo/registry:/root/.cargo/registry -v ${{ github.workspace }}:/build -w /build
# turn on some optimization while building Rust codes to prevent tests timeout
run: |
set -e &&
export CARGO_PROFILE_DEV_OPT_LEVEL=1 &&
rustup toolchain install "${RUST_TOOLCHAIN}" &&
rustup default "${RUST_TOOLCHAIN}" &&
rustup target add x86_64-unknown-linux-gnu &&

2
.github/workflows/pull_request_stats.yml vendored
View file

@ -5,7 +5,7 @@ on:
name: Generate Pull Request Stats
env:
NAPI_CLI_VERSION: 2.13.3
NAPI_CLI_VERSION: 2.14.7
TURBO_VERSION: 1.6.3
RUST_TOOLCHAIN: nightly-2022-11-04
PNPM_VERSION: 7.24.3

9
docs/advanced-features/output-file-tracing.md
View file

@ -101,15 +101,8 @@ module.exports = {
// if there is `process.cwd()` expression in your code, you can set this option to tell `turbotrace` the value of `process.cwd()` while tracing.
// for example the require(process.cwd() + '/package.json') will be traced as require('/path/to/cwd/package.json')
processCwd?: string
// control the maximum number of files that are passed to the `turbotrace`
// default is 128
maxFiles?: number
// control the maximum memory usage of the `turbotrace`, in `MB`
// control the maximum memory usage of the `turbotrace`, in `MB`, default is `6000`.
memoryLimit?: number
// control if the webpack entries should be skipped when tracing
// tracing the entries is not necessary for applications that all assets could be traced in the dist files
// default is `false`
skipEntries?: boolean
},
},
}

2
packages/next-swc/package.json
View file

@ -31,6 +31,6 @@
}
},
"devDependencies": {
"@napi-rs/cli": "2.13.3"
"@napi-rs/cli": "2.14.7"
}
}

2
packages/next/package.json
View file

@ -125,7 +125,7 @@
"@babel/types": "7.18.0",
"@edge-runtime/primitives": "2.0.0",
"@hapi/accept": "5.0.2",
"@napi-rs/cli": "2.13.3",
"@napi-rs/cli": "2.14.7",
"@napi-rs/triples": "1.1.0",
"@next/polyfill-module": "13.1.7-canary.8",
"@next/polyfill-nomodule": "13.1.7-canary.8",

208
packages/next/src/build/index.ts
View file

@ -60,6 +60,7 @@ import {
MIDDLEWARE_BUILD_MANIFEST,
MIDDLEWARE_REACT_LOADABLE_MANIFEST,
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
TRACE_OUTPUT_VERSION,
} from '../shared/lib/constants'
import { getSortedRoutes, isDynamicRoute } from '../shared/lib/router/utils'
import { __ApiPreviewProps } from '../server/api-utils'
@ -924,7 +925,8 @@ export default async function build(
ignore: [] as string[],
}))
const webpackBuildDuration = await webpackBuild()
const { duration: webpackBuildDuration, turbotraceContext } =
await webpackBuild()
telemetry.record(
eventBuildCompleted(pagesPaths, {
@ -933,6 +935,89 @@ export default async function build(
})
)
let turboTasks: unknown
if (turbotraceContext) {
let binding = (await loadBindings()) as any
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
let turbotraceOutputPath: string | undefined
let turbotraceFiles: string[] | undefined
turboTasks = binding.turbo.createTurboTasks(
config.experimental.turbotrace?.memoryLimit ??
TURBO_TRACE_DEFAULT_MEMORY_LIMIT
)
const { entriesTrace, chunksTrace } = turbotraceContext
if (entriesTrace) {
const {
appDir: turbotraceContextAppDir,
depModArray,
entryNameMap,
outputPath,
action,
} = entriesTrace
const depModSet = new Set(depModArray)
const filesTracedInEntries: string[] =
await binding.turbo.startTrace(action, turboTasks)
const { contextDirectory, input: entriesToTrace } = action
// only trace the assets under the appDir
// exclude files from node_modules, entries and processed by webpack
const filesTracedFromEntries = filesTracedInEntries
.map((f) => path.join(contextDirectory, f))
.filter(
(f) =>
!f.includes('/node_modules/') &&
f.startsWith(turbotraceContextAppDir) &&
!entriesToTrace.includes(f) &&
!depModSet.has(f)
)
if (filesTracedFromEntries.length) {
// The turbo trace doesn't provide the traced file type and reason at present
// let's write the traced files into the first [entry].nft.json
const [[, entryName]] = Array.from(entryNameMap.entries()).filter(
([k]) => k.startsWith(turbotraceContextAppDir)
)
const traceOutputPath = path.join(
outputPath,
`../${entryName}.js.nft.json`
)
const traceOutputDir = path.dirname(traceOutputPath)
turbotraceOutputPath = traceOutputPath
turbotraceFiles = filesTracedFromEntries.map((file) =>
path.relative(traceOutputDir, file)
)
}
}
if (chunksTrace) {
const { action } = chunksTrace
await binding.turbo.startTrace(action, turboTasks)
if (turbotraceOutputPath && turbotraceFiles) {
const existedNftFile = await promises
.readFile(turbotraceOutputPath, 'utf8')
.then((existedContent) => JSON.parse(existedContent))
.catch(() => ({
version: TRACE_OUTPUT_VERSION,
files: [],
}))
existedNftFile.files.push(...turbotraceFiles)
const filesSet = new Set(existedNftFile.files)
existedNftFile.files = [...filesSet]
await promises.writeFile(
turbotraceOutputPath,
JSON.stringify(existedNftFile),
'utf8'
)
}
}
}
}
// For app directory, we run type checking after build.
if (appDir) {
await startTypeChecking()
@ -1574,7 +1659,7 @@ export default async function build(
if (config.experimental.turbotrace) {
let binding = (await loadBindings()) as any
if (!binding?.isWasm) {
nodeFileTrace = binding.turbo?.startTrace
nodeFileTrace = binding.turbo.startTrace
}
}
@ -1714,7 +1799,10 @@ export default async function build(
config.experimental?.turbotrace?.contextDirectory ??
config.experimental?.outputFileTracingRoot ??
dir
const toTrace = [require.resolve('next/dist/server/next-server')]
const nextServerEntry = require.resolve(
'next/dist/server/next-server'
)
const toTrace = [nextServerEntry]
// ensure we trace any dependencies needed for custom
// incremental cache handler
@ -1725,49 +1813,60 @@ export default async function build(
}
let serverResult: import('next/dist/compiled/@vercel/nft').NodeFileTraceResult
const ignores = [
'**/*.d.ts',
'**/*.map',
'**/next/dist/pages/**/*',
'**/next/dist/compiled/webpack/(bundle4|bundle5).js',
'**/node_modules/webpack5/**/*',
'**/next/dist/server/lib/squoosh/**/*.wasm',
'**/next/dist/server/lib/route-resolver*',
...(ciEnvironment.hasNextSupport
? [
// only ignore image-optimizer code when
// this is being handled outside of next-server
'**/next/dist/server/image-optimizer.js',
'**/node_modules/sharp/**/*',
]
: []),
...(!hasSsrAmpPages
? ['**/next/dist/compiled/@ampproject/toolbox-optimizer/**/*']
: []),
...(config.experimental.outputFileTracingIgnores || []),
]
const ignoreFn = (pathname: string) => {
return isMatch(pathname, ignores, { contains: true, dot: true })
}
const traceContext = path.join(nextServerEntry, '..', '..')
const tracedFiles = new Set<string>()
if (config.experimental.turbotrace) {
// handle the cache in the turbo-tracing side in the future
await nodeFileTrace({
action: 'annotate',
input: toTrace,
contextDirectory: root,
logLevel: config.experimental.turbotrace.logLevel,
processCwd: config.experimental.turbotrace.processCwd,
logDetail: config.experimental.turbotrace.logDetail,
showAll: config.experimental.turbotrace.logAll,
memoryLimit:
config.experimental.turbotrace.memoryLimit ??
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
})
} else {
const ignores = [
'**/next/dist/pages/**/*',
'**/next/dist/compiled/webpack/(bundle4|bundle5).js',
'**/node_modules/webpack5/**/*',
'**/next/dist/server/lib/squoosh/**/*.wasm',
'**/next/dist/server/lib/route-resolver*',
...(ciEnvironment.hasNextSupport
? [
// only ignore image-optimizer code when
// this is being handled outside of next-server
'**/next/dist/server/image-optimizer.js',
'**/node_modules/sharp/**/*',
]
: []),
...(!hasSsrAmpPages
? ['**/next/dist/compiled/@ampproject/toolbox-optimizer/**/*']
: []),
...(config.experimental.outputFileTracingIgnores || []),
]
const ignoreFn = (pathname: string) => {
return isMatch(pathname, ignores, { contains: true, dot: true })
const files: string[] = await nodeFileTrace(
{
action: 'print',
input: toTrace,
contextDirectory: traceContext,
logLevel: config.experimental.turbotrace.logLevel,
processCwd: config.experimental.turbotrace.processCwd,
logDetail: config.experimental.turbotrace.logDetail,
showAll: config.experimental.turbotrace.logAll,
},
turboTasks
)
for (const file of files) {
if (!ignoreFn(path.join(traceContext, file))) {
tracedFiles.add(
path
.relative(distDir, path.join(traceContext, file))
.replace(/\\/g, '/')
)
}
}
} else {
serverResult = await nodeFileTrace(toTrace, {
base: root,
processCwd: dir,
ignore: ignoreFn,
})
const tracedFiles = new Set()
serverResult.fileList.forEach((file) => {
tracedFiles.add(
@ -1776,23 +1875,22 @@ export default async function build(
.replace(/\\/g, '/')
)
})
await promises.writeFile(
nextServerTraceOutput,
JSON.stringify({
version: 1,
cacheKey,
files: [...tracedFiles],
} as {
version: number
files: string[]
})
)
await promises.unlink(cachedTracePath).catch(() => {})
await promises
.copyFile(nextServerTraceOutput, cachedTracePath)
.catch(() => {})
}
await promises.writeFile(
nextServerTraceOutput,
JSON.stringify({
version: 1,
cacheKey,
files: Array.from(tracedFiles),
} as {
version: number
files: string[]
})
)
await promises.unlink(cachedTracePath).catch(() => {})
await promises
.copyFile(nextServerTraceOutput, cachedTracePath)
.catch(() => {})
})
}

77
packages/next/src/build/webpack-build.ts
View file

@ -20,6 +20,10 @@ import { createEntrypoints } from './entries'
import loadConfig from '../server/config'
import { trace } from '../trace'
import { WEBPACK_LAYERS } from '../lib/constants'
import {
TraceEntryPointsPlugin,
TurbotraceContext,
} from './webpack/plugins/next-trace-entrypoints-plugin'
type CompilerResult = {
errors: webpack.StatsError[]
@ -37,7 +41,16 @@ function isTelemetryPlugin(plugin: unknown): plugin is TelemetryPlugin {
return plugin instanceof TelemetryPlugin
}
async function webpackBuildImpl(): Promise<number> {
function isTraceEntryPointsPlugin(
plugin: unknown
): plugin is TraceEntryPointsPlugin {
return plugin instanceof TraceEntryPointsPlugin
}
async function webpackBuildImpl(): Promise<{
duration: number
turbotraceContext?: TurbotraceContext
}> {
let result: CompilerResult | null = {
warnings: [],
errors: [],
@ -117,6 +130,7 @@ async function webpackBuildImpl(): Promise<number> {
})
const clientConfig = configs[0]
const serverConfig = configs[1]
if (
clientConfig.optimization &&
@ -141,7 +155,7 @@ async function webpackBuildImpl(): Promise<number> {
// injected to this set and then will be consumed by the client compiler.
injectedClientEntries.clear()
const serverResult = await runCompiler(configs[1], {
const serverResult = await runCompiler(serverConfig, {
runWebpackSpan,
})
const edgeServerResult = configs[2]
@ -206,6 +220,10 @@ async function webpackBuildImpl(): Promise<number> {
clientConfig as webpack.Configuration
).plugins?.find(isTelemetryPlugin)
const traceEntryPointsPlugin = (
serverConfig as webpack.Configuration
).plugins?.find(isTraceEntryPointsPlugin)
const webpackBuildEnd = process.hrtime(webpackBuildStart)
if (buildSpinner) {
buildSpinner.stopAndPersist()
@ -257,7 +275,10 @@ async function webpackBuildImpl(): Promise<number> {
} else {
Log.info('Compiled successfully')
}
return webpackBuildEnd[0]
return {
duration: webpackBuildEnd[0],
turbotraceContext: traceEntryPointsPlugin?.turbotraceContext,
}
}
}
@ -279,7 +300,19 @@ async function workerMain() {
try {
const result = await webpackBuildImpl()
parentPort!.postMessage(result)
const { entriesTrace } = result.turbotraceContext ?? {}
if (entriesTrace) {
const { entryNameMap, depModArray } = entriesTrace
if (depModArray) {
result.turbotraceContext!.entriesTrace!.depModArray = depModArray
}
if (entryNameMap) {
const entryEntries = Array.from(entryNameMap?.entries() ?? [])
// @ts-expect-error
result.turbotraceContext.entriesTrace.entryNameMap = entryEntries
}
}
parentPort!.postMessage(JSON.stringify(result))
} catch (e) {
parentPort!.postMessage(e)
} finally {
@ -305,17 +338,35 @@ async function webpackBuildWithWorker() {
},
})
const result = await new Promise((resolve, reject) => {
worker.on('message', resolve)
worker.on('error', reject)
worker.on('exit', (code) => {
if (code !== 0) {
reject(new Error(`Worker stopped with exit code ${code}`))
}
const result = JSON.parse(
await new Promise((resolve, reject) => {
worker.on('message', (data) => {
if (data instanceof Error) {
reject(data)
} else {
resolve(data)
}
})
worker.on('error', reject)
worker.on('exit', (code) => {
if (code !== 0) {
reject(new Error(`Worker stopped with exit code ${code}`))
}
})
})
})
) as {
duration: number
turbotraceContext?: TurbotraceContext
}
return result as number
if (result.turbotraceContext?.entriesTrace) {
const { entryNameMap } = result.turbotraceContext.entriesTrace
if (entryNameMap) {
result.turbotraceContext.entriesTrace.entryNameMap = new Map(entryNameMap)
}
}
return result
}
export async function webpackBuild() {

207
packages/next/src/build/webpack/plugins/next-trace-entrypoints-plugin.ts
View file

@ -1,5 +1,4 @@
import nodePath from 'path'
import nodeFs from 'fs'
import { Span } from '../../../trace'
import { spans } from './profiling-plugin'
import isError from '../../../lib/is-error'
@ -7,10 +6,7 @@ import {
nodeFileTrace,
NodeFileTraceReasons,
} from 'next/dist/compiled/@vercel/nft'
import {
TRACE_OUTPUT_VERSION,
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
} from '../../../shared/lib/constants'
import { TRACE_OUTPUT_VERSION } from '../../../shared/lib/constants'
import { webpack, sources } from 'next/dist/compiled/webpack/webpack'
import {
NODE_ESM_RESOLVE_OPTIONS,
@ -40,8 +36,6 @@ const NOT_TRACEABLE = [
'.svg',
]
const TURBO_TRACE_DEFAULT_MAX_FILES = 128
function getModuleFromDependency(
compilation: any,
dep: any
@ -103,7 +97,34 @@ function getFilesMapFromReasons(
return parentFilesMap
}
export interface TurbotraceAction {
action: 'print' | 'annotate'
input: string[]
contextDirectory: string
processCwd: string
logLevel?: NonNullable<
NextConfigComplete['experimental']['turbotrace']
>['logLevel']
showAll?: boolean
memoryLimit?: number
}
export interface TurbotraceContext {
entriesTrace?: {
action: TurbotraceAction
appDir: string
outputPath: string
depModArray: string[]
entryNameMap: Map<string, string>
}
chunksTrace?: {
action: TurbotraceAction
}
}
export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
public turbotraceContext: TurbotraceContext = {}
private appDir: string
private appDirEnabled?: boolean
private tracingRoot: string
@ -112,8 +133,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
private esmExternals?: NextConfigComplete['experimental']['esmExternals']
private turbotrace?: NextConfigComplete['experimental']['turbotrace']
private chunksToTrace: string[] = []
private turbotraceOutputPath?: string
private turbotraceFiles?: string[]
constructor({
appDir,
@ -184,7 +203,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
}
// startTrace existed and callable
if (this.turbotrace && !this.turbotrace.skipEntries) {
if (this.turbotrace) {
let binding = (await loadBindings()) as any
if (
!binding?.isWasm &&
@ -410,75 +429,24 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
await finishModulesSpan
.traceChild('turbo-trace', {
traceEntryCount: entriesToTrace.length + '',
})
.traceAsyncFn(async () => {
const contextDirectory =
this.turbotrace?.contextDirectory ?? this.tracingRoot
const maxFiles =
this.turbotrace?.maxFiles ?? TURBO_TRACE_DEFAULT_MAX_FILES
let chunks = [...entriesToTrace]
let restChunks =
chunks.length > maxFiles ? chunks.splice(maxFiles) : []
let filesTracedInEntries: string[] = []
while (chunks.length) {
filesTracedInEntries = filesTracedInEntries.concat(
await binding.turbo.startTrace({
action: 'print',
input: chunks,
contextDirectory,
processCwd:
this.turbotrace?.processCwd ?? this.appDir,
logLevel: this.turbotrace?.logLevel,
showAll: this.turbotrace?.logAll,
memoryLimit:
this.turbotrace?.memoryLimit ??
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
})
)
chunks = restChunks
if (restChunks.length) {
restChunks =
chunks.length > maxFiles
? chunks.splice(maxFiles)
: []
}
}
const contextDirectory =
this.turbotrace?.contextDirectory ?? this.tracingRoot
const chunks = [...entriesToTrace]
// only trace the assets under the appDir
// exclude files from node_modules, entries and processed by webpack
const filesTracedFromEntries = filesTracedInEntries
.map((f) => nodePath.join(contextDirectory, f))
.filter(
(f) =>
!f.includes('/node_modules/') &&
f.startsWith(this.appDir) &&
!entriesToTrace.includes(f) &&
!depModMap.has(f)
)
if (!filesTracedFromEntries.length) {
return
}
// The turbo trace doesn't provide the traced file type and reason at present
// let's write the traced files into the first [entry].nft.json
const [[, entryName]] = Array.from(
entryNameMap.entries()
).filter(([k]) => k.startsWith(this.appDir))
const outputPath = compilation.outputOptions.path!
const traceOutputPath = nodePath.join(
outputPath,
`../${entryName}.js.nft.json`
)
const traceOutputDir = nodePath.dirname(traceOutputPath)
this.turbotraceOutputPath = traceOutputPath
this.turbotraceFiles = filesTracedFromEntries.map((file) =>
nodePath.relative(traceOutputDir, file)
)
})
this.turbotraceContext.entriesTrace = {
action: {
action: 'print',
input: chunks,
contextDirectory,
processCwd: this.turbotrace?.processCwd ?? this.appDir,
logLevel: this.turbotrace?.logLevel,
showAll: this.turbotrace?.logAll,
},
appDir: this.appDir,
depModArray: Array.from(depModMap.keys()),
entryNameMap,
outputPath: compilation.outputOptions.path!,
}
return
}
}
@ -804,68 +772,31 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
})
if (this.turbotrace) {
compiler.hooks.afterEmit.tapPromise(PLUGIN_NAME, async (compilation) => {
const compilationSpan = spans.get(compilation) || spans.get(compiler)!
const traceEntrypointsPluginSpan = compilationSpan.traceChild(
'next-trace-entrypoint-plugin'
)
const turbotraceAfterEmitSpan = traceEntrypointsPluginSpan.traceChild(
'after-emit-turbo-trace'
)
await turbotraceAfterEmitSpan.traceAsyncFn(async () => {
let binding = (await loadBindings()) as any
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
const maxFiles =
this.turbotrace?.maxFiles ?? TURBO_TRACE_DEFAULT_MAX_FILES
const ignores = [...TRACE_IGNORES, ...this.traceIgnores]
compiler.hooks.afterEmit.tapPromise(PLUGIN_NAME, async () => {
let binding = (await loadBindings()) as any
if (
!binding?.isWasm &&
typeof binding.turbo.startTrace === 'function'
) {
const ignores = [...TRACE_IGNORES, ...this.traceIgnores]
const ignoreFn = (path: string) => {
return isMatch(path, ignores, { contains: true, dot: true })
}
let chunks = this.chunksToTrace.filter((chunk) => !ignoreFn(chunk))
let restChunks =
chunks.length > maxFiles ? chunks.splice(maxFiles) : []
while (chunks.length) {
await binding.turbo.startTrace({
action: 'annotate',
input: chunks,
contextDirectory:
this.turbotrace?.contextDirectory ?? this.tracingRoot,
processCwd: this.turbotrace?.processCwd ?? this.appDir,
showAll: this.turbotrace?.logAll,
logLevel: this.turbotrace?.logLevel,
memoryLimit:
this.turbotrace?.memoryLimit ??
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
})
chunks = restChunks
if (restChunks.length) {
restChunks =
chunks.length > maxFiles ? chunks.splice(maxFiles) : []
}
}
if (this.turbotraceOutputPath && this.turbotraceFiles) {
const existedNftFile = await nodeFs.promises
.readFile(this.turbotraceOutputPath, 'utf8')
.then((content) => JSON.parse(content))
.catch(() => ({
version: TRACE_OUTPUT_VERSION,
files: [],
}))
existedNftFile.files.push(...this.turbotraceFiles)
const filesSet = new Set(existedNftFile.files)
existedNftFile.files = [...filesSet]
nodeFs.promises.writeFile(
this.turbotraceOutputPath,
JSON.stringify(existedNftFile)
)
}
const ignoreFn = (path: string) => {
return isMatch(path, ignores, { contains: true, dot: true })
}
})
const chunks = this.chunksToTrace.filter((chunk) => !ignoreFn(chunk))
this.turbotraceContext.chunksTrace = {
action: {
action: 'annotate',
input: chunks,
contextDirectory:
this.turbotrace?.contextDirectory ?? this.tracingRoot,
processCwd: this.turbotrace?.processCwd ?? this.appDir,
showAll: this.turbotrace?.logAll,
logLevel: this.turbotrace?.logLevel,
},
}
}
})
}
}

6
packages/next/src/server/config-schema.ts
View file

@ -481,15 +481,9 @@ const configSchema = {
processCwd: {
type: 'string',
},
maxFiles: {
type: 'integer',
},
memoryLimit: {
type: 'integer',
},
skipEntries: {
type: 'boolean',
},
},
},
},

3
packages/next/src/server/config-shared.ts
View file

@ -206,9 +206,8 @@ export interface ExperimentalConfig {
logAll?: boolean
contextDirectory?: string
processCwd?: string
maxFiles?: number
/** in `MB` */
memoryLimit?: number
skipEntries?: boolean
}
mdxRs?: boolean
/**

12
pnpm-lock.yaml
View file

@ -508,7 +508,7 @@ importers:
'@babel/types': 7.18.0
'@edge-runtime/primitives': 2.0.0
'@hapi/accept': 5.0.2
'@napi-rs/cli': 2.13.3
'@napi-rs/cli': 2.14.7
'@napi-rs/triples': 1.1.0
'@next/env': 13.1.7-canary.8
'@next/polyfill-module': 13.1.7-canary.8
@ -718,7 +718,7 @@ importers:
'@babel/types': 7.18.0
'@edge-runtime/primitives': 2.0.0
'@hapi/accept': 5.0.2
'@napi-rs/cli': 2.13.3
'@napi-rs/cli': 2.14.7
'@napi-rs/triples': 1.1.0
'@next/polyfill-module': link:../next-polyfill-module
'@next/polyfill-nomodule': link:../next-polyfill-nomodule
@ -961,9 +961,9 @@ importers:
packages/next-swc:
specifiers:
'@napi-rs/cli': 2.13.3
'@napi-rs/cli': 2.14.7
devDependencies:
'@napi-rs/cli': 2.13.3
'@napi-rs/cli': 2.14.7
packages/react-dev-overlay:
specifiers:
@ -5912,8 +5912,8 @@ packages:
glob-to-regexp: 0.3.0
dev: true
/@napi-rs/cli/2.13.3:
resolution: {integrity: sha512-nAlbKuakQ+YHZE+M3Afih9UA1jr+gx63Gt4xHA+j2xD1NY6TjQ+QCgF9Yaj/YZIkCc2t3CZh52znFrfbU8b2bA==}
/@napi-rs/cli/2.14.7:
resolution: {integrity: sha512-H+YCgdbBNMWn8c878C3RsRYcMxamWI7rB0H+1TqOy9n6u31isikcRyW1VDyoNHp7ApfmzV0+tpPopVi0W7WHAA==}
engines: {node: '>= 10'}
hasBin: true
dev: true

3
test/integration/build-trace-extra-entries-turbo/test/index.test.js
View file

@ -6,8 +6,7 @@ import { nextBuild } from 'next-test-utils'
const appDir = join(__dirname, '../app')
// TODO: investigate failure from incorrect exit code
describe.skip('build trace with extra entries', () => {
describe('build trace with extra entries', () => {
it('should build and trace correctly', async () => {
const result = await nextBuild(appDir, undefined, {
cwd: appDir,

3
test/integration/turbotrace-with-webpack-worker/app/content/hello.json Normal file
View file

@ -0,0 +1,3 @@
{
"hello": "world"
}

1
test/integration/turbotrace-with-webpack-worker/app/include-me/hello.txt Normal file
View file

@ -0,0 +1 @@
first

1
test/integration/turbotrace-with-webpack-worker/app/include-me/second.txt Normal file
View file

@ -0,0 +1 @@
second

12
test/integration/turbotrace-with-webpack-worker/app/lib/fetch-data.js Normal file
View file

@ -0,0 +1,12 @@
import fs from 'fs'
import path from 'path'
const getCmsData = require('some-cms')
try {
fs.readdirSync(path.join(process.cwd(), 'public/exclude-me'))
} catch (_) {}
export function fetchData() {
return getCmsData()
}

8
test/integration/turbotrace-with-webpack-worker/app/lib/get-data.js Normal file
View file

@ -0,0 +1,8 @@
import fs from 'fs'
import path from 'path'
export function getData() {
return JSON.parse(
fs.readFileSync(path.join(process.cwd(), 'content/hello.json'), 'utf8')
)
}

29
test/integration/turbotrace-with-webpack-worker/app/next.config.js Normal file
View file

@ -0,0 +1,29 @@
const path = require('path')
module.exports = {
webpack(cfg, { isServer, nextRuntime }) {
console.log(cfg.entry)
const origEntry = cfg.entry
cfg.entry = async () => {
const origEntries = await origEntry()
if (isServer && nextRuntime === 'nodejs') {
const curEntry = origEntries['pages/_app']
origEntries['pages/_app'] = [
path.join(__dirname, 'lib/get-data.js'),
...curEntry,
]
console.log(origEntries)
}
return origEntries
}
return cfg
},
experimental: {
turbotrace: {
contextDirectory: path.join(__dirname, '..', '..', '..', '..'),
memoryLimit: 4096,
},
webpackBuildWorker: true,
},
}

4
test/integration/turbotrace-with-webpack-worker/app/node_modules/nested-structure/constants/package.json generated vendored Normal file
View file

@ -0,0 +1,4 @@
{
"name": "constants",
"main": "../lib/constants"
}

0
test/integration/turbotrace-with-webpack-worker/app/node_modules/nested-structure/lib/constants.js generated vendored Normal file
View file

0
test/integration/turbotrace-with-webpack-worker/app/node_modules/nested-structure/lib/index.js generated vendored Normal file
View file

4
test/integration/turbotrace-with-webpack-worker/app/node_modules/nested-structure/package.json generated vendored Normal file
View file

@ -0,0 +1,4 @@
{
"name": "nested-structure",
"main": "./lib/index.js"
}

5
test/integration/turbotrace-with-webpack-worker/app/node_modules/some-cms/index.js generated vendored Normal file
View file

@ -0,0 +1,5 @@
function getCmsData() {
return 'hello'
}
module.exports = getCmsData

4
test/integration/turbotrace-with-webpack-worker/app/node_modules/some-cms/package.json generated vendored Normal file
View file

@ -0,0 +1,4 @@
{
"name": "some-cms",
"main": "./index.js"
}

11
test/integration/turbotrace-with-webpack-worker/app/pages/another.js Normal file
View file

@ -0,0 +1,11 @@
import 'nested-structure/constants'
export default function Page() {
return <p>another page</p>
}
export function getStaticProps() {
return {
props: {},
}
}

24
test/integration/turbotrace-with-webpack-worker/app/pages/image-import.js Normal file
View file

@ -0,0 +1,24 @@
import fs from 'fs'
import path from 'path'
import Image from 'next/image'
import testImage from '../public/test.jpg'
export default function Page(props) {
return (
<div>
<Image src={testImage} placeholder="blur" alt="test" />
</div>
)
}
export function getServerSideProps() {
try {
// this should be included in the trace since it's not an
// import
fs.readFileSync(path.join(process.cwd(), 'public/another.jpg'))
} catch (_) {}
return {
props: {},
}
}

18
test/integration/turbotrace-with-webpack-worker/app/pages/index.js Normal file
View file

@ -0,0 +1,18 @@
import { fetchData } from '../lib/fetch-data'
export const config = {
unstable_includeFiles: ['include-me/*'],
unstable_excludeFiles: ['public/exclude-me/**/*'],
}
export default function Page() {
return 'index page'
}
export function getStaticProps() {
fetchData()
return {
props: {},
}
}

BIN
test/integration/turbotrace-with-webpack-worker/app/public/another.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 6.6 KiB

1
test/integration/turbotrace-with-webpack-worker/app/public/exclude-me/another.txt Normal file
View file

@ -0,0 +1 @@
second

1
test/integration/turbotrace-with-webpack-worker/app/public/exclude-me/hello.txt Normal file
View file

@ -0,0 +1 @@
first

BIN
test/integration/turbotrace-with-webpack-worker/app/public/test.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 6.6 KiB

73
test/integration/turbotrace-with-webpack-worker/test/index.test.js Normal file
View file

@ -0,0 +1,73 @@
/* eslint-env jest */
import fs from 'fs-extra'
import { join } from 'path'
import { nextBuild } from 'next-test-utils'
const appDir = join(__dirname, '../app')
describe('build trace with extra entries', () => {
it('should build and trace correctly', async () => {
const result = await nextBuild(appDir, undefined, {
cwd: appDir,
stderr: true,
stdout: true,
})
console.log(result)
expect(result.code).toBe(0)
const appTrace = await fs.readJSON(
join(appDir, '.next/server/pages/_app.js.nft.json')
)
const indexTrace = await fs.readJSON(
join(appDir, '.next/server/pages/index.js.nft.json')
)
const anotherTrace = await fs.readJSON(
join(appDir, '.next/server/pages/another.js.nft.json')
)
const imageTrace = await fs.readJSON(
join(appDir, '.next/server/pages/image-import.js.nft.json')
)
const tracedFiles = [
...appTrace.files,
...indexTrace.files,
...anotherTrace.files,
...imageTrace.files,
]
expect(tracedFiles.some((file) => file.endsWith('hello.json'))).toBe(true)
expect(tracedFiles.some((file) => file.includes('some-cms/index.js'))).toBe(
true
)
expect(
tracedFiles.some((file) => file === '../../../include-me/hello.txt')
).toBe(true)
expect(
tracedFiles.some((file) => file === '../../../include-me/second.txt')
).toBe(true)
expect(indexTrace.files.some((file) => file.includes('exclude-me'))).toBe(
false
)
expect(
tracedFiles.some((file) =>
file.includes('nested-structure/constants/package.json')
)
).toBe(true)
expect(
tracedFiles.some((file) => file.includes('nested-structure/package.json'))
).toBe(true)
expect(
tracedFiles.some((file) =>
file.includes('nested-structure/lib/constants.js')
)
).toBe(true)
expect(
tracedFiles.some((file) => file.includes('public/another.jpg'))
).toBe(true)
expect(tracedFiles.some((file) => file.includes('public/test.jpg'))).toBe(
false
)
})
})