Make turbotrace run after the webpack build (#45621)
- [x] Depends on https://github.com/vercel/next.js/pull/45776 Turbotrace occupies too many memories while running; this PR makes it run after the webpack build is finished, it can reduce the memory hogging by webpack and turbotrace, thus avoiding OOM The `maxFiles` option in turbotrace is removed because there is `memoryLimit` option takes over its role. Close WEB-556
This commit is contained in:
parent
e13ace5824
commit
ac7f2b6fe7
32 changed files with 500 additions and 234 deletions
4
.github/workflows/build_test_deploy.yml
vendored
4
.github/workflows/build_test_deploy.yml
vendored
|
@ -7,7 +7,7 @@ on:
|
|||
name: Build, test, and deploy
|
||||
|
||||
env:
|
||||
NAPI_CLI_VERSION: 2.13.3
|
||||
NAPI_CLI_VERSION: 2.14.7
|
||||
TURBO_VERSION: 1.6.3
|
||||
RUST_TOOLCHAIN: nightly-2022-11-04
|
||||
PNPM_VERSION: 7.24.3
|
||||
|
@ -932,8 +932,10 @@ jobs:
|
|||
with:
|
||||
image: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-x64
|
||||
options: -e RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} -e NAPI_CLI_VERSION=${{ env.NAPI_CLI_VERSION }} -e TURBO_VERSION=${{ env.TURBO_VERSION }} -v ${{ env.HOME }}/.cargo/git:/root/.cargo/git -v ${{ env.HOME }}/.cargo/registry:/root/.cargo/registry -v ${{ github.workspace }}:/build -w /build
|
||||
# turn on some optimization while building Rust codes to prevent tests timeout
|
||||
run: |
|
||||
set -e &&
|
||||
export CARGO_PROFILE_DEV_OPT_LEVEL=1 &&
|
||||
rustup toolchain install "${RUST_TOOLCHAIN}" &&
|
||||
rustup default "${RUST_TOOLCHAIN}" &&
|
||||
rustup target add x86_64-unknown-linux-gnu &&
|
||||
|
|
2
.github/workflows/pull_request_stats.yml
vendored
2
.github/workflows/pull_request_stats.yml
vendored
|
@ -5,7 +5,7 @@ on:
|
|||
name: Generate Pull Request Stats
|
||||
|
||||
env:
|
||||
NAPI_CLI_VERSION: 2.13.3
|
||||
NAPI_CLI_VERSION: 2.14.7
|
||||
TURBO_VERSION: 1.6.3
|
||||
RUST_TOOLCHAIN: nightly-2022-11-04
|
||||
PNPM_VERSION: 7.24.3
|
||||
|
|
|
@ -101,15 +101,8 @@ module.exports = {
|
|||
// if there is `process.cwd()` expression in your code, you can set this option to tell `turbotrace` the value of `process.cwd()` while tracing.
|
||||
// for example the require(process.cwd() + '/package.json') will be traced as require('/path/to/cwd/package.json')
|
||||
processCwd?: string
|
||||
// control the maximum number of files that are passed to the `turbotrace`
|
||||
// default is 128
|
||||
maxFiles?: number
|
||||
// control the maximum memory usage of the `turbotrace`, in `MB`
|
||||
// control the maximum memory usage of the `turbotrace`, in `MB`, default is `6000`.
|
||||
memoryLimit?: number
|
||||
// control if the webpack entries should be skipped when tracing
|
||||
// tracing the entries is not necessary for applications that all assets could be traced in the dist files
|
||||
// default is `false`
|
||||
skipEntries?: boolean
|
||||
},
|
||||
},
|
||||
}
|
||||
|
|
|
@ -31,6 +31,6 @@
|
|||
}
|
||||
},
|
||||
"devDependencies": {
|
||||
"@napi-rs/cli": "2.13.3"
|
||||
"@napi-rs/cli": "2.14.7"
|
||||
}
|
||||
}
|
||||
|
|
|
@ -125,7 +125,7 @@
|
|||
"@babel/types": "7.18.0",
|
||||
"@edge-runtime/primitives": "2.0.0",
|
||||
"@hapi/accept": "5.0.2",
|
||||
"@napi-rs/cli": "2.13.3",
|
||||
"@napi-rs/cli": "2.14.7",
|
||||
"@napi-rs/triples": "1.1.0",
|
||||
"@next/polyfill-module": "13.1.7-canary.8",
|
||||
"@next/polyfill-nomodule": "13.1.7-canary.8",
|
||||
|
|
|
@ -60,6 +60,7 @@ import {
|
|||
MIDDLEWARE_BUILD_MANIFEST,
|
||||
MIDDLEWARE_REACT_LOADABLE_MANIFEST,
|
||||
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
|
||||
TRACE_OUTPUT_VERSION,
|
||||
} from '../shared/lib/constants'
|
||||
import { getSortedRoutes, isDynamicRoute } from '../shared/lib/router/utils'
|
||||
import { __ApiPreviewProps } from '../server/api-utils'
|
||||
|
@ -924,7 +925,8 @@ export default async function build(
|
|||
ignore: [] as string[],
|
||||
}))
|
||||
|
||||
const webpackBuildDuration = await webpackBuild()
|
||||
const { duration: webpackBuildDuration, turbotraceContext } =
|
||||
await webpackBuild()
|
||||
|
||||
telemetry.record(
|
||||
eventBuildCompleted(pagesPaths, {
|
||||
|
@ -933,6 +935,89 @@ export default async function build(
|
|||
})
|
||||
)
|
||||
|
||||
let turboTasks: unknown
|
||||
|
||||
if (turbotraceContext) {
|
||||
let binding = (await loadBindings()) as any
|
||||
if (
|
||||
!binding?.isWasm &&
|
||||
typeof binding.turbo.startTrace === 'function'
|
||||
) {
|
||||
let turbotraceOutputPath: string | undefined
|
||||
let turbotraceFiles: string[] | undefined
|
||||
turboTasks = binding.turbo.createTurboTasks(
|
||||
config.experimental.turbotrace?.memoryLimit ??
|
||||
TURBO_TRACE_DEFAULT_MEMORY_LIMIT
|
||||
)
|
||||
|
||||
const { entriesTrace, chunksTrace } = turbotraceContext
|
||||
if (entriesTrace) {
|
||||
const {
|
||||
appDir: turbotraceContextAppDir,
|
||||
depModArray,
|
||||
entryNameMap,
|
||||
outputPath,
|
||||
action,
|
||||
} = entriesTrace
|
||||
const depModSet = new Set(depModArray)
|
||||
const filesTracedInEntries: string[] =
|
||||
await binding.turbo.startTrace(action, turboTasks)
|
||||
|
||||
const { contextDirectory, input: entriesToTrace } = action
|
||||
|
||||
// only trace the assets under the appDir
|
||||
// exclude files from node_modules, entries and processed by webpack
|
||||
const filesTracedFromEntries = filesTracedInEntries
|
||||
.map((f) => path.join(contextDirectory, f))
|
||||
.filter(
|
||||
(f) =>
|
||||
!f.includes('/node_modules/') &&
|
||||
f.startsWith(turbotraceContextAppDir) &&
|
||||
!entriesToTrace.includes(f) &&
|
||||
!depModSet.has(f)
|
||||
)
|
||||
if (filesTracedFromEntries.length) {
|
||||
// The turbo trace doesn't provide the traced file type and reason at present
|
||||
// let's write the traced files into the first [entry].nft.json
|
||||
const [[, entryName]] = Array.from(entryNameMap.entries()).filter(
|
||||
([k]) => k.startsWith(turbotraceContextAppDir)
|
||||
)
|
||||
const traceOutputPath = path.join(
|
||||
outputPath,
|
||||
`../${entryName}.js.nft.json`
|
||||
)
|
||||
const traceOutputDir = path.dirname(traceOutputPath)
|
||||
|
||||
turbotraceOutputPath = traceOutputPath
|
||||
turbotraceFiles = filesTracedFromEntries.map((file) =>
|
||||
path.relative(traceOutputDir, file)
|
||||
)
|
||||
}
|
||||
}
|
||||
if (chunksTrace) {
|
||||
const { action } = chunksTrace
|
||||
await binding.turbo.startTrace(action, turboTasks)
|
||||
if (turbotraceOutputPath && turbotraceFiles) {
|
||||
const existedNftFile = await promises
|
||||
.readFile(turbotraceOutputPath, 'utf8')
|
||||
.then((existedContent) => JSON.parse(existedContent))
|
||||
.catch(() => ({
|
||||
version: TRACE_OUTPUT_VERSION,
|
||||
files: [],
|
||||
}))
|
||||
existedNftFile.files.push(...turbotraceFiles)
|
||||
const filesSet = new Set(existedNftFile.files)
|
||||
existedNftFile.files = [...filesSet]
|
||||
await promises.writeFile(
|
||||
turbotraceOutputPath,
|
||||
JSON.stringify(existedNftFile),
|
||||
'utf8'
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// For app directory, we run type checking after build.
|
||||
if (appDir) {
|
||||
await startTypeChecking()
|
||||
|
@ -1574,7 +1659,7 @@ export default async function build(
|
|||
if (config.experimental.turbotrace) {
|
||||
let binding = (await loadBindings()) as any
|
||||
if (!binding?.isWasm) {
|
||||
nodeFileTrace = binding.turbo?.startTrace
|
||||
nodeFileTrace = binding.turbo.startTrace
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1714,7 +1799,10 @@ export default async function build(
|
|||
config.experimental?.turbotrace?.contextDirectory ??
|
||||
config.experimental?.outputFileTracingRoot ??
|
||||
dir
|
||||
const toTrace = [require.resolve('next/dist/server/next-server')]
|
||||
const nextServerEntry = require.resolve(
|
||||
'next/dist/server/next-server'
|
||||
)
|
||||
const toTrace = [nextServerEntry]
|
||||
|
||||
// ensure we trace any dependencies needed for custom
|
||||
// incremental cache handler
|
||||
|
@ -1725,22 +1813,9 @@ export default async function build(
|
|||
}
|
||||
|
||||
let serverResult: import('next/dist/compiled/@vercel/nft').NodeFileTraceResult
|
||||
if (config.experimental.turbotrace) {
|
||||
// handle the cache in the turbo-tracing side in the future
|
||||
await nodeFileTrace({
|
||||
action: 'annotate',
|
||||
input: toTrace,
|
||||
contextDirectory: root,
|
||||
logLevel: config.experimental.turbotrace.logLevel,
|
||||
processCwd: config.experimental.turbotrace.processCwd,
|
||||
logDetail: config.experimental.turbotrace.logDetail,
|
||||
showAll: config.experimental.turbotrace.logAll,
|
||||
memoryLimit:
|
||||
config.experimental.turbotrace.memoryLimit ??
|
||||
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
|
||||
})
|
||||
} else {
|
||||
const ignores = [
|
||||
'**/*.d.ts',
|
||||
'**/*.map',
|
||||
'**/next/dist/pages/**/*',
|
||||
'**/next/dist/compiled/webpack/(bundle4|bundle5).js',
|
||||
'**/node_modules/webpack5/**/*',
|
||||
|
@ -1762,12 +1837,36 @@ export default async function build(
|
|||
const ignoreFn = (pathname: string) => {
|
||||
return isMatch(pathname, ignores, { contains: true, dot: true })
|
||||
}
|
||||
const traceContext = path.join(nextServerEntry, '..', '..')
|
||||
const tracedFiles = new Set<string>()
|
||||
if (config.experimental.turbotrace) {
|
||||
const files: string[] = await nodeFileTrace(
|
||||
{
|
||||
action: 'print',
|
||||
input: toTrace,
|
||||
contextDirectory: traceContext,
|
||||
logLevel: config.experimental.turbotrace.logLevel,
|
||||
processCwd: config.experimental.turbotrace.processCwd,
|
||||
logDetail: config.experimental.turbotrace.logDetail,
|
||||
showAll: config.experimental.turbotrace.logAll,
|
||||
},
|
||||
turboTasks
|
||||
)
|
||||
for (const file of files) {
|
||||
if (!ignoreFn(path.join(traceContext, file))) {
|
||||
tracedFiles.add(
|
||||
path
|
||||
.relative(distDir, path.join(traceContext, file))
|
||||
.replace(/\\/g, '/')
|
||||
)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
serverResult = await nodeFileTrace(toTrace, {
|
||||
base: root,
|
||||
processCwd: dir,
|
||||
ignore: ignoreFn,
|
||||
})
|
||||
const tracedFiles = new Set()
|
||||
|
||||
serverResult.fileList.forEach((file) => {
|
||||
tracedFiles.add(
|
||||
|
@ -1776,13 +1875,13 @@ export default async function build(
|
|||
.replace(/\\/g, '/')
|
||||
)
|
||||
})
|
||||
|
||||
}
|
||||
await promises.writeFile(
|
||||
nextServerTraceOutput,
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
cacheKey,
|
||||
files: [...tracedFiles],
|
||||
files: Array.from(tracedFiles),
|
||||
} as {
|
||||
version: number
|
||||
files: string[]
|
||||
|
@ -1792,7 +1891,6 @@ export default async function build(
|
|||
await promises
|
||||
.copyFile(nextServerTraceOutput, cachedTracePath)
|
||||
.catch(() => {})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
@ -20,6 +20,10 @@ import { createEntrypoints } from './entries'
|
|||
import loadConfig from '../server/config'
|
||||
import { trace } from '../trace'
|
||||
import { WEBPACK_LAYERS } from '../lib/constants'
|
||||
import {
|
||||
TraceEntryPointsPlugin,
|
||||
TurbotraceContext,
|
||||
} from './webpack/plugins/next-trace-entrypoints-plugin'
|
||||
|
||||
type CompilerResult = {
|
||||
errors: webpack.StatsError[]
|
||||
|
@ -37,7 +41,16 @@ function isTelemetryPlugin(plugin: unknown): plugin is TelemetryPlugin {
|
|||
return plugin instanceof TelemetryPlugin
|
||||
}
|
||||
|
||||
async function webpackBuildImpl(): Promise<number> {
|
||||
function isTraceEntryPointsPlugin(
|
||||
plugin: unknown
|
||||
): plugin is TraceEntryPointsPlugin {
|
||||
return plugin instanceof TraceEntryPointsPlugin
|
||||
}
|
||||
|
||||
async function webpackBuildImpl(): Promise<{
|
||||
duration: number
|
||||
turbotraceContext?: TurbotraceContext
|
||||
}> {
|
||||
let result: CompilerResult | null = {
|
||||
warnings: [],
|
||||
errors: [],
|
||||
|
@ -117,6 +130,7 @@ async function webpackBuildImpl(): Promise<number> {
|
|||
})
|
||||
|
||||
const clientConfig = configs[0]
|
||||
const serverConfig = configs[1]
|
||||
|
||||
if (
|
||||
clientConfig.optimization &&
|
||||
|
@ -141,7 +155,7 @@ async function webpackBuildImpl(): Promise<number> {
|
|||
// injected to this set and then will be consumed by the client compiler.
|
||||
injectedClientEntries.clear()
|
||||
|
||||
const serverResult = await runCompiler(configs[1], {
|
||||
const serverResult = await runCompiler(serverConfig, {
|
||||
runWebpackSpan,
|
||||
})
|
||||
const edgeServerResult = configs[2]
|
||||
|
@ -206,6 +220,10 @@ async function webpackBuildImpl(): Promise<number> {
|
|||
clientConfig as webpack.Configuration
|
||||
).plugins?.find(isTelemetryPlugin)
|
||||
|
||||
const traceEntryPointsPlugin = (
|
||||
serverConfig as webpack.Configuration
|
||||
).plugins?.find(isTraceEntryPointsPlugin)
|
||||
|
||||
const webpackBuildEnd = process.hrtime(webpackBuildStart)
|
||||
if (buildSpinner) {
|
||||
buildSpinner.stopAndPersist()
|
||||
|
@ -257,7 +275,10 @@ async function webpackBuildImpl(): Promise<number> {
|
|||
} else {
|
||||
Log.info('Compiled successfully')
|
||||
}
|
||||
return webpackBuildEnd[0]
|
||||
return {
|
||||
duration: webpackBuildEnd[0],
|
||||
turbotraceContext: traceEntryPointsPlugin?.turbotraceContext,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -279,7 +300,19 @@ async function workerMain() {
|
|||
|
||||
try {
|
||||
const result = await webpackBuildImpl()
|
||||
parentPort!.postMessage(result)
|
||||
const { entriesTrace } = result.turbotraceContext ?? {}
|
||||
if (entriesTrace) {
|
||||
const { entryNameMap, depModArray } = entriesTrace
|
||||
if (depModArray) {
|
||||
result.turbotraceContext!.entriesTrace!.depModArray = depModArray
|
||||
}
|
||||
if (entryNameMap) {
|
||||
const entryEntries = Array.from(entryNameMap?.entries() ?? [])
|
||||
// @ts-expect-error
|
||||
result.turbotraceContext.entriesTrace.entryNameMap = entryEntries
|
||||
}
|
||||
}
|
||||
parentPort!.postMessage(JSON.stringify(result))
|
||||
} catch (e) {
|
||||
parentPort!.postMessage(e)
|
||||
} finally {
|
||||
|
@ -305,8 +338,15 @@ async function webpackBuildWithWorker() {
|
|||
},
|
||||
})
|
||||
|
||||
const result = await new Promise((resolve, reject) => {
|
||||
worker.on('message', resolve)
|
||||
const result = JSON.parse(
|
||||
await new Promise((resolve, reject) => {
|
||||
worker.on('message', (data) => {
|
||||
if (data instanceof Error) {
|
||||
reject(data)
|
||||
} else {
|
||||
resolve(data)
|
||||
}
|
||||
})
|
||||
worker.on('error', reject)
|
||||
worker.on('exit', (code) => {
|
||||
if (code !== 0) {
|
||||
|
@ -314,8 +354,19 @@ async function webpackBuildWithWorker() {
|
|||
}
|
||||
})
|
||||
})
|
||||
) as {
|
||||
duration: number
|
||||
turbotraceContext?: TurbotraceContext
|
||||
}
|
||||
|
||||
return result as number
|
||||
if (result.turbotraceContext?.entriesTrace) {
|
||||
const { entryNameMap } = result.turbotraceContext.entriesTrace
|
||||
if (entryNameMap) {
|
||||
result.turbotraceContext.entriesTrace.entryNameMap = new Map(entryNameMap)
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
export async function webpackBuild() {
|
||||
|
|
|
@ -1,5 +1,4 @@
|
|||
import nodePath from 'path'
|
||||
import nodeFs from 'fs'
|
||||
import { Span } from '../../../trace'
|
||||
import { spans } from './profiling-plugin'
|
||||
import isError from '../../../lib/is-error'
|
||||
|
@ -7,10 +6,7 @@ import {
|
|||
nodeFileTrace,
|
||||
NodeFileTraceReasons,
|
||||
} from 'next/dist/compiled/@vercel/nft'
|
||||
import {
|
||||
TRACE_OUTPUT_VERSION,
|
||||
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
|
||||
} from '../../../shared/lib/constants'
|
||||
import { TRACE_OUTPUT_VERSION } from '../../../shared/lib/constants'
|
||||
import { webpack, sources } from 'next/dist/compiled/webpack/webpack'
|
||||
import {
|
||||
NODE_ESM_RESOLVE_OPTIONS,
|
||||
|
@ -40,8 +36,6 @@ const NOT_TRACEABLE = [
|
|||
'.svg',
|
||||
]
|
||||
|
||||
const TURBO_TRACE_DEFAULT_MAX_FILES = 128
|
||||
|
||||
function getModuleFromDependency(
|
||||
compilation: any,
|
||||
dep: any
|
||||
|
@ -103,7 +97,34 @@ function getFilesMapFromReasons(
|
|||
return parentFilesMap
|
||||
}
|
||||
|
||||
export interface TurbotraceAction {
|
||||
action: 'print' | 'annotate'
|
||||
input: string[]
|
||||
contextDirectory: string
|
||||
processCwd: string
|
||||
logLevel?: NonNullable<
|
||||
NextConfigComplete['experimental']['turbotrace']
|
||||
>['logLevel']
|
||||
showAll?: boolean
|
||||
memoryLimit?: number
|
||||
}
|
||||
|
||||
export interface TurbotraceContext {
|
||||
entriesTrace?: {
|
||||
action: TurbotraceAction
|
||||
appDir: string
|
||||
outputPath: string
|
||||
depModArray: string[]
|
||||
entryNameMap: Map<string, string>
|
||||
}
|
||||
chunksTrace?: {
|
||||
action: TurbotraceAction
|
||||
}
|
||||
}
|
||||
|
||||
export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
|
||||
public turbotraceContext: TurbotraceContext = {}
|
||||
|
||||
private appDir: string
|
||||
private appDirEnabled?: boolean
|
||||
private tracingRoot: string
|
||||
|
@ -112,8 +133,6 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
|
|||
private esmExternals?: NextConfigComplete['experimental']['esmExternals']
|
||||
private turbotrace?: NextConfigComplete['experimental']['turbotrace']
|
||||
private chunksToTrace: string[] = []
|
||||
private turbotraceOutputPath?: string
|
||||
private turbotraceFiles?: string[]
|
||||
|
||||
constructor({
|
||||
appDir,
|
||||
|
@ -184,7 +203,7 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
|
|||
}
|
||||
|
||||
// startTrace existed and callable
|
||||
if (this.turbotrace && !this.turbotrace.skipEntries) {
|
||||
if (this.turbotrace) {
|
||||
let binding = (await loadBindings()) as any
|
||||
if (
|
||||
!binding?.isWasm &&
|
||||
|
@ -410,75 +429,24 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
|
|||
!binding?.isWasm &&
|
||||
typeof binding.turbo.startTrace === 'function'
|
||||
) {
|
||||
await finishModulesSpan
|
||||
.traceChild('turbo-trace', {
|
||||
traceEntryCount: entriesToTrace.length + '',
|
||||
})
|
||||
.traceAsyncFn(async () => {
|
||||
const contextDirectory =
|
||||
this.turbotrace?.contextDirectory ?? this.tracingRoot
|
||||
const maxFiles =
|
||||
this.turbotrace?.maxFiles ?? TURBO_TRACE_DEFAULT_MAX_FILES
|
||||
let chunks = [...entriesToTrace]
|
||||
let restChunks =
|
||||
chunks.length > maxFiles ? chunks.splice(maxFiles) : []
|
||||
let filesTracedInEntries: string[] = []
|
||||
while (chunks.length) {
|
||||
filesTracedInEntries = filesTracedInEntries.concat(
|
||||
await binding.turbo.startTrace({
|
||||
const chunks = [...entriesToTrace]
|
||||
|
||||
this.turbotraceContext.entriesTrace = {
|
||||
action: {
|
||||
action: 'print',
|
||||
input: chunks,
|
||||
contextDirectory,
|
||||
processCwd:
|
||||
this.turbotrace?.processCwd ?? this.appDir,
|
||||
processCwd: this.turbotrace?.processCwd ?? this.appDir,
|
||||
logLevel: this.turbotrace?.logLevel,
|
||||
showAll: this.turbotrace?.logAll,
|
||||
memoryLimit:
|
||||
this.turbotrace?.memoryLimit ??
|
||||
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
|
||||
})
|
||||
)
|
||||
chunks = restChunks
|
||||
if (restChunks.length) {
|
||||
restChunks =
|
||||
chunks.length > maxFiles
|
||||
? chunks.splice(maxFiles)
|
||||
: []
|
||||
},
|
||||
appDir: this.appDir,
|
||||
depModArray: Array.from(depModMap.keys()),
|
||||
entryNameMap,
|
||||
outputPath: compilation.outputOptions.path!,
|
||||
}
|
||||
}
|
||||
|
||||
// only trace the assets under the appDir
|
||||
// exclude files from node_modules, entries and processed by webpack
|
||||
const filesTracedFromEntries = filesTracedInEntries
|
||||
.map((f) => nodePath.join(contextDirectory, f))
|
||||
.filter(
|
||||
(f) =>
|
||||
!f.includes('/node_modules/') &&
|
||||
f.startsWith(this.appDir) &&
|
||||
!entriesToTrace.includes(f) &&
|
||||
!depModMap.has(f)
|
||||
)
|
||||
if (!filesTracedFromEntries.length) {
|
||||
return
|
||||
}
|
||||
|
||||
// The turbo trace doesn't provide the traced file type and reason at present
|
||||
// let's write the traced files into the first [entry].nft.json
|
||||
const [[, entryName]] = Array.from(
|
||||
entryNameMap.entries()
|
||||
).filter(([k]) => k.startsWith(this.appDir))
|
||||
const outputPath = compilation.outputOptions.path!
|
||||
const traceOutputPath = nodePath.join(
|
||||
outputPath,
|
||||
`../${entryName}.js.nft.json`
|
||||
)
|
||||
const traceOutputDir = nodePath.dirname(traceOutputPath)
|
||||
|
||||
this.turbotraceOutputPath = traceOutputPath
|
||||
this.turbotraceFiles = filesTracedFromEntries.map((file) =>
|
||||
nodePath.relative(traceOutputDir, file)
|
||||
)
|
||||
})
|
||||
return
|
||||
}
|
||||
}
|
||||
|
@ -804,32 +772,21 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
|
|||
})
|
||||
|
||||
if (this.turbotrace) {
|
||||
compiler.hooks.afterEmit.tapPromise(PLUGIN_NAME, async (compilation) => {
|
||||
const compilationSpan = spans.get(compilation) || spans.get(compiler)!
|
||||
const traceEntrypointsPluginSpan = compilationSpan.traceChild(
|
||||
'next-trace-entrypoint-plugin'
|
||||
)
|
||||
const turbotraceAfterEmitSpan = traceEntrypointsPluginSpan.traceChild(
|
||||
'after-emit-turbo-trace'
|
||||
)
|
||||
await turbotraceAfterEmitSpan.traceAsyncFn(async () => {
|
||||
compiler.hooks.afterEmit.tapPromise(PLUGIN_NAME, async () => {
|
||||
let binding = (await loadBindings()) as any
|
||||
if (
|
||||
!binding?.isWasm &&
|
||||
typeof binding.turbo.startTrace === 'function'
|
||||
) {
|
||||
const maxFiles =
|
||||
this.turbotrace?.maxFiles ?? TURBO_TRACE_DEFAULT_MAX_FILES
|
||||
const ignores = [...TRACE_IGNORES, ...this.traceIgnores]
|
||||
|
||||
const ignoreFn = (path: string) => {
|
||||
return isMatch(path, ignores, { contains: true, dot: true })
|
||||
}
|
||||
let chunks = this.chunksToTrace.filter((chunk) => !ignoreFn(chunk))
|
||||
let restChunks =
|
||||
chunks.length > maxFiles ? chunks.splice(maxFiles) : []
|
||||
while (chunks.length) {
|
||||
await binding.turbo.startTrace({
|
||||
const chunks = this.chunksToTrace.filter((chunk) => !ignoreFn(chunk))
|
||||
|
||||
this.turbotraceContext.chunksTrace = {
|
||||
action: {
|
||||
action: 'annotate',
|
||||
input: chunks,
|
||||
contextDirectory:
|
||||
|
@ -837,36 +794,10 @@ export class TraceEntryPointsPlugin implements webpack.WebpackPluginInstance {
|
|||
processCwd: this.turbotrace?.processCwd ?? this.appDir,
|
||||
showAll: this.turbotrace?.logAll,
|
||||
logLevel: this.turbotrace?.logLevel,
|
||||
memoryLimit:
|
||||
this.turbotrace?.memoryLimit ??
|
||||
TURBO_TRACE_DEFAULT_MEMORY_LIMIT,
|
||||
})
|
||||
chunks = restChunks
|
||||
if (restChunks.length) {
|
||||
restChunks =
|
||||
chunks.length > maxFiles ? chunks.splice(maxFiles) : []
|
||||
}
|
||||
}
|
||||
if (this.turbotraceOutputPath && this.turbotraceFiles) {
|
||||
const existedNftFile = await nodeFs.promises
|
||||
.readFile(this.turbotraceOutputPath, 'utf8')
|
||||
.then((content) => JSON.parse(content))
|
||||
.catch(() => ({
|
||||
version: TRACE_OUTPUT_VERSION,
|
||||
files: [],
|
||||
}))
|
||||
|
||||
existedNftFile.files.push(...this.turbotraceFiles)
|
||||
const filesSet = new Set(existedNftFile.files)
|
||||
existedNftFile.files = [...filesSet]
|
||||
nodeFs.promises.writeFile(
|
||||
this.turbotraceOutputPath,
|
||||
JSON.stringify(existedNftFile)
|
||||
)
|
||||
},
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -481,15 +481,9 @@ const configSchema = {
|
|||
processCwd: {
|
||||
type: 'string',
|
||||
},
|
||||
maxFiles: {
|
||||
type: 'integer',
|
||||
},
|
||||
memoryLimit: {
|
||||
type: 'integer',
|
||||
},
|
||||
skipEntries: {
|
||||
type: 'boolean',
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
|
@ -206,9 +206,8 @@ export interface ExperimentalConfig {
|
|||
logAll?: boolean
|
||||
contextDirectory?: string
|
||||
processCwd?: string
|
||||
maxFiles?: number
|
||||
/** in `MB` */
|
||||
memoryLimit?: number
|
||||
skipEntries?: boolean
|
||||
}
|
||||
mdxRs?: boolean
|
||||
/**
|
||||
|
|
|
@ -508,7 +508,7 @@ importers:
|
|||
'@babel/types': 7.18.0
|
||||
'@edge-runtime/primitives': 2.0.0
|
||||
'@hapi/accept': 5.0.2
|
||||
'@napi-rs/cli': 2.13.3
|
||||
'@napi-rs/cli': 2.14.7
|
||||
'@napi-rs/triples': 1.1.0
|
||||
'@next/env': 13.1.7-canary.8
|
||||
'@next/polyfill-module': 13.1.7-canary.8
|
||||
|
@ -718,7 +718,7 @@ importers:
|
|||
'@babel/types': 7.18.0
|
||||
'@edge-runtime/primitives': 2.0.0
|
||||
'@hapi/accept': 5.0.2
|
||||
'@napi-rs/cli': 2.13.3
|
||||
'@napi-rs/cli': 2.14.7
|
||||
'@napi-rs/triples': 1.1.0
|
||||
'@next/polyfill-module': link:../next-polyfill-module
|
||||
'@next/polyfill-nomodule': link:../next-polyfill-nomodule
|
||||
|
@ -961,9 +961,9 @@ importers:
|
|||
|
||||
packages/next-swc:
|
||||
specifiers:
|
||||
'@napi-rs/cli': 2.13.3
|
||||
'@napi-rs/cli': 2.14.7
|
||||
devDependencies:
|
||||
'@napi-rs/cli': 2.13.3
|
||||
'@napi-rs/cli': 2.14.7
|
||||
|
||||
packages/react-dev-overlay:
|
||||
specifiers:
|
||||
|
@ -5912,8 +5912,8 @@ packages:
|
|||
glob-to-regexp: 0.3.0
|
||||
dev: true
|
||||
|
||||
/@napi-rs/cli/2.13.3:
|
||||
resolution: {integrity: sha512-nAlbKuakQ+YHZE+M3Afih9UA1jr+gx63Gt4xHA+j2xD1NY6TjQ+QCgF9Yaj/YZIkCc2t3CZh52znFrfbU8b2bA==}
|
||||
/@napi-rs/cli/2.14.7:
|
||||
resolution: {integrity: sha512-H+YCgdbBNMWn8c878C3RsRYcMxamWI7rB0H+1TqOy9n6u31isikcRyW1VDyoNHp7ApfmzV0+tpPopVi0W7WHAA==}
|
||||
engines: {node: '>= 10'}
|
||||
hasBin: true
|
||||
dev: true
|
||||
|
|
|
@ -6,8 +6,7 @@ import { nextBuild } from 'next-test-utils'
|
|||
|
||||
const appDir = join(__dirname, '../app')
|
||||
|
||||
// TODO: investigate failure from incorrect exit code
|
||||
describe.skip('build trace with extra entries', () => {
|
||||
describe('build trace with extra entries', () => {
|
||||
it('should build and trace correctly', async () => {
|
||||
const result = await nextBuild(appDir, undefined, {
|
||||
cwd: appDir,
|
||||
|
|
|
@ -0,0 +1,3 @@
|
|||
{
|
||||
"hello": "world"
|
||||
}
|
|
@ -0,0 +1 @@
|
|||
first
|
|
@ -0,0 +1 @@
|
|||
second
|
|
@ -0,0 +1,12 @@
|
|||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
|
||||
const getCmsData = require('some-cms')
|
||||
|
||||
try {
|
||||
fs.readdirSync(path.join(process.cwd(), 'public/exclude-me'))
|
||||
} catch (_) {}
|
||||
|
||||
export function fetchData() {
|
||||
return getCmsData()
|
||||
}
|
|
@ -0,0 +1,8 @@
|
|||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
|
||||
export function getData() {
|
||||
return JSON.parse(
|
||||
fs.readFileSync(path.join(process.cwd(), 'content/hello.json'), 'utf8')
|
||||
)
|
||||
}
|
|
@ -0,0 +1,29 @@
|
|||
const path = require('path')
|
||||
|
||||
module.exports = {
|
||||
webpack(cfg, { isServer, nextRuntime }) {
|
||||
console.log(cfg.entry)
|
||||
const origEntry = cfg.entry
|
||||
cfg.entry = async () => {
|
||||
const origEntries = await origEntry()
|
||||
|
||||
if (isServer && nextRuntime === 'nodejs') {
|
||||
const curEntry = origEntries['pages/_app']
|
||||
origEntries['pages/_app'] = [
|
||||
path.join(__dirname, 'lib/get-data.js'),
|
||||
...curEntry,
|
||||
]
|
||||
console.log(origEntries)
|
||||
}
|
||||
return origEntries
|
||||
}
|
||||
return cfg
|
||||
},
|
||||
experimental: {
|
||||
turbotrace: {
|
||||
contextDirectory: path.join(__dirname, '..', '..', '..', '..'),
|
||||
memoryLimit: 4096,
|
||||
},
|
||||
webpackBuildWorker: true,
|
||||
},
|
||||
}
|
4
test/integration/turbotrace-with-webpack-worker/app/node_modules/nested-structure/constants/package.json
generated
vendored
Normal file
4
test/integration/turbotrace-with-webpack-worker/app/node_modules/nested-structure/constants/package.json
generated
vendored
Normal file
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"name": "constants",
|
||||
"main": "../lib/constants"
|
||||
}
|
0
test/integration/turbotrace-with-webpack-worker/app/node_modules/nested-structure/lib/constants.js
generated
vendored
Normal file
0
test/integration/turbotrace-with-webpack-worker/app/node_modules/nested-structure/lib/constants.js
generated
vendored
Normal file
0
test/integration/turbotrace-with-webpack-worker/app/node_modules/nested-structure/lib/index.js
generated
vendored
Normal file
0
test/integration/turbotrace-with-webpack-worker/app/node_modules/nested-structure/lib/index.js
generated
vendored
Normal file
4
test/integration/turbotrace-with-webpack-worker/app/node_modules/nested-structure/package.json
generated
vendored
Normal file
4
test/integration/turbotrace-with-webpack-worker/app/node_modules/nested-structure/package.json
generated
vendored
Normal file
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"name": "nested-structure",
|
||||
"main": "./lib/index.js"
|
||||
}
|
5
test/integration/turbotrace-with-webpack-worker/app/node_modules/some-cms/index.js
generated
vendored
Normal file
5
test/integration/turbotrace-with-webpack-worker/app/node_modules/some-cms/index.js
generated
vendored
Normal file
|
@ -0,0 +1,5 @@
|
|||
function getCmsData() {
|
||||
return 'hello'
|
||||
}
|
||||
|
||||
module.exports = getCmsData
|
4
test/integration/turbotrace-with-webpack-worker/app/node_modules/some-cms/package.json
generated
vendored
Normal file
4
test/integration/turbotrace-with-webpack-worker/app/node_modules/some-cms/package.json
generated
vendored
Normal file
|
@ -0,0 +1,4 @@
|
|||
{
|
||||
"name": "some-cms",
|
||||
"main": "./index.js"
|
||||
}
|
|
@ -0,0 +1,11 @@
|
|||
import 'nested-structure/constants'
|
||||
|
||||
export default function Page() {
|
||||
return <p>another page</p>
|
||||
}
|
||||
|
||||
export function getStaticProps() {
|
||||
return {
|
||||
props: {},
|
||||
}
|
||||
}
|
|
@ -0,0 +1,24 @@
|
|||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import Image from 'next/image'
|
||||
import testImage from '../public/test.jpg'
|
||||
|
||||
export default function Page(props) {
|
||||
return (
|
||||
<div>
|
||||
<Image src={testImage} placeholder="blur" alt="test" />
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export function getServerSideProps() {
|
||||
try {
|
||||
// this should be included in the trace since it's not an
|
||||
// import
|
||||
fs.readFileSync(path.join(process.cwd(), 'public/another.jpg'))
|
||||
} catch (_) {}
|
||||
|
||||
return {
|
||||
props: {},
|
||||
}
|
||||
}
|
|
@ -0,0 +1,18 @@
|
|||
import { fetchData } from '../lib/fetch-data'
|
||||
|
||||
export const config = {
|
||||
unstable_includeFiles: ['include-me/*'],
|
||||
unstable_excludeFiles: ['public/exclude-me/**/*'],
|
||||
}
|
||||
|
||||
export default function Page() {
|
||||
return 'index page'
|
||||
}
|
||||
|
||||
export function getStaticProps() {
|
||||
fetchData()
|
||||
|
||||
return {
|
||||
props: {},
|
||||
}
|
||||
}
|
Binary file not shown.
After Width: | Height: | Size: 6.6 KiB |
|
@ -0,0 +1 @@
|
|||
second
|
|
@ -0,0 +1 @@
|
|||
first
|
Binary file not shown.
After Width: | Height: | Size: 6.6 KiB |
|
@ -0,0 +1,73 @@
|
|||
/* eslint-env jest */
|
||||
|
||||
import fs from 'fs-extra'
|
||||
import { join } from 'path'
|
||||
import { nextBuild } from 'next-test-utils'
|
||||
|
||||
const appDir = join(__dirname, '../app')
|
||||
|
||||
describe('build trace with extra entries', () => {
|
||||
it('should build and trace correctly', async () => {
|
||||
const result = await nextBuild(appDir, undefined, {
|
||||
cwd: appDir,
|
||||
stderr: true,
|
||||
stdout: true,
|
||||
})
|
||||
console.log(result)
|
||||
expect(result.code).toBe(0)
|
||||
|
||||
const appTrace = await fs.readJSON(
|
||||
join(appDir, '.next/server/pages/_app.js.nft.json')
|
||||
)
|
||||
const indexTrace = await fs.readJSON(
|
||||
join(appDir, '.next/server/pages/index.js.nft.json')
|
||||
)
|
||||
const anotherTrace = await fs.readJSON(
|
||||
join(appDir, '.next/server/pages/another.js.nft.json')
|
||||
)
|
||||
const imageTrace = await fs.readJSON(
|
||||
join(appDir, '.next/server/pages/image-import.js.nft.json')
|
||||
)
|
||||
|
||||
const tracedFiles = [
|
||||
...appTrace.files,
|
||||
...indexTrace.files,
|
||||
...anotherTrace.files,
|
||||
...imageTrace.files,
|
||||
]
|
||||
|
||||
expect(tracedFiles.some((file) => file.endsWith('hello.json'))).toBe(true)
|
||||
expect(tracedFiles.some((file) => file.includes('some-cms/index.js'))).toBe(
|
||||
true
|
||||
)
|
||||
expect(
|
||||
tracedFiles.some((file) => file === '../../../include-me/hello.txt')
|
||||
).toBe(true)
|
||||
expect(
|
||||
tracedFiles.some((file) => file === '../../../include-me/second.txt')
|
||||
).toBe(true)
|
||||
expect(indexTrace.files.some((file) => file.includes('exclude-me'))).toBe(
|
||||
false
|
||||
)
|
||||
|
||||
expect(
|
||||
tracedFiles.some((file) =>
|
||||
file.includes('nested-structure/constants/package.json')
|
||||
)
|
||||
).toBe(true)
|
||||
expect(
|
||||
tracedFiles.some((file) => file.includes('nested-structure/package.json'))
|
||||
).toBe(true)
|
||||
expect(
|
||||
tracedFiles.some((file) =>
|
||||
file.includes('nested-structure/lib/constants.js')
|
||||
)
|
||||
).toBe(true)
|
||||
expect(
|
||||
tracedFiles.some((file) => file.includes('public/another.jpg'))
|
||||
).toBe(true)
|
||||
expect(tracedFiles.some((file) => file.includes('public/test.jpg'))).toBe(
|
||||
false
|
||||
)
|
||||
})
|
||||
})
|
Loading…
Reference in a new issue