use a shared worker pool for collecting page data and static page generation (#27924)

this avoid loading all code twice and hopefully improving performance
This commit is contained in:
Tobias Koppers 2021-08-12 21:54:49 +02:00 committed by GitHub
parent b6411408c0
commit 8bbb1cd353
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 297 additions and 238 deletions

View file

@ -9,10 +9,11 @@ When restarted it will retry all uncompleted jobs, but if a job was unsuccessful
#### Possible Ways to Fix It
- Make sure that there is no infinite loop during execution.
- Make sure all Promises in `getStaticProps` `resolve` or `reject` correctly.
- Make sure all Promises in `getStaticPaths`/`getStaticProps` `resolve` or `reject` correctly.
- Avoid very long timeouts for network requests.
- Increase the timeout by changing the `experimental.staticPageGenerationTimeout` configuration option (default `60` in seconds).
### Useful Links
- [`getStaticPaths`](https://nextjs.org/docs/basic-features/data-fetching#getstaticpaths-static-generation)
- [`getStaticProps`](https://nextjs.org/docs/basic-features/data-fetching#getstaticprops-static-generation)

View file

@ -91,8 +91,6 @@ import { normalizeLocalePath } from '../shared/lib/i18n/normalize-locale-path'
import { isWebpack5 } from 'next/dist/compiled/webpack/webpack'
import { NextConfigComplete } from '../server/config-shared'
const staticCheckWorker = require.resolve('./utils')
export type SsgRoute = {
initialRevalidateSeconds: number | false
srcRoute: string | null
@ -670,6 +668,62 @@ export default async function build(
await promises.readFile(buildManifestPath, 'utf8')
) as BuildManifest
const timeout = config.experimental.staticPageGenerationTimeout || 0
const sharedPool = config.experimental.sharedPool || false
const staticWorker = sharedPool
? require.resolve('./worker')
: require.resolve('./utils')
let infoPrinted = false
const staticWorkers = new Worker(staticWorker, {
timeout: timeout * 1000,
onRestart: (method, [arg], attempts) => {
if (method === 'exportPage') {
const { path: pagePath } = arg
if (attempts >= 3) {
throw new Error(
`Static page generation for ${pagePath} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout`
)
}
Log.warn(
`Restarted static page genertion for ${pagePath} because it took more than ${timeout} seconds`
)
} else {
const pagePath = arg
if (attempts >= 2) {
throw new Error(
`Collecting page data for ${pagePath} is still timing out after 2 attempts. See more info here https://nextjs.org/docs/messages/page-data-collection-timeout`
)
}
Log.warn(
`Restarted collecting page data for ${pagePath} because it took more than ${timeout} seconds`
)
}
if (!infoPrinted) {
Log.warn(
'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout'
)
infoPrinted = true
}
},
numWorkers: config.experimental.cpus,
enableWorkerThreads: config.experimental.workerThreads,
exposedMethods: sharedPool
? [
'hasCustomGetInitialProps',
'isPageStatic',
'getNamedExports',
'exportPage',
]
: ['hasCustomGetInitialProps', 'isPageStatic', 'getNamedExports'],
}) as Worker &
Pick<
typeof import('./worker'),
| 'hasCustomGetInitialProps'
| 'isPageStatic'
| 'getNamedExports'
| 'exportPage'
>
const analysisBegin = process.hrtime()
const staticCheckSpan = nextBuildSpan.traceChild('static-check')
@ -682,39 +736,6 @@ export default async function build(
} = await staticCheckSpan.traceAsyncFn(async () => {
process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
const timeout = config.experimental.pageDataCollectionTimeout || 0
let infoPrinted = false
const staticCheckWorkers = new Worker(staticCheckWorker, {
timeout: timeout * 1000,
onRestart: (_method, [pagePath], attempts) => {
if (attempts >= 2) {
throw new Error(
`Collecting page data for ${pagePath} is still timing out after 2 attempts. See more info here https://nextjs.org/docs/messages/page-data-collection-timeout`
)
}
Log.warn(
`Restarted collecting page data for ${pagePath} because it took more than ${timeout} seconds`
)
if (!infoPrinted) {
Log.warn(
'See more info here https://nextjs.org/docs/messages/page-data-collection-timeout'
)
infoPrinted = true
}
},
numWorkers: config.experimental.cpus,
enableWorkerThreads: config.experimental.workerThreads,
exposedMethods: [
'hasCustomGetInitialProps',
'isPageStatic',
'getNamedExports',
],
}) as Worker &
Pick<
typeof import('./utils'),
'hasCustomGetInitialProps' | 'isPageStatic' | 'getNamedExports'
>
const runtimeEnvConfig = {
publicRuntimeConfig: config.publicRuntimeConfig,
serverRuntimeConfig: config.serverRuntimeConfig,
@ -726,7 +747,7 @@ export default async function build(
const errorPageHasCustomGetInitialProps = nonStaticErrorPageSpan.traceAsyncFn(
async () =>
hasCustomErrorPage &&
(await staticCheckWorkers.hasCustomGetInitialProps(
(await staticWorkers.hasCustomGetInitialProps(
'/_error',
distDir,
isLikeServerless,
@ -738,7 +759,7 @@ export default async function build(
const errorPageStaticResult = nonStaticErrorPageSpan.traceAsyncFn(
async () =>
hasCustomErrorPage &&
staticCheckWorkers.isPageStatic(
staticWorkers.isPageStatic(
'/_error',
distDir,
isLikeServerless,
@ -753,7 +774,7 @@ export default async function build(
// from _error instead
const appPageToCheck = isLikeServerless ? '/_error' : '/_app'
const customAppGetInitialPropsPromise = staticCheckWorkers.hasCustomGetInitialProps(
const customAppGetInitialPropsPromise = staticWorkers.hasCustomGetInitialProps(
appPageToCheck,
distDir,
isLikeServerless,
@ -761,7 +782,7 @@ export default async function build(
true
)
const namedExportsPromise = staticCheckWorkers.getNamedExports(
const namedExportsPromise = staticWorkers.getNamedExports(
appPageToCheck,
distDir,
isLikeServerless,
@ -808,7 +829,7 @@ export default async function build(
'is-page-static'
)
let workerResult = await isPageStaticSpan.traceAsyncFn(() => {
return staticCheckWorkers.isPageStatic(
return staticWorkers.isPageStatic(
page,
distDir,
isLikeServerless,
@ -926,7 +947,7 @@ export default async function build(
hasNonStaticErrorPage: nonStaticErrorPage,
}
staticCheckWorkers.end()
if (!sharedPool) staticWorkers.end()
return returnValue
})
@ -1082,7 +1103,8 @@ export default async function build(
ssgPages,
additionalSsgPaths
)
const exportApp = require('../export').default
const exportApp: typeof import('../export').default = require('../export')
.default
const exportOptions = {
silent: false,
buildExport: true,
@ -1090,6 +1112,14 @@ export default async function build(
pages: combinedPages,
outdir: path.join(distDir, 'export'),
statusMessage: 'Generating static pages',
exportPageWorker: sharedPool
? staticWorkers.exportPage.bind(staticWorkers)
: undefined,
endWorker: sharedPool
? async () => {
await staticWorkers.end()
}
: undefined,
}
const exportConfig: any = {
...config,

View file

@ -0,0 +1,2 @@
export * from './utils'
export { default as exportPage } from '../export/worker'

View file

@ -137,6 +137,8 @@ interface ExportOptions {
pages?: string[]
buildExport?: boolean
statusMessage?: string
exportPageWorker?: typeof import('./worker').default
endWorker?: () => Promise<void>
}
export default async function exportApp(
@ -519,29 +521,40 @@ export default async function exportApp(
const timeout = configuration?.experimental.staticPageGenerationTimeout || 0
let infoPrinted = false
const worker = new Worker(require.resolve('./worker'), {
timeout: timeout * 1000,
onRestart: (_method, [{ path }], attempts) => {
if (attempts >= 3) {
throw new Error(
`Static page generation for ${path} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout`
)
}
Log.warn(
`Restarted static page genertion for ${path} because it took more than ${timeout} seconds`
)
if (!infoPrinted) {
let exportPage: typeof import('./worker').default
let endWorker: () => Promise<void>
if (options.exportPageWorker) {
exportPage = options.exportPageWorker
endWorker = options.endWorker || (() => Promise.resolve())
} else {
const worker = new Worker(require.resolve('./worker'), {
timeout: timeout * 1000,
onRestart: (_method, [{ path }], attempts) => {
if (attempts >= 3) {
throw new Error(
`Static page generation for ${path} is still timing out after 3 attempts. See more info here https://nextjs.org/docs/messages/static-page-generation-timeout`
)
}
Log.warn(
'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout'
`Restarted static page genertion for ${path} because it took more than ${timeout} seconds`
)
infoPrinted = true
}
},
maxRetries: 0,
numWorkers: threads,
enableWorkerThreads: nextConfig.experimental.workerThreads,
exposedMethods: ['default'],
}) as Worker & typeof import('./worker')
if (!infoPrinted) {
Log.warn(
'See more info here https://nextjs.org/docs/messages/static-page-generation-timeout'
)
infoPrinted = true
}
},
maxRetries: 0,
numWorkers: threads,
enableWorkerThreads: nextConfig.experimental.workerThreads,
exposedMethods: ['default'],
}) as Worker & typeof import('./worker')
exportPage = worker.default.bind(worker)
endWorker = async () => {
await worker.end()
}
}
let renderError = false
const errorPaths: string[] = []
@ -553,7 +566,7 @@ export default async function exportApp(
return pageExportSpan.traceAsyncFn(async () => {
const pathMap = exportPathMap[path]
const result = await worker.default({
const result = await exportPage({
path,
pathMap,
distDir,
@ -604,7 +617,7 @@ export default async function exportApp(
})
)
worker.end()
const endWorkerPromise = endWorker()
// copy prerendered routes to outDir
if (!options.buildExport && prerenderManifest) {
@ -681,5 +694,7 @@ export default async function exportApp(
if (telemetry) {
await telemetry.flush()
}
await endWorkerPromise
})
}

View file

@ -105,6 +105,7 @@ export type NextConfig = { [key: string]: any } & {
swcMinify?: boolean
swcLoader?: boolean
cpus?: number
sharedPool?: boolean
plugins?: boolean
profiling?: boolean
isrFlushToDisk?: boolean
@ -128,7 +129,6 @@ export type NextConfig = { [key: string]: any } & {
craCompat?: boolean
esmExternals?: boolean | 'loose'
staticPageGenerationTimeout?: number
pageDataCollectionTimeout?: number
isrMemoryCacheSize?: number
concurrentFeatures?: boolean
}
@ -184,6 +184,7 @@ export const defaultConfig: NextConfig = {
(Number(process.env.CIRCLE_NODE_TOTAL) ||
(os.cpus() || { length: 1 }).length) - 1
),
sharedPool: false,
plugins: false,
profiling: false,
isrFlushToDisk: true,
@ -200,7 +201,6 @@ export const defaultConfig: NextConfig = {
craCompat: false,
esmExternals: false,
staticPageGenerationTimeout: 60,
pageDataCollectionTimeout: 60,
// default to 50MB limit
isrMemoryCacheSize: 50 * 1024 * 1024,
concurrentFeatures: false,

View file

@ -13,182 +13,193 @@ const fixturesDir = join(__dirname, '..', 'fixtures')
const nextConfig = new File(join(fixturesDir, 'basic-app/next.config.js'))
describe('Build Output', () => {
for (const gzipSize of [true, false, undefined]) {
describe(
'Basic Application Output' +
(gzipSize !== undefined
? ` (with experimental.gzipSize: ${gzipSize})`
: ''),
() => {
let stdout
const appDir = join(fixturesDir, 'basic-app')
const configs = [{}]
for (const gzipSize of [true, false]) {
configs.push(...configs.map((c) => ({ ...c, gzipSize })))
}
for (const sharedPool of [true]) {
configs.push(...configs.map((c) => ({ ...c, sharedPool })))
}
for (const workerThreads of [true]) {
configs.push(...configs.map((c) => ({ ...c, workerThreads })))
}
beforeAll(async () => {
await remove(join(appDir, '.next'))
if (gzipSize !== undefined) {
nextConfig.write(
`module.exports = { experimental: { gzipSize: ${gzipSize} } };`
)
}
})
for (const experimental of configs) {
describe(`Basic Application Output (experimental: ${JSON.stringify(
experimental
)})`, () => {
let stdout
const appDir = join(fixturesDir, 'basic-app')
if (gzipSize !== undefined) {
afterAll(async () => {
nextConfig.delete()
})
const hasExperimentalConfig = Object.keys(experimental).length > 0
beforeAll(async () => {
await remove(join(appDir, '.next'))
if (hasExperimentalConfig) {
nextConfig.write(
`module.exports = { experimental: ${JSON.stringify(
experimental
)} };`
)
}
})
it('should not include internal pages', async () => {
;({ stdout } = await nextBuild(appDir, [], {
stdout: true,
}))
expect(stdout).toMatch(/\/ (.* )?\d{1,} B/)
expect(stdout).toMatch(/\+ First Load JS shared by all [ 0-9.]* kB/)
expect(stdout).toMatch(/ chunks\/main\.[0-9a-z]{6}\.js [ 0-9.]* kB/)
expect(stdout).toMatch(
/ chunks\/framework\.[0-9a-z]{6}\.js [ 0-9. ]* kB/
)
expect(stdout).not.toContain(' /_document')
expect(stdout).not.toContain(' /_app')
expect(stdout).not.toContain(' /_error')
expect(stdout).not.toContain('<buildId>')
expect(stdout).toContain('○ /')
})
it('should not deviate from snapshot', async () => {
console.log(stdout)
if (process.env.NEXT_PRIVATE_SKIP_SIZE_TESTS) {
return
}
const parsePageSize = (page) =>
stdout.match(
new RegExp(` ${page} .*?((?:\\d|\\.){1,} (?:\\w{1,})) `)
)[1]
const parsePageFirstLoad = (page) =>
stdout.match(
new RegExp(
` ${page} .*?(?:(?:\\d|\\.){1,}) .*? ((?:\\d|\\.){1,} (?:\\w{1,}))`
)
)[1]
const parseSharedSize = (sharedPartName) => {
const matches = stdout.match(
new RegExp(`${sharedPartName} .*? ((?:\\d|\\.){1,} (?:\\w{1,}))`)
)
if (!matches) {
throw new Error(`Could not match ${sharedPartName}`)
}
return matches[1]
}
const indexSize = parsePageSize('/')
const indexFirstLoad = parsePageFirstLoad('/')
const err404Size = parsePageSize('/404')
const err404FirstLoad = parsePageFirstLoad('/404')
const sharedByAll = parseSharedSize('shared by all')
const _appSize = parseSharedSize('_app\\..*?\\.js')
const webpackSize = parseSharedSize('webpack\\..*?\\.js')
const mainSize = parseSharedSize('main\\..*?\\.js')
const frameworkSize = parseSharedSize('framework\\..*?\\.js')
for (const size of [
indexSize,
indexFirstLoad,
err404Size,
err404FirstLoad,
sharedByAll,
_appSize,
webpackSize,
mainSize,
frameworkSize,
]) {
expect(parseFloat(size)).toBeGreaterThan(0)
}
// const gz = gzipSize !== false
// expect(parseFloat(indexSize) / 1000).toBeCloseTo(
// gz ? 0.251 : 0.394,
// 2
// )
expect(indexSize.endsWith('B')).toBe(true)
// expect(parseFloat(indexFirstLoad)).toBeCloseTo(gz ? 64 : 196, 1)
expect(indexFirstLoad.endsWith('kB')).toBe(true)
// expect(parseFloat(err404Size)).toBeCloseTo(gz ? 3.17 : 8.51, 1)
expect(err404Size.endsWith('B')).toBe(true)
// expect(parseFloat(err404FirstLoad)).toBeCloseTo(gz ? 66.9 : 204, 1)
expect(err404FirstLoad.endsWith('kB')).toBe(true)
// expect(parseFloat(sharedByAll)).toBeCloseTo(gz ? 63.7 : 196, 1)
expect(sharedByAll.endsWith('kB')).toBe(true)
// const appSizeValue = _appSize.endsWith('kB')
// ? parseFloat(_appSize)
// : parseFloat(_appSize) / 1000
// expect(appSizeValue).toBeCloseTo(gz ? 0.799 : 1.63, 1)
expect(_appSize.endsWith('kB') || _appSize.endsWith(' B')).toBe(true)
// const webpackSizeValue = webpackSize.endsWith('kB')
// ? parseFloat(webpackSize)
// : parseFloat(webpackSize) / 1000
// expect(webpackSizeValue).toBeCloseTo(gz ? 0.766 : 1.46, 2)
expect(webpackSize.endsWith('kB') || webpackSize.endsWith(' B')).toBe(
true
)
// expect(parseFloat(mainSize)).toBeCloseTo(gz ? 20.1 : 62.7, 1)
expect(mainSize.endsWith('kB')).toBe(true)
// expect(parseFloat(frameworkSize)).toBeCloseTo(gz ? 42.0 : 130, 1)
expect(frameworkSize.endsWith('kB')).toBe(true)
})
it('should print duration when rendering or get static props takes long', () => {
const matches = stdout.match(
/ \/slow-static\/.+\/.+(?: \(\d+ ms\))?| \[\+\d+ more paths\]/g
)
expect(matches).toEqual([
// summary
expect.stringMatching(
/\/\[propsDuration\]\/\[renderDuration\] \(\d+ ms\)/
),
// ordered by duration, includes duration
expect.stringMatching(/\/2000\/10 \(\d+ ms\)$/),
expect.stringMatching(/\/10\/1000 \(\d+ ms\)$/),
expect.stringMatching(/\/300\/10 \(\d+ ms\)$/),
// kept in original order
expect.stringMatching(/\/5\/5$/),
expect.stringMatching(/\/25\/25$/),
expect.stringMatching(/\/20\/20$/),
expect.stringMatching(/\/10\/10$/),
// max of 7 preview paths
' [+2 more paths]',
])
})
it('should not emit extracted comments', async () => {
const files = await recursiveReadDir(
join(appDir, '.next'),
/\.txt|\.LICENSE\./
)
expect(files).toEqual([])
if (hasExperimentalConfig) {
afterAll(async () => {
nextConfig.delete()
})
}
)
it('should not include internal pages', async () => {
;({ stdout } = await nextBuild(appDir, [], {
stdout: true,
}))
expect(stdout).toMatch(/\/ (.* )?\d{1,} B/)
expect(stdout).toMatch(/\+ First Load JS shared by all [ 0-9.]* kB/)
expect(stdout).toMatch(/ chunks\/main\.[0-9a-z]{6}\.js [ 0-9.]* kB/)
expect(stdout).toMatch(
/ chunks\/framework\.[0-9a-z]{6}\.js [ 0-9. ]* kB/
)
expect(stdout).not.toContain(' /_document')
expect(stdout).not.toContain(' /_app')
expect(stdout).not.toContain(' /_error')
expect(stdout).not.toContain('<buildId>')
expect(stdout).toContain('○ /')
})
it('should not deviate from snapshot', async () => {
console.log(stdout)
if (process.env.NEXT_PRIVATE_SKIP_SIZE_TESTS) {
return
}
const parsePageSize = (page) =>
stdout.match(
new RegExp(` ${page} .*?((?:\\d|\\.){1,} (?:\\w{1,})) `)
)[1]
const parsePageFirstLoad = (page) =>
stdout.match(
new RegExp(
` ${page} .*?(?:(?:\\d|\\.){1,}) .*? ((?:\\d|\\.){1,} (?:\\w{1,}))`
)
)[1]
const parseSharedSize = (sharedPartName) => {
const matches = stdout.match(
new RegExp(`${sharedPartName} .*? ((?:\\d|\\.){1,} (?:\\w{1,}))`)
)
if (!matches) {
throw new Error(`Could not match ${sharedPartName}`)
}
return matches[1]
}
const indexSize = parsePageSize('/')
const indexFirstLoad = parsePageFirstLoad('/')
const err404Size = parsePageSize('/404')
const err404FirstLoad = parsePageFirstLoad('/404')
const sharedByAll = parseSharedSize('shared by all')
const _appSize = parseSharedSize('_app\\..*?\\.js')
const webpackSize = parseSharedSize('webpack\\..*?\\.js')
const mainSize = parseSharedSize('main\\..*?\\.js')
const frameworkSize = parseSharedSize('framework\\..*?\\.js')
for (const size of [
indexSize,
indexFirstLoad,
err404Size,
err404FirstLoad,
sharedByAll,
_appSize,
webpackSize,
mainSize,
frameworkSize,
]) {
expect(parseFloat(size)).toBeGreaterThan(0)
}
// const gz = experimental.gzipSize !== false
// expect(parseFloat(indexSize) / 1000).toBeCloseTo(
// gz ? 0.251 : 0.394,
// 2
// )
expect(indexSize.endsWith('B')).toBe(true)
// expect(parseFloat(indexFirstLoad)).toBeCloseTo(gz ? 64 : 196, 1)
expect(indexFirstLoad.endsWith('kB')).toBe(true)
// expect(parseFloat(err404Size)).toBeCloseTo(gz ? 3.17 : 8.51, 1)
expect(err404Size.endsWith('B')).toBe(true)
// expect(parseFloat(err404FirstLoad)).toBeCloseTo(gz ? 66.9 : 204, 1)
expect(err404FirstLoad.endsWith('kB')).toBe(true)
// expect(parseFloat(sharedByAll)).toBeCloseTo(gz ? 63.7 : 196, 1)
expect(sharedByAll.endsWith('kB')).toBe(true)
// const appSizeValue = _appSize.endsWith('kB')
// ? parseFloat(_appSize)
// : parseFloat(_appSize) / 1000
// expect(appSizeValue).toBeCloseTo(gz ? 0.799 : 1.63, 1)
expect(_appSize.endsWith('kB') || _appSize.endsWith(' B')).toBe(true)
// const webpackSizeValue = webpackSize.endsWith('kB')
// ? parseFloat(webpackSize)
// : parseFloat(webpackSize) / 1000
// expect(webpackSizeValue).toBeCloseTo(gz ? 0.766 : 1.46, 2)
expect(webpackSize.endsWith('kB') || webpackSize.endsWith(' B')).toBe(
true
)
// expect(parseFloat(mainSize)).toBeCloseTo(gz ? 20.1 : 62.7, 1)
expect(mainSize.endsWith('kB')).toBe(true)
// expect(parseFloat(frameworkSize)).toBeCloseTo(gz ? 42.0 : 130, 1)
expect(frameworkSize.endsWith('kB')).toBe(true)
})
it('should print duration when rendering or get static props takes long', () => {
const matches = stdout.match(
/ \/slow-static\/.+\/.+(?: \(\d+ ms\))?| \[\+\d+ more paths\]/g
)
expect(matches).toEqual([
// summary
expect.stringMatching(
/\/\[propsDuration\]\/\[renderDuration\] \(\d+ ms\)/
),
// ordered by duration, includes duration
expect.stringMatching(/\/2000\/10 \(\d+ ms\)$/),
expect.stringMatching(/\/10\/1000 \(\d+ ms\)$/),
expect.stringMatching(/\/300\/10 \(\d+ ms\)$/),
// kept in original order
expect.stringMatching(/\/5\/5$/),
expect.stringMatching(/\/25\/25$/),
expect.stringMatching(/\/20\/20$/),
expect.stringMatching(/\/10\/10$/),
// max of 7 preview paths
' [+2 more paths]',
])
})
it('should not emit extracted comments', async () => {
const files = await recursiveReadDir(
join(appDir, '.next'),
/\.txt|\.LICENSE\./
)
expect(files).toEqual([])
})
})
}
describe('Custom App Output', () => {