archive/rsnext
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add initial standalone build handling (#31003)

Browse source 
* Add initial standalone handling

* apply suggestions

* Apply suggestions from code review

Co-authored-by: Steven <steven@ceriously.com>
This commit is contained in:
JJ Kasper 2021-11-09 11:03:20 -06:00 committed by GitHub
parent 31985c5cae
commit eb7b40171a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 233 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

47
packages/next/build/index.ts
View file

@ -91,6 +91,7 @@ import {
printTreeView,
getCssFilePaths,
getUnresolvedModuleFromError,
copyTracedFiles,
isReservedPage,
isCustomErrorPage,
} from './utils'
@ -103,6 +104,7 @@ import isError, { NextError } from '../lib/is-error'
import { TelemetryPlugin } from './webpack/plugins/telemetry-plugin'
import { MiddlewareManifest } from './webpack/plugins/middleware-plugin'
import type { webpack5 as webpack } from 'next/dist/compiled/webpack/webpack'
import { recursiveCopy } from '../lib/recursive-copy'
export type SsgRoute = {
initialRevalidateSeconds: number | false
@ -552,6 +554,7 @@ export default async function build(
path.relative(distDir, manifestPath),
BUILD_MANIFEST,
PRERENDER_MANIFEST,
path.join(SERVER_DIRECTORY, MIDDLEWARE_MANIFEST),
hasServerComponents
? path.join(SERVER_DIRECTORY, MIDDLEWARE_FLIGHT_MANIFEST + '.js')
: null,
@ -1362,6 +1365,23 @@ export default async function build(
'utf8'
)
const outputFileTracingRoot =
config.experimental.outputFileTracingRoot || dir
if (config.experimental.outputStandalone) {
await nextBuildSpan
.traceChild('copy-traced-files')
.traceAsyncFn(async () => {
await copyTracedFiles(
dir,
distDir,
pageKeys,
outputFileTracingRoot,
requiredServerFiles.config
)
})
}
const finalPrerenderRoutes: { [route: string]: SsgRoute } = {}
const tbdPrerenderRoutes: string[] = []
let ssgNotFoundPaths: string[] = []
@ -1957,6 +1977,33 @@ export default async function build(
return Promise.reject(err)
})
if (config.experimental.outputStandalone) {
for (const file of [
...requiredServerFiles.files,
path.join(config.distDir, SERVER_FILES_MANIFEST),
]) {
const filePath = path.join(dir, file)
await promises.copyFile(
filePath,
path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, filePath)
)
)
}
await recursiveCopy(
path.join(distDir, SERVER_DIRECTORY, 'pages'),
path.join(
distDir,
'standalone',
path.relative(outputFileTracingRoot, distDir),
SERVER_DIRECTORY,
'pages'
)
)
}
staticPages.forEach((pg) => allStaticPages.add(pg))
pageInfos.forEach((info: PageInfo, key: string) => {
allPageInfos.set(key, info)

113
packages/next/build/utils.ts
View file

@ -24,7 +24,10 @@ import { isDynamicRoute } from '../shared/lib/router/utils/is-dynamic'
import escapePathDelimiters from '../shared/lib/router/utils/escape-path-delimiters'
import { findPageFile } from '../server/lib/find-page-file'
import { GetStaticPaths, PageConfig } from 'next/types'
import { denormalizePagePath } from '../server/normalize-page-path'
import {
denormalizePagePath,
normalizePagePath,
} from '../server/normalize-page-path'
import { BuildManifest } from '../server/get-page-files'
import { removePathTrailingSlash } from '../client/normalize-trailing-slash'
import { UnwrapPromise } from '../lib/coalesced-function'
@ -35,6 +38,8 @@ import { trace } from '../trace'
import { setHttpAgentOptions } from '../server/config'
import { NextConfigComplete } from '../server/config-shared'
import isError from '../lib/is-error'
import { recursiveDelete } from '../lib/recursive-delete'
import { Sema } from 'next/dist/compiled/async-sema'
const { builtinModules } = require('module')
const RESERVED_PAGE = /^\/(_app|_error|_document|api(\/|$))/
@ -1146,6 +1151,112 @@ export function getUnresolvedModuleFromError(
return builtinModules.find((item: string) => item === moduleName)
}
export async function copyTracedFiles(
dir: string,
distDir: string,
pageKeys: string[],
tracingRoot: string,
serverConfig: { [key: string]: any }
) {
const outputPath = path.join(distDir, 'standalone')
const copiedFiles = new Set()
await recursiveDelete(outputPath)
async function handleTraceFiles(traceFilePath: string) {
const traceData = JSON.parse(await fs.readFile(traceFilePath, 'utf8')) as {
files: string[]
}
const copySema = new Sema(10, { capacity: traceData.files.length })
const traceFileDir = path.dirname(traceFilePath)
await Promise.all(
traceData.files.map(async (relativeFile) => {
await copySema.acquire()
const tracedFilePath = path.join(traceFileDir, relativeFile)
const fileOutputPath = path.join(
outputPath,
path.relative(tracingRoot, tracedFilePath)
)
if (!copiedFiles.has(fileOutputPath)) {
copiedFiles.add(fileOutputPath)
await fs.mkdir(path.dirname(fileOutputPath), { recursive: true })
const symlink = await fs.readlink(tracedFilePath).catch(() => null)
if (symlink) {
console.log('symlink', path.relative(tracingRoot, symlink))
await fs.symlink(
path.relative(tracingRoot, symlink),
fileOutputPath
)
} else {
await fs.copyFile(tracedFilePath, fileOutputPath)
}
}
await copySema.release()
})
)
}
for (const page of pageKeys) {
const pageFile = path.join(
distDir,
'server',
'pages',
`${normalizePagePath(page)}.js`
)
const pageTraceFile = `${pageFile}.nft.json`
await handleTraceFiles(pageTraceFile)
}
await handleTraceFiles(path.join(distDir, 'next-server.js.nft.json'))
const serverOutputPath = path.join(
outputPath,
path.relative(tracingRoot, dir),
'server.js'
)
await fs.writeFile(
serverOutputPath,
`
process.env.NODE_ENV = 'production'
process.chdir(__dirname)
const NextServer = require('next/dist/server/next-server').default
const http = require('http')
const path = require('path')
const nextServer = new NextServer({
dir: path.join(__dirname),
dev: false,
conf: ${JSON.stringify({
...serverConfig,
distDir: `./${path.relative(dir, distDir)}`,
})},
})
const handler = nextServer.getRequestHandler()
const server = http.createServer(async (req, res) => {
try {
await handler(req, res)
} catch (err) {
console.error(err);
res.statusCode = 500
res.end('internal server error')
}
})
const currentPort = process.env.PORT || 3000
server.listen(currentPort, (err) => {
if (err) {
console.error("Failed to start server", err)
process.exit(1)
}
console.log("Listening on port", currentPort)
})
`
)
}
export function isReservedPage(page: string) {
return RESERVED_PAGE.test(page)
}

2
packages/next/lib/recursive-copy.ts
View file

@ -47,7 +47,7 @@ export async function recursiveCopy(
if (isDirectory) {
try {
await promises.mkdir(target)
await promises.mkdir(target, { recursive: true })
} catch (err) {
// do not throw `folder already exists` errors
if (isError(err) && err.code !== 'EEXIST') {

36
packages/next/lib/recursive-delete.ts
View file

@ -1,13 +1,17 @@
import { Dirent, promises } from 'fs'
import { join } from 'path'
import { join, isAbsolute, dirname } from 'path'
import { promisify } from 'util'
import isError from './is-error'
const sleep = promisify(setTimeout)
const unlinkFile = async (p: string, t = 1): Promise<void> => {
const unlinkPath = async (p: string, isDir = false, t = 1): Promise<void> => {
try {
if (isDir) {
await promises.rmdir(p)
} else {
await promises.unlink(p)
}
} catch (e) {
const code = isError(e) && e.code
if (
@ -18,7 +22,7 @@ const unlinkFile = async (p: string, t = 1): Promise<void> => {
t < 3
) {
await sleep(t * 100)
return unlinkFile(p, t++)
return unlinkPath(p, isDir, t++)
}
if (code === 'ENOENT') {
@ -58,19 +62,29 @@ export async function recursiveDelete(
// readdir does not follow symbolic links
// if part is a symbolic link, follow it using stat
let isDirectory = part.isDirectory()
if (part.isSymbolicLink()) {
const stats = await promises.stat(absolutePath)
const isSymlink = part.isSymbolicLink()
if (isSymlink) {
const linkPath = await promises.readlink(absolutePath)
try {
const stats = await promises.stat(
isAbsolute(linkPath)
? linkPath
: join(dirname(absolutePath), linkPath)
)
isDirectory = stats.isDirectory()
} catch (_) {}
}
const pp = join(previousPath, part.name)
if (isDirectory && (!exclude || !exclude.test(pp))) {
await recursiveDelete(absolutePath, exclude, pp)
return promises.rmdir(absolutePath)
}
const isNotExcluded = !exclude || !exclude.test(pp)
if (!exclude || !exclude.test(pp)) {
return unlinkFile(absolutePath)
if (isNotExcluded) {
if (isDirectory) {
await recursiveDelete(absolutePath, exclude, pp)
}
return unlinkPath(absolutePath, !isSymlink && isDirectory)
}
})
)

2
packages/next/server/config-shared.ts
View file

@ -157,6 +157,7 @@ export type NextConfig = { [key: string]: any } & {
fullySpecified?: boolean
urlImports?: NonNullable<webpack5.Configuration['experiments']>['buildHttp']
outputFileTracingRoot?: string
outputStandalone?: boolean
}
}
@ -239,6 +240,7 @@ export const defaultConfig: NextConfig = {
serverComponents: false,
fullySpecified: false,
outputFileTracingRoot: process.env.NEXT_PRIVATE_OUTPUT_TRACE_ROOT || '',
outputStandalone: !!process.env.NEXT_PRIVATE_STANDALONE,
},
future: {
strictPostcssConfiguration: false,

4
packages/next/server/next-server.ts
View file

@ -2551,7 +2551,9 @@ export default class Server {
}
let nextFilesStatic: string[] = []
nextFilesStatic = !this.minimalMode
nextFilesStatic =
!this.minimalMode && fs.existsSync(join(this.distDir, 'static'))
? recursiveReadDirSync(join(this.distDir, 'static')).map((f) =>
join('.', relative(this.dir, this.distDir), 'static', f)
)

112
test/production/required-server-files.test.ts
View file

@ -1,7 +1,7 @@
import glob from 'glob'
import _fs from 'fs-extra'
import fs from 'fs-extra'
import cheerio from 'cheerio'
import { join, dirname } from 'path'
import { join } from 'path'
import { createNext, FileRef } from 'e2e-utils'
import { NextInstance } from 'test/lib/next-modes/base'
import {
@ -33,6 +33,9 @@ describe('should set-up next', () => {
eslint: {
ignoreDuringBuilds: true,
},
experimental: {
outputStandalone: true,
},
async rewrites() {
return [
{
@ -48,94 +51,47 @@ describe('should set-up next', () => {
},
})
await next.stop()
const keptFiles = new Set<string>()
const nextServerTrace = require(join(
next.testDir,
'.next/next-server.js.nft.json'
))
requiredFilesManifest = JSON.parse(
await next.readFile('.next/required-server-files.json')
)
requiredFilesManifest.files.forEach((file) => keptFiles.add(file))
const pageTraceFiles = glob.sync('**/*.nft.json', {
cwd: join(next.testDir, '.next/server/pages'),
})
for (const traceFile of pageTraceFiles) {
const pageDir = dirname(join('.next/server/pages', traceFile))
const trace = await _fs.readJSON(
join(next.testDir, '.next/server/pages', traceFile)
await fs.move(
join(next.testDir, '.next/standalone'),
join(next.testDir, 'standalone')
)
keptFiles.add(
join('.next/server/pages', traceFile.replace('.nft.json', ''))
)
for (const file of trace.files) {
keptFiles.add(join(pageDir, file))
for (const file of await fs.readdir(next.testDir)) {
if (file !== 'standalone') {
await fs.remove(join(next.testDir, file))
console.log('removed', file)
}
}
const allFiles = glob.sync('**/*', {
cwd: next.testDir,
const files = glob.sync('**/*', {
cwd: join(next.testDir, 'standalone/.next/server/pages'),
dot: true,
})
const nextServerTraceFiles = nextServerTrace.files.map((file) => {
return join(next.testDir, '.next', file)
})
console.error({ files })
for (const file of allFiles) {
const filePath = join(next.testDir, file)
if (
!keptFiles.has(file) &&
!(await _fs.stat(filePath).catch(() => null))?.isDirectory() &&
!nextServerTraceFiles.includes(filePath) &&
!file.match(/node_modules\/(react|react-dom)\//) &&
file !== 'node_modules/next/dist/server/next-server.js'
) {
await _fs.remove(filePath)
for (const file of files) {
if (file.endsWith('.json') || file.endsWith('.html')) {
await fs.remove(join(next.testDir, '.next/server', file))
}
}
appPort = await findPort()
const testServer = join(next.testDir, 'server.js')
await _fs.writeFile(
const testServer = join(next.testDir, 'standalone/server.js')
await fs.writeFile(
testServer,
`
const http = require('http')
const NextServer = require('next/dist/server/next-server').default
const appPort = ${appPort}
const nextApp = new NextServer({
conf: ${JSON.stringify(requiredFilesManifest.config)},
dir: "${next.testDir}",
quiet: false,
minimalMode: true,
})
server = http.createServer(async (req, res) => {
try {
await nextApp.getRequestHandler()(req, res)
} catch (err) {
console.error('top-level', err)
res.statusCode = 500
res.end('error')
}
})
server.listen(appPort, (err) => {
if (err) throw err
console.log(\`Listening at ::${appPort}\`)
})
`
(await fs.readFile(testServer, 'utf8'))
.replace('console.error(err)', `console.error('top-level', err)`)
.replace('conf:', 'minimalMode: true,conf:')
)
appPort = await findPort()
server = await initNextServerScript(
testServer,
/Listening at/,
/Listening on/,
{
...process.env,
NODE_ENV: 'production',
PORT: appPort,
},
undefined,
{
@ -165,7 +121,7 @@ describe('should set-up next', () => {
})
it('should set correct SWR headers with notFound gsp', async () => {
await next.patchFile('data.txt', 'show')
await next.patchFile('standalone/data.txt', 'show')
const res = await fetchViaHTTP(appPort, '/gsp', undefined, {
redirect: 'manual ',
@ -175,7 +131,7 @@ describe('should set-up next', () => {
's-maxage=1, stale-while-revalidate'
)
await next.patchFile('data.txt', 'hide')
await next.patchFile('standalone/data.txt', 'hide')
const res2 = await fetchViaHTTP(appPort, '/gsp', undefined, {
redirect: 'manual ',
@ -187,7 +143,7 @@ describe('should set-up next', () => {
})
it('should set correct SWR headers with notFound gssp', async () => {
await next.patchFile('data.txt', 'show')
await next.patchFile('standalone/data.txt', 'show')
const res = await fetchViaHTTP(appPort, '/gssp', undefined, {
redirect: 'manual ',
@ -197,7 +153,7 @@ describe('should set-up next', () => {
's-maxage=1, stale-while-revalidate'
)
await next.patchFile('data.txt', 'hide')
await next.patchFile('standalone/data.txt', 'hide')
const res2 = await fetchViaHTTP(appPort, '/gssp', undefined, {
redirect: 'manual ',
@ -576,7 +532,7 @@ describe('should set-up next', () => {
errors = []
const res = await fetchViaHTTP(appPort, '/errors/gip', { crash: '1' })
expect(res.status).toBe(500)
expect(await res.text()).toBe('error')
expect(await res.text()).toBe('internal server error')
await check(
() => (errors[0].includes('gip hit an oops') ? 'success' : errors[0]),
@ -588,7 +544,7 @@ describe('should set-up next', () => {
errors = []
const res = await fetchViaHTTP(appPort, '/errors/gssp', { crash: '1' })
expect(res.status).toBe(500)
expect(await res.text()).toBe('error')
expect(await res.text()).toBe('internal server error')
await check(
() => (errors[0].includes('gssp hit an oops') ? 'success' : errors[0]),
'success'
@ -599,7 +555,7 @@ describe('should set-up next', () => {
errors = []
const res = await fetchViaHTTP(appPort, '/errors/gsp/crash')
expect(res.status).toBe(500)
expect(await res.text()).toBe('error')
expect(await res.text()).toBe('internal server error')
await check(
() => (errors[0].includes('gsp hit an oops') ? 'success' : errors[0]),
'success'
@ -610,7 +566,7 @@ describe('should set-up next', () => {
errors = []
const res = await fetchViaHTTP(appPort, '/api/error')
expect(res.status).toBe(500)
expect(await res.text()).toBe('error')
expect(await res.text()).toBe('internal server error')
await check(
() =>
errors[0].includes('some error from /api/error')

2
test/unit/recursive-delete.test.ts
View file

@ -1,4 +1,5 @@
/* eslint-env jest */
import fs from 'fs-extra'
import { recursiveDelete } from 'next/dist/lib/recursive-delete'
import { recursiveReadDir } from 'next/dist/lib/recursive-readdir'
import { recursiveCopy } from 'next/dist/lib/recursive-copy'
@ -13,6 +14,7 @@ describe('recursiveDelete', () => {
expect.assertions(1)
try {
await recursiveCopy(resolveDataDir, testResolveDataDir)
await fs.symlink('./aa', join(testResolveDataDir, 'symlink'))
await recursiveDelete(testResolveDataDir)
const result = await recursiveReadDir(testResolveDataDir, /.*/)
expect(result.length).toBe(0)