Optimize next-app-loader resolving speed (#50745)

## What? We recently implemented an optimized resolving method for `app` in Turbopack, this ports some of the main changes in that resolving logic to optimize `next-app-loader` which during compilation resolves the tree structure that we use to render in `app-render.tsx`. Here's the results for a page that is nested a few levels deep on vercel.com using App Router. These results only cover `next-app-loader`, not any modules compiled below it. ### Before <img width="671" alt="CleanShot 2023-06-03 at 22 36 26@2x" src="https://github.com/vercel/next.js/assets/6324199/0edeb060-2460-4a7d-95a7-1c22ea26a065"> ### After <img width="673" alt="CleanShot 2023-06-03 at 22 55 10@2x" src="https://github.com/vercel/next.js/assets/6324199/f40964fc-b169-4d95-8711-73cbff3ec76a"> ## Raw numbers <table> <tr> <td>Before</td> <td>After</td> <td>Delta</td> <td>Delta (percent)</td> </tr> <tr> <td>1.620 ms</td> <td>76.39 ms</td> <td>-1.543.61 ms</td> <td>-95.2%</td> </tr> </table> ## How? Changed the resolving logic to use `fileExists`, looping over the provided pageExtensions. For Turbopack we have a process that does only one pass for generating all trees. That also only reads directories instead of checking individual files, which is even better (<5ms for generating all possible trees) but this PR is a quick win that has a big impact already without refactoring the entire entries generation in webpack.
2023-06-05 08:51:02 +02:00 · 2023-06-05 08:51:02 +02:00 · dd714796d7
commit dd714796d7
parent 18d112fb5c
6 changed files with 123 additions and 116 deletions
--- a/packages/next/src/build/index.ts
+++ b/packages/next/src/build/index.ts
@ -28,7 +28,7 @@ import {
  PAGES_DIR_ALIAS,
  INSTRUMENTATION_HOOK_FILENAME,
 } from '../lib/constants'
-import { fileExists } from '../lib/file-exists'
+import { FileType, fileExists } from '../lib/file-exists'
 import { findPagesDir } from '../lib/find-pages-dir'
 import loadCustomRoutes, {
  CustomRoutes,
@ -588,7 +588,7 @@ export default async function build(
          for (const page in mappedPages) {
            const hasPublicPageFile = await fileExists(
              path.join(publicDir, page === '/' ? '/index' : page),
-              'file'
+              FileType.File
            )
            if (hasPublicPageFile) {
              conflictingPublicFiles.push(page)
--- a/packages/next/src/build/webpack/loaders/metadata/discover.ts
+++ b/packages/next/src/build/webpack/loaders/metadata/discover.ts
@ -1,4 +1,3 @@
-import type webpack from 'webpack'
 import type {
  CollectingMetadata,
  PossibleStaticMetadataFileNameConvention,
@ -7,6 +6,7 @@ import path from 'path'
 import { stringify } from 'querystring'
 import { STATIC_METADATA_IMAGES } from '../../../../lib/metadata/is-metadata-route'
 import { WEBPACK_RESOURCE_QUERIES } from '../../../../lib/constants'
+import { MetadataResolver } from '../next-app-loader'

 const METADATA_TYPE = 'metadata'

@ -16,16 +16,14 @@ async function enumMetadataFiles(
  filename: string,
  extensions: readonly string[],
  {
-    resolvePath,
-    loaderContext,
+    metadataResolver,
    // When set to true, possible filename without extension could: icon, icon0, ..., icon9
    numericSuffix,
  }: {
-    resolvePath: (pathname: string) => Promise<string>
-    loaderContext: webpack.LoaderContext<any>
+    metadataResolver: MetadataResolver
    numericSuffix: boolean
  }
-) {
+): Promise<string[]> {
  const collectedFiles: string[] = []

  const possibleFileNames = [filename].concat(
@ -36,19 +34,9 @@ async function enumMetadataFiles(
      : []
  )
  for (const name of possibleFileNames) {
-    for (const ext of extensions) {
-      const pathname = path.join(dir, `${name}.${ext}`)
-      try {
-        const resolved = await resolvePath(pathname)
-        loaderContext.addDependency(resolved)
-
-        collectedFiles.push(resolved)
-      } catch (err: any) {
-        if (!err.message.includes("Can't resolve")) {
-          throw err
-        }
-        loaderContext.addMissingDependency(pathname)
-      }
+    const resolved = await metadataResolver(path.join(dir, name), extensions)
+    if (resolved) {
+      collectedFiles.push(resolved)
    }
  }

@ -59,16 +47,14 @@ export async function createStaticMetadataFromRoute(
  resolvedDir: string,
  {
    segment,
-    resolvePath,
+    metadataResolver,
    isRootLayoutOrRootPage,
-    loaderContext,
    pageExtensions,
    basePath,
  }: {
    segment: string
-    resolvePath: (pathname: string) => Promise<string>
+    metadataResolver: MetadataResolver
    isRootLayoutOrRootPage: boolean
-    loaderContext: webpack.LoaderContext<any>
    pageExtensions: string[]
    basePath: string
  }
@ -82,11 +68,6 @@ export async function createStaticMetadataFromRoute(
    manifest: undefined,
  }

-  const opts = {
-    resolvePath,
-    loaderContext,
-  }
-
  async function collectIconModuleIfExists(
    type: PossibleStaticMetadataFileNameConvention
  ) {
@ -96,7 +77,7 @@ export async function createStaticMetadataFromRoute(
        resolvedDir,
        'manifest',
        staticManifestExtension.concat(pageExtensions),
-        { ...opts, numericSuffix: false }
+        { metadataResolver, numericSuffix: false }
      )
      if (manifestFile.length > 0) {
        hasStaticMetadataFiles = true
@ -116,7 +97,7 @@ export async function createStaticMetadataFromRoute(
        ...STATIC_METADATA_IMAGES[type].extensions,
        ...(type === 'favicon' ? [] : pageExtensions),
      ],
-      { ...opts, numericSuffix: true }
+      { metadataResolver, numericSuffix: true }
    )
    resolvedMetadataFiles
      .sort((a, b) => a.localeCompare(b))
--- a/packages/next/src/build/webpack/loaders/next-app-loader.ts
+++ b/packages/next/src/build/webpack/loaders/next-app-loader.ts
@ -5,7 +5,6 @@ import type { ModuleReference, CollectedMetadata } from './metadata/types'
 import path from 'path'
 import { stringify } from 'querystring'
 import chalk from 'next/dist/compiled/chalk'
-import { NODE_RESOLVE_OPTIONS } from '../../webpack-config'
 import { getModuleBuildInfo } from './get-module-build-info'
 import { verifyRootLayout } from '../../../lib/verifyRootLayout'
 import * as Log from '../../output/log'
@ -22,6 +21,7 @@ import { AppPathnameNormalizer } from '../../../server/future/normalizers/built/
 import { RouteKind } from '../../../server/future/route-kind'
 import { AppRouteRouteModuleOptions } from '../../../server/future/route-modules/app-route/module'
 import { AppBundlePathNormalizer } from '../../../server/future/normalizers/built/app/app-bundle-path-normalizer'
+import { FileType, fileExists } from '../../../lib/file-exists'

 export type AppLoaderOptions = {
  name: string
@ -40,8 +40,6 @@ export type AppLoaderOptions = {
 }
 type AppLoader = webpack.LoaderDefinitionFunction<AppLoaderOptions>

-const isNotResolvedError = (err: any) => err.message.includes("Can't resolve")
-
 const FILE_TYPES = {
  layout: 'layout',
  template: 'template',
@ -53,10 +51,13 @@ const FILE_TYPES = {
 const GLOBAL_ERROR_FILE_TYPE = 'global-error'
 const PAGE_SEGMENT = 'page$'

+type DirResolver = (pathToResolve: string) => string
 type PathResolver = (
+  pathname: string
+) => Promise<string | undefined> | string | undefined
+export type MetadataResolver = (
  pathname: string,
-  resolveDir?: boolean,
-  internal?: boolean
+  extensions: readonly string[]
 ) => Promise<string | undefined>

 export type ComponentsType = {
@ -73,14 +74,14 @@ async function createAppRouteCode({
  name,
  page,
  pagePath,
-  resolver,
+  resolveAppRoute,
  pageExtensions,
  nextConfigOutput,
 }: {
  name: string
  page: string
  pagePath: string
-  resolver: PathResolver
+  resolveAppRoute: PathResolver
  pageExtensions: string[]
  nextConfigOutput: NextConfig['output']
 }): Promise<string> {
@ -90,7 +91,7 @@ async function createAppRouteCode({

  // This, when used with the resolver will give us the pathname to the built
  // route handler file.
-  let resolvedPagePath = await resolver(routePath)
+  let resolvedPagePath = await resolveAppRoute(routePath)
  if (!resolvedPagePath) {
    throw new Error(
      `Invariant: could not resolve page path for ${name} at ${routePath}`
@ -181,15 +182,16 @@ const isDirectory = async (pathname: string) => {
 async function createTreeCodeFromPath(
  pagePath: string,
  {
+    resolveDir,
    resolver,
-    resolvePath,
    resolveParallelSegments,
-    loaderContext,
+    metadataResolver,
    pageExtensions,
    basePath,
  }: {
+    resolveDir: DirResolver
    resolver: PathResolver
-    resolvePath: (pathname: string) => Promise<string>
+    metadataResolver: MetadataResolver
    resolveParallelSegments: (
      pathname: string
    ) => [key: string, segment: string | string[]][]
@ -197,7 +199,12 @@ async function createTreeCodeFromPath(
    pageExtensions: string[]
    basePath: string
  }
-) {
+): Promise<{
+  treeCode: string
+  pages: string
+  rootLayout: string | undefined
+  globalError: string | undefined
+}> {
  const splittedPath = pagePath.split(/[\\/]/)
  const appDirPrefix = splittedPath[0]
  const pages: string[] = []
@ -208,9 +215,8 @@ async function createTreeCodeFromPath(
  async function resolveAdjacentParallelSegments(
    segmentPath: string
  ): Promise<string[]> {
-    const absoluteSegmentPath = await resolver(
-      `${appDirPrefix}${segmentPath}`,
-      true
+    const absoluteSegmentPath = await resolveDir(
+      `${appDirPrefix}${segmentPath}`
    )

    if (!absoluteSegmentPath) {
@ -264,24 +270,17 @@ async function createTreeCodeFromPath(

    let metadata: Awaited<ReturnType<typeof createStaticMetadataFromRoute>> =
      null
-    try {
-      const routerDirPath = `${appDirPrefix}${segmentPath}`
-      const resolvedRouteDir = await resolver(routerDirPath, true)
+    const routerDirPath = `${appDirPrefix}${segmentPath}`
+    const resolvedRouteDir = await resolveDir(routerDirPath)

-      if (resolvedRouteDir) {
-        metadata = await createStaticMetadataFromRoute(resolvedRouteDir, {
-          basePath,
-          segment: segmentPath,
-          resolvePath,
-          isRootLayoutOrRootPage,
-          loaderContext,
-          pageExtensions,
-        })
-      }
-    } catch (err: any) {
-      if (isNotResolvedError(err)) {
-        throw err
-      }
+    if (resolvedRouteDir) {
+      metadata = await createStaticMetadataFromRoute(resolvedRouteDir, {
+        basePath,
+        segment: segmentPath,
+        metadataResolver,
+        isRootLayoutOrRootPage,
+        pageExtensions,
+      })
    }

    for (const [parallelKey, parallelSegment] of parallelSegments) {
@ -448,20 +447,6 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {

  const extensions = pageExtensions.map((extension) => `.${extension}`)

-  const resolveOptions: any = {
-    ...NODE_RESOLVE_OPTIONS,
-    extensions,
-  }
-
-  const resolve = this.getResolve(resolveOptions)
-
-  // a resolver for internal next files. We need to override the extensions, in case
-  // a project doesn't have the same ones as used by next.
-  const internalResolve = this.getResolve({
-    ...resolveOptions,
-    extensions: [...extensions, '.js', '.jsx', '.ts', '.tsx'],
-  })
-
  const normalizedAppPaths =
    typeof appPaths === 'string' ? [appPaths] : appPaths || []

@ -496,29 +481,55 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {

    return Object.entries(matched)
  }
-  const resolver: PathResolver = async (pathname, resolveDir, internal) => {
-    if (resolveDir) {
-      return createAbsolutePath(appDir, pathname)
-    }

-    try {
-      const resolved = await (internal ? internalResolve : resolve)(
-        this.rootContext,
-        pathname
-      )
-      this.addDependency(resolved)
-      return resolved
-    } catch (err: any) {
-      const absolutePath = createAbsolutePath(appDir, pathname)
-      for (const ext of extensions) {
-        const absolutePathWithExtension = `${absolutePath}${ext}`
+  const resolveDir: DirResolver = (pathToResolve) => {
+    return createAbsolutePath(appDir, pathToResolve)
+  }
+
+  const resolveAppRoute: PathResolver = (pathToResolve) => {
+    return createAbsolutePath(appDir, pathToResolve)
+  }
+
+  const resolver: PathResolver = async (pathname) => {
+    const absolutePath = createAbsolutePath(appDir, pathname)
+
+    let result: string | undefined
+
+    for (const ext of extensions) {
+      const absolutePathWithExtension = `${absolutePath}${ext}`
+      if (
+        !result &&
+        (await fileExists(absolutePathWithExtension, FileType.File))
+      ) {
+        // Ensures we call `addMissingDependency` for all files that didn't match
+        result = absolutePathWithExtension
+      } else {
        this.addMissingDependency(absolutePathWithExtension)
      }
-      if (isNotResolvedError(err)) {
-        return undefined
-      }
-      throw err
    }
+
+    return result
+  }
+
+  const metadataResolver: MetadataResolver = async (pathname, exts) => {
+    const absolutePath = createAbsolutePath(appDir, pathname)
+
+    let result: string | undefined
+
+    for (const ext of exts) {
+      // Compared to `resolver` above the exts do not have the `.` included already, so it's added here.
+      const absolutePathWithExtension = `${absolutePath}.${ext}`
+      if (
+        !result &&
+        (await fileExists(absolutePathWithExtension, FileType.File))
+      ) {
+        result = absolutePathWithExtension
+      } else {
+        this.addMissingDependency(absolutePathWithExtension)
+      }
+    }
+
+    return result
  }

  if (isAppRouteRoute(name)) {
@ -527,27 +538,23 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
      page: loaderOptions.page,
      name,
      pagePath,
-      resolver,
+      resolveAppRoute,
      pageExtensions,
      nextConfigOutput,
    })
  }

-  const {
-    treeCode,
-    pages: pageListCode,
-    rootLayout,
-    globalError,
-  } = await createTreeCodeFromPath(pagePath, {
+  let treeCodeResult = await createTreeCodeFromPath(pagePath, {
+    resolveDir,
    resolver,
-    resolvePath: (pathname: string) => resolve(this.rootContext, pathname),
+    metadataResolver,
    resolveParallelSegments,
    loaderContext: this,
    pageExtensions,
    basePath,
  })

-  if (!rootLayout) {
+  if (!treeCodeResult.rootLayout) {
    if (!isDev) {
      // If we're building and missing a root layout, exit the build
      Log.error(
@ -581,18 +588,29 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {

        throw new Error(message)
      }
+
+      // Get the new result with the created root layout.
+      treeCodeResult = await createTreeCodeFromPath(pagePath, {
+        resolveDir,
+        resolver,
+        metadataResolver,
+        resolveParallelSegments,
+        loaderContext: this,
+        pageExtensions,
+        basePath,
+      })
    }
  }

  const result = `
-    export ${treeCode}
-    export ${pageListCode}
+    export ${treeCodeResult.treeCode}
+    export ${treeCodeResult.pages}

    export { default as AppRouter } from 'next/dist/client/components/app-router'
    export { default as LayoutRouter } from 'next/dist/client/components/layout-router'
    export { default as RenderFromTemplateContext } from 'next/dist/client/components/render-from-template-context'
    export { default as GlobalError } from ${JSON.stringify(
-      globalError || 'next/dist/client/components/error-boundary'
+      treeCodeResult.globalError || 'next/dist/client/components/error-boundary'
    )}

    export { staticGenerationAsyncStorage } from 'next/dist/client/components/static-generation-async-storage'
--- a/packages/next/src/cli/next-dev.ts
+++ b/packages/next/src/cli/next-dev.ts
@ -18,7 +18,7 @@ import { Telemetry } from '../telemetry/storage'
 import loadConfig from '../server/config'
 import { findPagesDir } from '../lib/find-pages-dir'
 import { findRootDir } from '../lib/find-root'
-import { fileExists } from '../lib/file-exists'
+import { fileExists, FileType } from '../lib/file-exists'
 import { getNpxCommand } from '../lib/helpers/get-npx-command'
 import Watchpack from 'next/dist/compiled/watchpack'
 import stripAnsi from 'next/dist/compiled/strip-ansi'
@ -161,7 +161,7 @@ const nextDev: CliCommand = async (argv) => {
  dir = getProjectDir(process.env.NEXT_PRIVATE_DEV_DIR || args._[0])

  // Check if pages dir exists and warn if not
-  if (!(await fileExists(dir, 'directory'))) {
+  if (!(await fileExists(dir, FileType.Directory))) {
    printAndExit(`> No such directory exists as the project root: ${dir}`)
  }

--- a/packages/next/src/lib/file-exists.ts
+++ b/packages/next/src/lib/file-exists.ts
@ -1,15 +1,20 @@
 import { constants, promises } from 'fs'
 import isError from './is-error'

+export enum FileType {
+  File = 'file',
+  Directory = 'directory',
+}
+
 export async function fileExists(
  fileName: string,
-  type?: 'file' | 'directory'
+  type?: FileType
 ): Promise<boolean> {
  try {
-    if (type === 'file') {
+    if (type === FileType.File) {
      const stats = await promises.stat(fileName)
      return stats.isFile()
-    } else if (type === 'directory') {
+    } else if (type === FileType.Directory) {
      const stats = await promises.stat(fileName)
      return stats.isDirectory()
    } else {
--- a/packages/next/src/lib/verify-partytown-setup.ts
+++ b/packages/next/src/lib/verify-partytown-setup.ts
@ -6,7 +6,7 @@ import {
  hasNecessaryDependencies,
  NecessaryDependencies,
 } from './has-necessary-dependencies'
-import { fileExists } from './file-exists'
+import { fileExists, FileType } from './file-exists'
 import { FatalError } from './fatal-error'
 import { recursiveDelete } from './recursive-delete'
 import * as Log from '../build/output/log'
@ -44,7 +44,10 @@ async function copyPartytownStaticFiles(
  staticDir: string
 ) {
  const partytownLibDir = path.join(staticDir, '~partytown')
-  const hasPartytownLibDir = await fileExists(partytownLibDir, 'directory')
+  const hasPartytownLibDir = await fileExists(
+    partytownLibDir,
+    FileType.Directory
+  )

  if (hasPartytownLibDir) {
    await recursiveDelete(partytownLibDir)