Resolve build issues with symlinked files (#57412)

This resolves an issue introduced in #52361 where `shouldOmit` is
overwritten based on if it is a directory or not without considering
include conditions.

In an environment where the the application being built is orchestrated
and managed by bazel, all files are symbolic links and thus without this
the file includes predicate checks aren't considered - and the build
fails under certain conditions.

<!-- Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that
you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:

## For Contributors

### Improving Documentation

- Run `pnpm prettier-fix` to fix formatting issues before opening the
PR.
- Read the Docs Contribution Guide to ensure your contribution follows
the docs guidelines:
https://nextjs.org/docs/community/contribution-guide

### Adding or Updating Examples

- The "examples guidelines" are followed from our contributing doc
https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md
- Make sure the linting passes by running `pnpm build && pnpm lint`. See
https://github.com/vercel/next.js/blob/canary/contributing/repository/linting.md

### Fixing a bug

- Related issues linked using `fixes #number`
- Tests added. See:
https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md

### Adding a feature

- Implements an existing feature request or RFC. Make sure the feature
request has been accepted for implementation before opening a PR. (A
discussion must be opened, see
https://github.com/vercel/next.js/discussions/new?category=ideas)
- Related issues/discussions are linked using `fixes #number`
- e2e tests added
(https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs)
- Documentation added
- Telemetry added. In case of a feature if it's used or not.
- Errors have a helpful link attached, see
https://github.com/vercel/next.js/blob/canary/contributing.md


## For Maintainers

- Minimal description (aim for explaining to someone not on the team to
understand the PR)
- When linking to a Slack thread, you might want to share details of the
conclusion
- Link both the Linear (Fixes NEXT-xxx) and the GitHub issues
- Add review comments if necessary to explain to the reviewer the logic
behind a change

### What?

### Why?

### How?

Closes NEXT-
Fixes #

-->

---------

Co-authored-by: Shu Ding <g@shud.in>

packages/next/src/build/index.ts

@@ -133,7 +133,7 @@ import {
   createDefineEnv,
 } from './swc'
 import { getNamedRouteRegex } from '../shared/lib/router/utils/route-regex'
-import { flatReaddir } from '../lib/flat-readdir'
+import { getFilesInDir } from '../lib/get-files-in-dir'
 import { eventSwcPlugins } from '../telemetry/events/swc-plugins'
 import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
 import {
@@ -567,16 +567,18 @@ export default async function build(
       const instrumentationHookEnabled = Boolean(
         config.experimental.instrumentationHook
       )
-      const rootPaths = (
-        await flatReaddir(rootDir, [
-          middlewareDetectionRegExp,
-          ...(instrumentationHookEnabled
-            ? [instrumentationHookDetectionRegExp]
-            : []),
-        ])
-      )
+
+      const includes = [
+        middlewareDetectionRegExp,
+        ...(instrumentationHookEnabled
+          ? [instrumentationHookDetectionRegExp]
+          : []),
+      ]
+
+      const rootPaths = (await getFilesInDir(rootDir))
+        .filter((file) => includes.some((include) => include.test(file)))
         .sort(sortByPageExts(config.pageExtensions))
-        .map((absoluteFile) => absoluteFile.replace(dir, ''))
+        .map((file) => path.join(rootDir, file).replace(dir, ''))
 
       const hasInstrumentationHook = rootPaths.some((p) =>
         p.includes(INSTRUMENTATION_HOOK_FILENAME)

packages/next/src/build/webpack/loaders/next-app-loader.ts

@@ -24,6 +24,7 @@ import { getFilenameAndExtension } from './next-metadata-route-loader'
 import { isAppBuiltinNotFoundPage } from '../../utils'
 import { loadEntrypoint } from '../../load-entrypoint'
 import { isGroupSegment } from '../../../shared/lib/segment'
+import { getFilesInDir } from '../../../lib/get-files-in-dir'
 
 export type AppLoaderOptions = {
   name: string
@@ -558,13 +559,8 @@ const nextAppLoader: AppLoader = async function nextAppLoader() {
       return existingFiles.has(fileName)
     }
     try {
-      const files = await fs.readdir(dirname, { withFileTypes: true })
-      const fileNames = new Set<string>()
-      for (const file of files) {
-        if (file.isFile()) {
-          fileNames.add(file.name)
-        }
-      }
+      const files = await getFilesInDir(dirname)
+      const fileNames = new Set<string>(files)
       filesInDir.set(dirname, fileNames)
       return fileNames.has(fileName)
     } catch (err) {

packages/next/src/lib/flat-readdir.ts

@@ -1,23 +0,0 @@
-import { join } from 'path'
-import fs from 'fs/promises'
-
-export async function flatReaddir(dir: string, includes: RegExp[]) {
-  const dirents = await fs.opendir(dir)
-  const result = []
-
-  for await (const part of dirents) {
-    let shouldOmit =
-      part.isDirectory() || !includes.some((include) => include.test(part.name))
-
-    if (part.isSymbolicLink()) {
-      const stats = await fs.stat(join(dir, part.name))
-      shouldOmit = stats.isDirectory()
-    }
-
-    if (!shouldOmit) {
-      result.push(join(dir, part.name))
-    }
-  }
-
-  return result
-}

packages/next/src/lib/get-files-in-dir.ts

@@ -0,0 +1,22 @@
+import { join } from 'path'
+import fs from 'fs/promises'
+import type { Dirent, StatsBase } from 'fs'
+
+export async function getFilesInDir(path: string): Promise<string[]> {
+  const dir = await fs.opendir(path)
+  const results = []
+
+  for await (const file of dir) {
+    let resolvedFile: Dirent | StatsBase<number> = file
+
+    if (file.isSymbolicLink()) {
+      resolvedFile = await fs.stat(join(path, file.name))
+    }
+
+    if (resolvedFile.isFile()) {
+      results.push(file.name)
+    }
+  }
+
+  return results
+}

test/production/app-dir/symbolic-file-links/README.md

@@ -0,0 +1,5 @@
+# Symbolic File Links
+
+This test represents what the Next.js file structure may look like when run
+under a build orchestrator, such as bazel, where its sandbox sets up all files
+to be symlinks to their original source.

test/production/app-dir/symbolic-file-links/next.config.js

@@ -0,0 +1,6 @@
+/**
+ * @type {import('next').NextConfig}
+ */
+const nextConfig = {}
+
+module.exports = nextConfig

test/production/app-dir/symbolic-file-links/somewhere-else/src/app/layout.tsx

@@ -0,0 +1,7 @@
+export default function Root({ children }: { children: React.ReactNode }) {
+  return (
+    <html>
+      <body>{children}</body>
+    </html>
+  )
+}

test/production/app-dir/symbolic-file-links/somewhere-else/src/app/page.tsx

@@ -0,0 +1,3 @@
+export default function Page() {
+  return <p>hello world</p>
+}

test/production/app-dir/symbolic-file-links/somewhere-else/src/i18n.ts

@@ -0,0 +1,11 @@
+import { cookies } from 'next/headers'
+
+// The purpose of this file is to demonstrate that without proper symbolic file checking
+// next accidentally marks files in the root of the project as client files.
+export default function () {
+  const locale = cookies().get('locale')?.value ?? 'en'
+
+  return {
+    locale,
+  }
+}

test/production/app-dir/symbolic-file-links/src/app/layout.tsx

@@ -0,0 +1 @@
+../../somewhere-else/src/app/layout.tsx

test/production/app-dir/symbolic-file-links/src/app/page.tsx

@@ -0,0 +1 @@
+../../somewhere-else/src/app/page.tsx

test/production/app-dir/symbolic-file-links/src/i18n.ts

@@ -0,0 +1 @@
+../somewhere-else/src/i18n.ts

test/production/app-dir/symbolic-file-links/symbolic-file-links.test.ts

@@ -0,0 +1,34 @@
+import { createNextDescribe } from 'e2e-utils'
+
+createNextDescribe(
+  'symbolic-file-links',
+  {
+    files: __dirname,
+  },
+  ({ next }) => {
+    // Recommended for tests that check HTML. Cheerio is a HTML parser that has a jQuery like API.
+    it('should work using cheerio', async () => {
+      const $ = await next.render$('/')
+      expect($('p').text()).toBe('hello world')
+    })
+
+    // Recommended for tests that need a full browser
+    it('should work using browser', async () => {
+      const browser = await next.browser('/')
+      expect(await browser.elementByCss('p').text()).toBe('hello world')
+    })
+
+    // In case you need the full HTML. Can also use $.html() with cheerio.
+    it('should work with html', async () => {
+      const html = await next.render('/')
+      expect(html).toContain('hello world')
+    })
+
+    // In case you need to test the response object
+    it('should work with fetch', async () => {
+      const res = await next.fetch('/')
+      const html = await res.text()
+      expect(html).toContain('hello world')
+    })
+  }
+)

test/unit/get-files-in-dir.test.ts

@@ -0,0 +1,58 @@
+/* eslint-env jest */
+
+import { getFilesInDir } from 'next/dist/lib/get-files-in-dir'
+import { join } from 'path'
+import fs from 'fs-extra'
+
+const testDir = join(__dirname, 'get-files-in-dir-test')
+
+const srcDir = join(testDir, 'src')
+
+const setupTestDir = async () => {
+  const paths = [
+    '.hidden',
+    'file',
+    'folder1/file1',
+    'folder1/file2',
+    'link',
+    'linkfolder',
+  ]
+
+  await fs.ensureDir(testDir)
+
+  // create src directory structure
+  await fs.ensureDir(srcDir)
+  await fs.outputFile(join(srcDir, '.hidden'), 'hidden')
+  await fs.outputFile(join(srcDir, 'file'), 'file')
+  await fs.outputFile(join(srcDir, 'folder1', 'file1'), 'file1')
+  await fs.outputFile(join(srcDir, 'folder1', 'file2'), 'file2')
+  await fs.ensureSymlink(join(srcDir, 'file'), join(srcDir, 'link'))
+  await fs.ensureSymlink(join(srcDir, 'link'), join(srcDir, 'link-level-2'))
+  await fs.ensureSymlink(
+    join(srcDir, 'link-level-2'),
+    join(srcDir, 'link-level-3')
+  )
+  await fs.ensureSymlink(join(srcDir, 'folder1'), join(srcDir, 'linkfolder'))
+  return paths
+}
+
+describe('getFilesInDir', () => {
+  if (process.platform === 'win32') {
+    it('should skip on windows to avoid symlink issues', () => {})
+    return
+  }
+  afterAll(() => fs.remove(testDir))
+
+  it('should work', async () => {
+    await fs.remove(testDir)
+    await setupTestDir()
+
+    expect(await getFilesInDir(srcDir)).toIncludeAllMembers([
+      '.hidden',
+      'file',
+      'link',
+      'link-level-2',
+      'link-level-3',
+    ])
+  })
+})