Dont apply existing externals path changing to rsc layers (#41744)

We have existing rule for pages ssr that `next/dist/server` and
`next/dist/shared` will not be bundled, but we shouldn't apply it to rsc
layers since the they should bundle the dependencies in their own way.

Adding a test that using `next/head` in the page, since head is exported
from `next/dist/shared`, expect the page is not broken but we don't
expect it's working

## Bug

- [ ] Related issues linked using `fixes #number`
- [x] Integration tests added
- [ ] Errors have a helpful link attached, see `contributing.md`

packages/next/build/webpack-config.ts

@@ -2,7 +2,7 @@ import ReactRefreshWebpackPlugin from 'next/dist/compiled/@next/react-refresh-ut
 import chalk from 'next/dist/compiled/chalk'
 import crypto from 'crypto'
 import { webpack } from 'next/dist/compiled/webpack/webpack'
-import path, { join as pathJoin, relative as relativePath } from 'path'
+import path from 'path'
 import { escapeStringRegexp } from '../shared/lib/escape-regexp'
 import {
   DOT_NEXT_ALIAS,
@@ -62,9 +62,12 @@ import { SubresourceIntegrityPlugin } from './webpack/plugins/subresource-integr
 import { FontLoaderManifestPlugin } from './webpack/plugins/font-loader-manifest-plugin'
 import { getSupportedBrowsers } from './utils'
 
-const NEXT_PROJECT_ROOT = pathJoin(__dirname, '..', '..')
-const NEXT_PROJECT_ROOT_DIST = pathJoin(NEXT_PROJECT_ROOT, 'dist')
-const NEXT_PROJECT_ROOT_DIST_CLIENT = pathJoin(NEXT_PROJECT_ROOT_DIST, 'client')
+const NEXT_PROJECT_ROOT = path.join(__dirname, '..', '..')
+const NEXT_PROJECT_ROOT_DIST = path.join(NEXT_PROJECT_ROOT, 'dist')
+const NEXT_PROJECT_ROOT_DIST_CLIENT = path.join(
+  NEXT_PROJECT_ROOT_DIST,
+  'client'
+)
 
 const babelIncludeRegexes: RegExp[] = [
   /next[\\/]dist[\\/](esm[\\/])?shared[\\/]lib/,
@@ -73,6 +76,8 @@ const babelIncludeRegexes: RegExp[] = [
   /[\\/](strip-ansi|ansi-regex|styled-jsx)[\\/]/,
 ]
 
+const reactPackagesRegex = /^(react(?:$|\/)|react-dom(?:$|\/))/
+
 const staticGenerationAsyncStorageRegex =
   /next[\\/]dist[\\/]client[\\/]components[\\/]static-generation-async-storage/
 
@@ -130,20 +135,12 @@ function isResourceInPackages(
       ? resource.startsWith(packageDirMapping.get(p)! + path.sep)
       : resource.includes(
           path.sep +
-            pathJoin('node_modules', p.replace(/\//g, path.sep)) +
+            path.join('node_modules', p.replace(/\//g, path.sep)) +
             path.sep
         )
   )
 }
 
-const bundledReactImports = [
-  'react',
-  'react-dom',
-  'react/jsx-runtime',
-  'react/jsx-dev-runtime',
-  'next/dist/compiled/react-server-dom-webpack/server.browser',
-]
-
 export function getDefineEnv({
   dev,
   config,
@@ -757,10 +754,12 @@ export default async function getBaseWebpackConfig(
               ),
               [CLIENT_STATIC_FILES_RUNTIME_AMP]:
                 `./` +
-                relativePath(
-                  dir,
-                  pathJoin(NEXT_PROJECT_ROOT_DIST_CLIENT, 'dev', 'amp-dev')
-                ).replace(/\\/g, '/'),
+                path
+                  .relative(
+                    dir,
+                    path.join(NEXT_PROJECT_ROOT_DIST_CLIENT, 'dev', 'amp-dev')
+                  )
+                  .replace(/\\/g, '/'),
             }
           : {}),
         [CLIENT_STATIC_FILES_RUNTIME_MAIN]:
@@ -1079,7 +1078,10 @@ export default async function getBaseWebpackConfig(
 
     // Special internal modules that must be bundled for Server Components.
     if (layer === WEBPACK_LAYERS.server) {
-      if (bundledReactImports.includes(request)) {
+      if (
+        reactPackagesRegex.test(request) ||
+        request === 'next/dist/compiled/react-server-dom-webpack/server.browser'
+      ) {
         return
       }
     }
@@ -1092,9 +1094,13 @@ export default async function getBaseWebpackConfig(
       if (/^(?:next$)/.test(request)) {
         return `commonjs ${request}`
       }
+
       if (/^(react(?:$|\/)|react-dom(?:$|\/))/.test(request)) {
         // override react-dom to server-rendering-stub for server
-        if (request === 'react-dom' && hasAppDir && !isClient) {
+        if (
+          request === 'react-dom' &&
+          (layer === WEBPACK_LAYERS.client || layer === WEBPACK_LAYERS.server)
+        ) {
           request = 'react-dom/server-rendering-stub'
         }
         return `commonjs ${hasAppDir ? 'next/dist/compiled/' : ''}${request}`
@@ -1123,9 +1129,12 @@ export default async function getBaseWebpackConfig(
       // we need to process shared `router/router` and `dynamic`,
       // so that the DefinePlugin can inject process.env values
       const isNextExternal =
-        /next[/\\]dist[/\\](shared|server)[/\\](?!lib[/\\](router[/\\]router|dynamic))/.test(
-          localRes
-        )
+        // Treat next internals as non-external for server layer
+        layer === WEBPACK_LAYERS.server
+          ? false
+          : /next[/\\]dist[/\\](shared|server)[/\\](?!lib[/\\](router[/\\]router|dynamic))/.test(
+              localRes
+            )
 
       if (isNextExternal) {
         // Generate Next.js external import
@@ -1142,7 +1151,7 @@ export default async function getBaseWebpackConfig(
             .replace(/\\/g, '/')
         )
         return `commonjs ${externalRequest}`
-      } else {
+      } else if (layer !== WEBPACK_LAYERS.client) {
         // We don't want to retry local requests
         // with other preferEsm options
         return
@@ -1216,7 +1225,7 @@ export default async function getBaseWebpackConfig(
     // It doesn't matter what the extension is, as we'll transpile it anyway.
     if (config.experimental.transpilePackages && !resolvedExternalPackageDirs) {
       resolvedExternalPackageDirs = new Map()
-      // We need to reoslve all the external package dirs initially.
+      // We need to resolve all the external package dirs initially.
       for (const pkg of config.experimental.transpilePackages) {
         const pkgRes = await resolveExternal(
           dir,
@@ -1251,22 +1260,13 @@ export default async function getBaseWebpackConfig(
         return
       }
 
-      // Treat react packages as external for SSR layer,
-      // then let require-hook mapping them to internals.
+      // Treat react packages and next internals as external for SSR layer,
+      // also map react to builtin ones with require-hook.
       if (layer === WEBPACK_LAYERS.client) {
-        if (
-          [
-            'react',
-            'react/jsx-runtime',
-            'react/jsx-dev-runtime',
-            'react-dom',
-            'scheduler',
-          ].includes(request)
-        ) {
+        if (reactPackagesRegex.test(request)) {
           return `commonjs next/dist/compiled/${request}`
-        } else {
-          return
         }
+        return
       }
 
       if (shouldBeBundled) return
@@ -1644,10 +1644,8 @@ export default async function getBaseWebpackConfig(
                     // react to the direct file path, not the package name. In that case the condition
                     // will be ignored completely.
                     react: 'next/dist/compiled/react',
-                    'react-dom$': isClient
-                      ? 'next/dist/compiled/react-dom/index'
-                      : 'next/dist/compiled/react-dom/server-rendering-stub',
-                    'react-dom/client$': 'next/dist/compiled/react-dom/client',
+                    'react-dom$':
+                      'next/dist/compiled/react-dom/server-rendering-stub',
                   },
                 },
               },
@@ -1687,11 +1685,6 @@ export default async function getBaseWebpackConfig(
               // RSC server compilation loaders
               {
                 test: codeCondition.test,
-                include: [
-                  dir,
-                  // To let the internal client components passing through flight loader
-                  NEXT_PROJECT_ROOT_DIST,
-                ],
                 exclude: [staticGenerationAsyncStorageRegex],
                 issuerLayer: WEBPACK_LAYERS.server,
                 use: {
@@ -1725,7 +1718,6 @@ export default async function getBaseWebpackConfig(
                 // Alias react for switching between default set and share subset.
                 oneOf: [
                   {
-                    // test: codeCondition.test,
                     exclude: [staticGenerationAsyncStorageRegex],
                     issuerLayer: WEBPACK_LAYERS.server,
                     test(req: string) {
@@ -1753,13 +1745,22 @@ export default async function getBaseWebpackConfig(
                     },
                   },
                   {
+                    issuerLayer: WEBPACK_LAYERS.client,
                     test: codeCondition.test,
                     resolve: {
                       alias: {
                         react: 'next/dist/compiled/react',
-                        'react-dom$': isClient
-                          ? 'next/dist/compiled/react-dom/index'
-                          : 'next/dist/compiled/react-dom/server-rendering-stub',
+                        'react-dom$':
+                          'next/dist/compiled/react-dom/server-rendering-stub',
+                      },
+                    },
+                  },
+                  {
+                    test: codeCondition.test,
+                    resolve: {
+                      alias: {
+                        react: 'next/dist/compiled/react',
+                        'react-dom$': 'next/dist/compiled/react-dom',
                         'react-dom/client$':
                           'next/dist/compiled/react-dom/client',
                       },

packages/next/client/components/hooks-client-context.ts

@@ -1,3 +1,5 @@
+'use client'
+
 import { createContext } from 'react'
 
 export const SearchParamsContext = createContext<URLSearchParams>(null as any)

packages/next/client/components/navigation.ts

@@ -1,12 +1,17 @@
-'use client'
+// useLayoutSegments() // Only the segments for the current place. ['children', 'dashboard', 'children', 'integrations'] -> /dashboard/integrations (/dashboard/layout.js would get ['children', 'dashboard', 'children', 'integrations'])
 
 import { useContext, useMemo } from 'react'
+import type { FlightRouterState } from '../../server/app-render'
+import {
+  AppRouterContext,
+  LayoutRouterContext,
+} from '../../shared/lib/app-router-context'
 import {
   SearchParamsContext,
   // ParamsContext,
   PathnameContext,
   // LayoutSegmentsContext,
-} from '../hooks-client-context'
+} from './hooks-client-context'
 
 const INTERNAL_URLSEARCHPARAMS_INSTANCE = Symbol(
   'internal for urlsearchparams readonly'
@@ -91,4 +96,73 @@ export function usePathname(): string {
 export {
   ServerInsertedHTMLContext,
   useServerInsertedHTML,
-} from '../../../shared/lib/server-inserted-html'
+} from '../../shared/lib/server-inserted-html'
+
+// TODO-APP: Move the other router context over to this one
+/**
+ * Get the router methods. For example router.push('/dashboard')
+ */
+export function useRouter(): import('../../shared/lib/app-router-context').AppRouterInstance {
+  return useContext(AppRouterContext)
+}
+
+// TODO-APP: handle parallel routes
+function getSelectedLayoutSegmentPath(
+  tree: FlightRouterState,
+  parallelRouteKey: string,
+  first = true,
+  segmentPath: string[] = []
+): string[] {
+  let node: FlightRouterState
+  if (first) {
+    // Use the provided parallel route key on the first parallel route
+    node = tree[1][parallelRouteKey]
+  } else {
+    // After first parallel route prefer children, if there's no children pick the first parallel route.
+    const parallelRoutes = tree[1]
+    node = parallelRoutes.children ?? Object.values(parallelRoutes)[0]
+  }
+
+  if (!node) return segmentPath
+  const segment = node[0]
+  const segmentValue = Array.isArray(segment) ? segment[1] : segment
+  if (!segmentValue) return segmentPath
+
+  segmentPath.push(segmentValue)
+
+  return getSelectedLayoutSegmentPath(
+    node,
+    parallelRouteKey,
+    false,
+    segmentPath
+  )
+}
+
+// TODO-APP: Expand description when the docs are written for it.
+/**
+ * Get the canonical segment path from the current level to the leaf node.
+ */
+export function useSelectedLayoutSegments(
+  parallelRouteKey: string = 'children'
+): string[] {
+  const { tree } = useContext(LayoutRouterContext)
+  return getSelectedLayoutSegmentPath(tree, parallelRouteKey)
+}
+
+// TODO-APP: Expand description when the docs are written for it.
+/**
+ * Get the segment below the current level
+ */
+export function useSelectedLayoutSegment(
+  parallelRouteKey: string = 'children'
+): string {
+  const selectedLayoutSegments = useSelectedLayoutSegments(parallelRouteKey)
+  if (selectedLayoutSegments.length === 0) {
+    throw new Error('No selected layout segment below the current level')
+  }
+
+  return selectedLayoutSegments[0]
+}
+
+export { redirect } from './redirect'
+export { notFound } from './not-found'

packages/next/client/components/navigation/index.ts

@@ -1,85 +0,0 @@
-// useLayoutSegments() // Only the segments for the current place. ['children', 'dashboard', 'children', 'integrations'] -> /dashboard/integrations (/dashboard/layout.js would get ['children', 'dashboard', 'children', 'integrations'])
-
-import type { FlightRouterState } from '../../../server/app-render'
-import { useContext } from 'react'
-
-import {
-  AppRouterContext,
-  LayoutRouterContext,
-} from '../../../shared/lib/app-router-context'
-
-// TODO-APP: Move the other router context over to this one
-/**
- * Get the router methods. For example router.push('/dashboard')
- */
-export function useRouter(): import('../../../shared/lib/app-router-context').AppRouterInstance {
-  return useContext(AppRouterContext)
-}
-
-// TODO-APP: handle parallel routes
-function getSelectedLayoutSegmentPath(
-  tree: FlightRouterState,
-  parallelRouteKey: string,
-  first = true,
-  segmentPath: string[] = []
-): string[] {
-  let node: FlightRouterState
-  if (first) {
-    // Use the provided parallel route key on the first parallel route
-    node = tree[1][parallelRouteKey]
-  } else {
-    // After first parallel route prefer children, if there's no children pick the first parallel route.
-    const parallelRoutes = tree[1]
-    node = parallelRoutes.children ?? Object.values(parallelRoutes)[0]
-  }
-
-  if (!node) return segmentPath
-  const segment = node[0]
-  const segmentValue = Array.isArray(segment) ? segment[1] : segment
-  if (!segmentValue) return segmentPath
-
-  segmentPath.push(segmentValue)
-
-  return getSelectedLayoutSegmentPath(
-    node,
-    parallelRouteKey,
-    false,
-    segmentPath
-  )
-}
-
-// TODO-APP: Expand description when the docs are written for it.
-/**
- * Get the canonical segment path from the current level to the leaf node.
- */
-export function useSelectedLayoutSegments(
-  parallelRouteKey: string = 'children'
-): string[] {
-  const { tree } = useContext(LayoutRouterContext)
-  return getSelectedLayoutSegmentPath(tree, parallelRouteKey)
-}
-
-// TODO-APP: Expand description when the docs are written for it.
-/**
- * Get the segment below the current level
- */
-export function useSelectedLayoutSegment(
-  parallelRouteKey: string = 'children'
-): string {
-  const selectedLayoutSegments = useSelectedLayoutSegments(parallelRouteKey)
-  if (selectedLayoutSegments.length === 0) {
-    throw new Error('No selected layout segment below the current level')
-  }
-
-  return selectedLayoutSegments[0]
-}
-
-export { redirect } from '../redirect'
-export { notFound } from '../not-found'
-
-export {
-  useSearchParams,
-  usePathname,
-  ServerInsertedHTMLContext,
-  useServerInsertedHTML,
-} from './client'

packages/next/server/app-render.tsx

@@ -3,16 +3,13 @@ import type { LoadComponentsReturnType } from './load-components'
 import type { ServerRuntime } from '../types'
 import type { FontLoaderManifest } from '../build/webpack/plugins/font-loader-manifest-plugin'
 
-// TODO-APP: investigate why require-hook doesn't work for app-render
-
+// Import builtin react directly to avoid require cache conflicts
 import React, { use } from 'next/dist/compiled/react'
-
 import { NotFound as DefaultNotFound } from '../client/components/error'
 
 // this needs to be required lazily so that `next-server` can set
 // the env before we require
 import ReactDOMServer from 'next/dist/compiled/react-dom/server.browser'
-
 import { ParsedUrlQuery } from 'querystring'
 import { NextParsedUrlQuery } from './request-meta'
 import RenderResult from './render-result'

packages/next/shared/lib/app-router-context.ts

@@ -1,3 +1,5 @@
+'use client'
+
 import React from 'react'
 import type { FocusAndScrollRef } from '../../client/components/reducer'
 import type { FlightRouterState, FlightData } from '../../server/app-render'

packages/next/shared/lib/server-inserted-html.tsx

@@ -1,3 +1,5 @@
+'use client'
+
 import React, { useContext } from 'react'
 
 export type ServerInsertedHTMLHook = (callbacks: () => React.ReactNode) => void

test/.stats-app/next.config.js

@@ -1,13 +1,5 @@
-// module.exports = {
-//   experimental: {
-//     appDir: true
-//   }
-// }
-const withBundleAnalyzer = require('@next/bundle-analyzer')({
-  enabled: process.env.ANALYZE === 'true',
-})
-module.exports = withBundleAnalyzer({
+module.exports = {
   experimental: {
     appDir: true,
   },
-})
+}

test/e2e/app-dir/app/app/internal/page.js

@@ -1,9 +1,14 @@
 import Link from 'next/link'
+import Head from 'next/head'
 
 export default function Page() {
   return (
     <div>
       <div>
+        {/* NOTE: next/head will not work in RSC for now but not break either */}
+        <Head>
+          <title>internal-title</title>
+        </Head>
         <Link href="/internal/test/rewrite" id="navigate-rewrite">
           Navigate Rewrite
         </Link>