Fix bad route path for custom metadata routes (#47286)

We introduced static route `robots.txt` and dynamic route `robots.js` for metadata, it should still allow users to create their own customized version. This issue is caused by a route conflicts. Only append `/route` to page path when there's not ending with `/route`


Fixes #47198
Closes NEXT-850

packages/next/src/build/webpack/loaders/next-app-loader.ts

@@ -75,7 +75,9 @@ async function createAppRouteCode({
   // This, when used with the resolver will give us the pathname to the built
   // route handler file.
   let resolvedPagePath = (await resolver(routePath))!
-  if (isMetadataRoute(name)) {
+
+  const filename = path.parse(resolvedPagePath).name
+  if (isMetadataRoute(name) && filename !== 'route') {
     resolvedPagePath = `next-metadata-route-loader?${stringify({
       pageExtensions,
     })}!${resolvedPagePath + METADATA_RESOURCE_QUERY}`

packages/next/src/lib/metadata/get-metadata-route.ts

@@ -23,7 +23,11 @@ export function normalizeMetadataRoute(page: string) {
     if (route === '/manifest') {
       route += '.webmanifest'
     }
-    route = `${route}/route`
+    // Support both /<metadata-route.ext> and custom routes /<metadata-route>/route.ts.
+    // If it's a metadata file route, we need to append /route to the page.
+    if (!route.endsWith('/route')) {
+      route = `${route}/route`
+    }
   }
   return route
 }

packages/next/src/server/lib/find-page-file.ts

@@ -2,7 +2,7 @@ import { fileExists } from '../../lib/file-exists'
 import { getPagePaths } from '../../shared/lib/page-path/get-page-paths'
 import { nonNullable } from '../../lib/non-nullable'
 import { join, sep, normalize } from 'path'
-import { promises } from 'fs'
+import { promises as fsPromises } from 'fs'
 import { warn } from '../../build/output/log'
 import chalk from '../../lib/chalk'
 import { isMetadataRouteFile } from '../../lib/metadata/is-metadata-route'
@@ -11,7 +11,7 @@ async function isTrueCasePagePath(pagePath: string, pagesDir: string) {
   const pageSegments = normalize(pagePath).split(sep).filter(Boolean)
   const segmentExistsPromises = pageSegments.map(async (segment, i) => {
     const segmentParentDir = join(pagesDir, ...pageSegments.slice(0, i))
-    const parentDirEntries = await promises.readdir(segmentParentDir)
+    const parentDirEntries = await fsPromises.readdir(segmentParentDir)
     return parentDirEntries.includes(segment)
   })
 
@@ -104,11 +104,13 @@ export function createValidFileMatcher(
    */
 
   /**
-   * Match the file if it's a metadata route file, static: if the file is a static metadata file
-   *
+   * Match the file if it's a metadata route file, static: if the file is a static metadata file.
+   * It needs to be a file which doesn't match the custom metadata routes e.g. `app/robots.txt/route.js`
    */
   function isMetadataFile(filePath: string) {
-    const appDirRelativePath = filePath.replace(appDirPath || '', '')
+    const appDirRelativePath = appDirPath
+      ? filePath.replace(appDirPath, '')
+      : filePath
 
     return isMetadataRouteFile(appDirRelativePath, pageExtensions, true)
   }

test/e2e/app-dir/app-routes/app-custom-routes.test.ts

@@ -521,6 +521,17 @@ createNextDescribe(
       })
     })
 
+    describe('customized metadata routes', () => {
+      it('should work if conflict with metadata routes convention', async () => {
+        const res = await next.fetch('/robots.txt')
+
+        expect(res.status).toEqual(200)
+        expect(await res.text()).toBe(
+          'User-agent: *\nAllow: /\n\nSitemap: https://www.example.com/sitemap.xml'
+        )
+      })
+    })
+
     if (isNextDev) {
       describe('lowercase exports', () => {
         it.each([

test/e2e/app-dir/app-routes/app/robots.txt/route.ts

@@ -0,0 +1,6 @@
+export async function GET() {
+  return new Response(`User-agent: *
+Allow: /
+
+Sitemap: https://www.example.com/sitemap.xml`)
+}

test/e2e/app-dir/app-routes/tsconfig.json

@@ -0,0 +1,24 @@
+{
+  "compilerOptions": {
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": false,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "incremental": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ]
+  },
+  "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"],
+  "exclude": ["node_modules", "**/*.test.ts"]
+}

test/unit/find-page-file.test.ts

@@ -4,7 +4,6 @@ import {
   createValidFileMatcher,
 } from 'next/dist/server/lib/find-page-file'
 import { normalizePagePath } from 'next/dist/shared/lib/page-path/normalize-page-path'
-
 import { join } from 'path'
 
 const resolveDataDir = join(__dirname, 'isolated', '_resolvedata')
@@ -71,9 +70,9 @@ describe('createPageFileMatcher', () => {
   })
 
   describe('isMetadataRouteFile', () => {
-    const pageExtensions = ['tsx', 'ts', 'jsx', 'js']
-    const fileMatcher = createValidFileMatcher(pageExtensions, 'app')
     it('should determine top level metadata routes', () => {
+      const pageExtensions = ['tsx', 'ts', 'jsx', 'js']
+      const fileMatcher = createValidFileMatcher(pageExtensions, 'app')
       expect(fileMatcher.isMetadataFile('app/route.js')).toBe(false)
       expect(fileMatcher.isMetadataFile('app/page.js')).toBe(false)
       expect(fileMatcher.isMetadataFile('pages/index.js')).toBe(false)