Support generate dynamic sitemaps for dynamic routes (#48867)

### What

For dynamic routes you might have different sitemap for different params

* Unloack using `sitemap.[ext]` in your app everywhere
* Support `generateSitemaps()` to create multiple sitemaps at the same time

### How

* Change the metadata regex to allow use sitemap in every routes
* Similar approach to `generateImageMetadata`, we make `sitemap.js` under dynamic routes to a catch all routes, and it can have multiple routes

Closes NEXT-1054

packages/next/src/build/webpack/loaders/next-metadata-route-loader.ts

@@ -101,7 +101,7 @@ const imageModule = { ..._imageModule }
 const handler = imageModule.default
 const generateImageMetadata = imageModule.generateImageMetadata
 
-export async function GET(req, ctx) {
+export async function GET(_, ctx) {
   const { __metadata_id__ = [], ...params } = ctx.params
   const id = __metadata_id__[0]
   const imageMetadata = generateImageMetadata ? await generateImageMetadata({ params }) : null
@@ -118,6 +118,46 @@ export async function GET(req, ctx) {
 `
 }
 
+function getDynamicSiteMapRouteCode(resourcePath: string) {
+  // generateSitemaps
+  return `\
+import { NextResponse } from 'next/server'
+import * as _sitemapModule from ${JSON.stringify(resourcePath)}
+import { resolveRouteData } from 'next/dist/build/webpack/loaders/metadata/resolve-route-data'
+
+const sitemapModule = { ..._sitemapModule }
+const handler = sitemapModule.default
+const generateSitemaps = sitemapModule.generateSitemaps
+const contentType = ${JSON.stringify(getContentType(resourcePath))}
+const fileType = ${JSON.stringify(getFilenameAndExtension(resourcePath).name)}
+
+export async function GET(_, ctx) {
+  const sitemaps = generateSitemaps ? await generateSitemaps() : null
+  let id = undefined
+
+  if (sitemaps) {
+    const { __metadata_id__ = [] } = ctx.params
+    const targetId = __metadata_id__[0]
+    id = sitemaps.find((item) => item.id.toString() === targetId)?.id
+    if (id == null) {
+      return new NextResponse(null, {
+        status: 404,
+      })
+    }
+  }
+
+  const data = await handler({ id })
+  const content = resolveRouteData(data, fileType)
+
+  return new NextResponse(content, {
+    headers: {
+      'Content-Type': contentType,
+      'Cache-Control': ${JSON.stringify(cacheHeader.revalidate)},
+    },
+  })
+}
+`
+}
 // `import.meta.url` is the resource name of the current module.
 // When it's static route, it could be favicon.ico, sitemap.xml, robots.txt etc.
 // TODO-METADATA: improve the cache control strategy
@@ -131,12 +171,10 @@ const nextMetadataRouterLoader: webpack.LoaderDefinitionFunction<MetadataRouteLo
 
     let code = ''
     if (isDynamic) {
-      if (
-        fileBaseName === 'sitemap' ||
-        fileBaseName === 'robots' ||
-        fileBaseName === 'manifest'
-      ) {
+      if (fileBaseName === 'robots' || fileBaseName === 'manifest') {
         code = getDynamicTextRouteCode(resourcePath)
+      } else if (fileBaseName === 'sitemap') {
+        code = getDynamicSiteMapRouteCode(resourcePath)
       } else {
         code = getDynamicImageRouteCode(resourcePath)
       }

packages/next/src/lib/metadata/get-metadata-route.ts

@@ -1,6 +1,7 @@
 import { isMetadataRoute, isMetadataRouteFile } from './is-metadata-route'
 import path from '../../shared/lib/isomorphic/path'
 import { djb2Hash } from '../../shared/lib/hash'
+import { isDynamicRoute } from '../../shared/lib/router/utils'
 
 /*
  * If there's special convention like (...) or @ in the page path,
@@ -30,19 +31,18 @@ export function getMetadataRouteSuffix(page: string) {
  */
 export function normalizeMetadataRoute(page: string) {
   let route = page
+  let suffix = ''
   if (isMetadataRoute(page)) {
-    // Remove the file extension, e.g. /route-path/robots.txt -> /route-path
-    const pathnamePrefix = page.slice(0, -(path.basename(page).length + 1))
-    const suffix = getMetadataRouteSuffix(pathnamePrefix)
-
-    if (route === '/sitemap') {
-      route += '.xml'
-    }
     if (route === '/robots') {
       route += '.txt'
-    }
-    if (route === '/manifest') {
+    } else if (route === '/manifest') {
       route += '.webmanifest'
+    } else if (route.endsWith('/sitemap')) {
+      route += '.xml'
+    } else {
+      // Remove the file extension, e.g. /route-path/robots.txt -> /route-path
+      const pathnamePrefix = page.slice(0, -(path.basename(page).length + 1))
+      suffix = getMetadataRouteSuffix(pathnamePrefix)
     }
     // Support both /<metadata-route.ext> and custom routes /<metadata-route>/route.ts.
     // If it's a metadata file route, we need to append /[id]/route to the page.
@@ -50,16 +50,19 @@ export function normalizeMetadataRoute(page: string) {
       const isStaticMetadataFile = isMetadataRouteFile(route, [], true)
       const { dir, name: baseName, ext } = path.parse(route)
 
-      const isSingleRoute =
-        page.startsWith('/sitemap') ||
-        page.startsWith('/robots') ||
-        page.startsWith('/manifest') ||
-        isStaticMetadataFile
+      // If it's dynamic routes, we need to append [[...__metadata_id__]] to the page;
+      // If it's static routes, we need to append nothing to the page.
+      // If its special routes like robots.txt and manifest.webmanifest, we leave them as static routes.
+      const isStaticRoute =
+        !isDynamicRoute(route) &&
+        (page.startsWith('/robots') ||
+          page.startsWith('/manifest') ||
+          isStaticMetadataFile)
 
       route = path.posix.join(
         dir,
         `${baseName}${suffix ? `-${suffix}` : ''}${ext}`,
-        isSingleRoute ? '' : '[[...__metadata_id__]]',
+        isStaticRoute ? '' : '[[...__metadata_id__]]',
         'route'
       )
     }

packages/next/src/lib/metadata/is-metadata-route.ts

@@ -47,13 +47,6 @@ export function isMetadataRouteFile(
           : ''
       }`
     ),
-    new RegExp(
-      `^[\\\\/]sitemap${
-        withExtension
-          ? `\\.${getExtensionRegexString(pageExtensions.concat('xml'))}`
-          : ''
-      }`
-    ),
     new RegExp(
       `^[\\\\/]manifest${
         withExtension
@@ -64,7 +57,13 @@ export function isMetadataRouteFile(
       }`
     ),
     new RegExp(`^[\\\\/]favicon\\.ico$`),
-    // TODO-METADATA: add dynamic routes for metadata images
+    new RegExp(
+      `[\\\\/]sitemap${
+        withExtension
+          ? `\\.${getExtensionRegexString(pageExtensions.concat('xml'))}`
+          : ''
+      }`
+    ),
     new RegExp(
       `[\\\\/]${STATIC_METADATA_IMAGES.icon.filename}${
         withExtension

test/e2e/app-dir/metadata-dynamic-routes/app/(group)/dynamic/[size]/sitemap.ts

@@ -0,0 +1,18 @@
+import { MetadataRoute } from 'next'
+
+export async function generateSitemaps() {
+  return [{ id: 0 }, { id: 1 }, { id: 2 }, { id: 3 }]
+}
+
+export default function sitemap({ id }): MetadataRoute.Sitemap {
+  return [
+    {
+      url: `https://example.com/dynamic/${id}`,
+      lastModified: '2021-01-01',
+    },
+    {
+      url: `https://example.com/dynamic/${id}/about`,
+      lastModified: '2021-01-01',
+    },
+  ]
+}

test/e2e/app-dir/metadata-dynamic-routes/index.test.ts

@@ -158,6 +158,20 @@ createNextDescribe(
         ])
       })
 
+      it('should support generate multi sitemaps with generateSitemaps', async () => {
+        const ids = [0, 1, 2, 3]
+        function fetchSitemap(id) {
+          return next
+            .fetch(`/dynamic/small/sitemap.xml/${id}`)
+            .then((res) => res.text())
+        }
+
+        for (const id of ids) {
+          const text = await fetchSitemap(id)
+          expect(text).toContain(`<loc>https://example.com/dynamic/${id}</loc>`)
+        }
+      })
+
       it('should fill params into dynamic routes url of metadata images', async () => {
         const $ = await next.render$('/dynamic/big')
         const ogImageUrl = $('meta[property="og:image"]').attr('content')

test/e2e/app-dir/metadata-dynamic-routes/tsconfig.json

@@ -1,24 +0,0 @@
-{
-  "compilerOptions": {
-    "lib": ["dom", "dom.iterable", "esnext"],
-    "allowJs": true,
-    "skipLibCheck": true,
-    "strict": false,
-    "forceConsistentCasingInFileNames": true,
-    "noEmit": true,
-    "incremental": true,
-    "esModuleInterop": true,
-    "module": "esnext",
-    "moduleResolution": "node",
-    "resolveJsonModule": true,
-    "isolatedModules": true,
-    "jsx": "preserve",
-    "plugins": [
-      {
-        "name": "next"
-      }
-    ]
-  },
-  "include": ["next-env.d.ts", ".next/types/**/*.ts", "**/*.ts", "**/*.tsx"],
-  "exclude": ["node_modules", "**/*.test.ts"]
-}

test/e2e/app-dir/metadata/metadata.test.ts

@@ -744,7 +744,7 @@ createNextDescribe(
         expect(invalidRobotsResponse.status).toBe(404)
       })
 
-      it('should support root dir sitemap.xml', async () => {
+      it('should support sitemap.xml under every routes', async () => {
         const res = await next.fetch('/sitemap.xml')
         expect(res.headers.get('content-type')).toBe('application/xml')
         const sitemap = await res.text()
@@ -753,7 +753,7 @@ createNextDescribe(
           '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
         )
         const invalidSitemapResponse = await next.fetch('/title/sitemap.xml')
-        expect(invalidSitemapResponse.status).toBe(404)
+        expect(invalidSitemapResponse.status).toBe(200)
       })
 
       it('should support static manifest.webmanifest', async () => {

test/unit/find-page-file.test.ts

@@ -81,7 +81,7 @@ describe('createPageFileMatcher', () => {
       expect(fileMatcher.isMetadataFile('app/path/robots.txt')).toBe(false)
 
       expect(fileMatcher.isMetadataFile('app/sitemap.xml')).toBe(true)
-      expect(fileMatcher.isMetadataFile('app/path/sitemap.xml')).toBe(false)
+      expect(fileMatcher.isMetadataFile('app/path/sitemap.xml')).toBe(true)
     })
   })
 })