Emit late streaming meta tags (#47207)

Currently if `notFound()` or `redirect()` is called when the shell was already sent out, we can no longer change the status code and head tags. In that case we inject these specific meta tags into the HTML stream so specific agents can read them.

fix NEXT-220 ([link](https://linear.app/vercel/issue/NEXT-220))

packages/next/src/server/app-render/index.tsx

@@ -1737,7 +1737,35 @@ export async function renderToHTMLOrFlight(
         )
 
         let polyfillsFlushed = false
-        const getServerInsertedHTML = (): Promise<string> => {
+        let flushedErrorMetaTagsUntilIndex = 0
+        const getServerInsertedHTML = () => {
+          // Loop through all the errors that have been captured but not yet
+          // flushed.
+          const errorMetaTags = []
+          for (
+            ;
+            flushedErrorMetaTagsUntilIndex < allCapturedErrors.length;
+            flushedErrorMetaTagsUntilIndex++
+          ) {
+            const error = allCapturedErrors[flushedErrorMetaTagsUntilIndex]
+            if (isNotFoundError(error)) {
+              errorMetaTags.push(
+                <meta name="robots" content="noindex" key={error.digest} />
+              )
+            } else if (isRedirectError(error)) {
+              const redirectUrl = getURLFromRedirectError(error)
+              if (redirectUrl) {
+                errorMetaTags.push(
+                  <meta
+                    httpEquiv="refresh"
+                    content={`0;url=${redirectUrl}`}
+                    key={error.digest}
+                  />
+                )
+              }
+            }
+          }
+
           const flushed = renderToString(
             <>
               {Array.from(serverInsertedHTMLCallbacks).map((callback) =>
@@ -1756,6 +1784,7 @@ export async function renderToHTMLOrFlight(
                       />
                     )
                   })}
+              {errorMetaTags}
             </>
           )
           polyfillsFlushed = true

test/e2e/app-dir/navigation/app/redirect/suspense/page.js

@@ -0,0 +1,41 @@
+import { Suspense } from 'react'
+import { redirect } from 'next/navigation'
+
+function createSuspenseyComponent(Component, { timeout = 0, expire = 10 }) {
+  let result
+  let promise
+  return function Data() {
+    if (result) return result
+    if (!promise)
+      promise = new Promise((resolve) => {
+        setTimeout(() => {
+          result = <Component />
+          setTimeout(() => {
+            result = undefined
+            promise = undefined
+          }, expire)
+          resolve()
+        }, timeout)
+      })
+    throw promise
+  }
+}
+
+function Redirect() {
+  redirect('/redirect/result')
+  return <></>
+}
+
+const SuspenseyRedirect = createSuspenseyComponent(Redirect, {
+  timeout: 300,
+})
+
+export default function () {
+  return (
+    <div className="suspense">
+      <Suspense fallback="fallback">
+        <SuspenseyRedirect />
+      </Suspense>
+    </div>
+  )
+}

test/e2e/app-dir/navigation/navigation.test.ts

@@ -105,9 +105,6 @@ createNextDescribe(
         ).toBe('noindex')
       })
       it('should trigger not-found while streaming', async () => {
-        const initialHtml = await next.render('/not-found/suspense')
-        expect(initialHtml).not.toContain('noindex')
-
         const browser = await next.browser('/not-found/suspense')
         expect(
           await browser.waitForElementByCss('#not-found-component').text()
@@ -261,5 +258,19 @@ createNextDescribe(
         }
       })
     })
+
+    describe('SEO', () => {
+      it('should emit noindex meta tag for not found page when streaming', async () => {
+        const html = await next.render('/not-found/suspense')
+        expect(html).toContain('<meta name="robots" content="noindex"/>')
+      })
+
+      it('should emit refresh meta tag for redirect page when streaming', async () => {
+        const html = await next.render('/redirect/suspense')
+        expect(html).toContain(
+          '<meta http-equiv="refresh" content="0;url=/redirect/result"/>'
+        )
+      })
+    })
   }
 )