Fix RSC navigation when overriding headers in middleware (#46049)

Fixes NEXT-583

Ran into this when looking into adding an integration test for the RSC
normalizing PR.

Middleware has the ability to override `headers` which causes
client-side navigation to break as it'll remove the `rsc` header which
causes Next.js to respond with the HTML response instead of the RSC
payload.

This PR ensures the RSC headers are always copied over as middleware
does not get access to them.

<!--
Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that
you follow the checklist sections below.
Choose the right checklist for the change(s) that you're making:
-->

## Bug

- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Errors have a helpful link attached, see
[`contributing.md`](https://github.com/vercel/next.js/blob/canary/contributing.md)

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the
feature request has been accepted for implementation before opening a
PR.
- [ ] Related issues linked using `fixes #number`
- [ ]
[e2e](https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs)
tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see
[`contributing.md`](https://github.com/vercel/next.js/blob/canary/contributing.md)

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm build && pnpm lint`
- [ ] The "examples guidelines" are followed from [our contributing
doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)

packages/next/src/server/web/adapter.ts

@@ -75,10 +75,16 @@ export async function adapter(params: {
   }
 
   const requestHeaders = fromNodeHeaders(params.request.headers)
+  const flightHeaders = new Map()
   // Parameters should only be stripped for middleware
   if (!isEdgeRendering) {
     for (const param of FLIGHT_PARAMETERS) {
-      requestHeaders.delete(param.toString().toLowerCase())
+      const key = param.toString().toLowerCase()
+      const value = requestHeaders.get(key)
+      if (value) {
+        flightHeaders.set(key, requestHeaders.get(key))
+        requestHeaders.delete(key)
+      }
     }
   }
 
@@ -192,8 +198,29 @@ export async function adapter(params: {
     }
   }
 
+  const finalResponse = response ? response : NextResponse.next()
+
+  // Flight headers are not overridable / removable so they are applied at the end.
+  const middlewareOverrideHeaders = finalResponse.headers.get(
+    'x-middleware-override-headers'
+  )
+  const overwrittenHeaders: string[] = []
+  if (middlewareOverrideHeaders) {
+    for (const [key, value] of flightHeaders) {
+      finalResponse.headers.set(`x-middleware-request-${key}`, value)
+      overwrittenHeaders.push(key)
+    }
+
+    if (overwrittenHeaders.length > 0) {
+      finalResponse.headers.set(
+        'x-middleware-override-headers',
+        middlewareOverrideHeaders + ',' + overwrittenHeaders.join(',')
+      )
+    }
+  }
+
   return {
-    response: response || NextResponse.next(),
+    response: finalResponse,
     waitUntil: Promise.all(event[waitUntilSymbol]),
   }
 }

test/e2e/app-dir/app/app/searchparams-normalization-bug/client-component.js

@@ -0,0 +1,20 @@
+'use client'
+import { usePathname, useRouter } from 'next/navigation'
+import { useCallback } from 'react'
+
+export default function Button({ value, children }) {
+  const router = useRouter()
+  const pathname = usePathname()
+
+  const setSearchParams = useCallback(() => {
+    const params = new URLSearchParams()
+    params.set('val', value)
+    router.replace(`${pathname}?${params}`)
+  }, [router, pathname, value])
+
+  return (
+    <button id={`button-${value}`} onClick={setSearchParams}>
+      {children}
+    </button>
+  )
+}

test/e2e/app-dir/app/app/searchparams-normalization-bug/page.js

@@ -0,0 +1,15 @@
+import Button from './client-component'
+import { headers } from 'next/headers'
+export default function Page() {
+  const headerStore = headers()
+  const headerValue = headerStore.get('test') || 'empty'
+
+  return (
+    <>
+      <h1 id={`header-${headerValue}`}>Header value: {headerValue}</h1>
+      <Button value="a">Set A</Button>
+      <Button value="b">Set B</Button>
+      <Button value="c">Set C</Button>
+    </>
+  )
+}

test/e2e/app-dir/app/index.test.ts

@@ -1455,6 +1455,36 @@ createNextDescribe(
           const val2 = await browser.elementByCss('#value-2').text()
           expect(val1).toBe(val2)
         })
+
+        it('middleware overriding headers', async () => {
+          const browser = await next.browser('/searchparams-normalization-bug')
+          await browser.eval(`window.didFullPageTransition = 'no'`)
+          expect(await browser.elementByCss('#header-empty').text()).toBe(
+            'Header value: empty'
+          )
+          expect(
+            await browser
+              .elementByCss('#button-a')
+              .click()
+              .waitForElementByCss('#header-a')
+              .text()
+          ).toBe('Header value: a')
+          expect(
+            await browser
+              .elementByCss('#button-b')
+              .click()
+              .waitForElementByCss('#header-b')
+              .text()
+          ).toBe('Header value: b')
+          expect(
+            await browser
+              .elementByCss('#button-c')
+              .click()
+              .waitForElementByCss('#header-c')
+              .text()
+          ).toBe('Header value: c')
+          expect(await browser.eval(`window.didFullPageTransition`)).toBe('no')
+        })
       })
 
       describe('should support React fetch instrumentation', () => {

test/e2e/app-dir/app/middleware.js

@@ -6,6 +6,17 @@ import { NextResponse } from 'next/server'
  * @returns {NextResponse | undefined}
  */
 export function middleware(request) {
+  if (request.nextUrl.pathname === '/searchparams-normalization-bug') {
+    const headers = new Headers(request.headers)
+    headers.set('test', request.nextUrl.searchParams.get('val') || '')
+    const response = NextResponse.next({
+      request: {
+        headers,
+      },
+    })
+
+    return response
+  }
   if (request.nextUrl.pathname === '/exists-but-not-routed') {
     return NextResponse.rewrite(new URL('/dashboard', request.url))
   }