Fix shallow routing with rewrites/middleware (#37716)

Shallow route changes did not work for rewritten pages when Middleware
was used, and made hard refreshes, although it was possible with static rewrites.

This happened because the router has a manifest of the static rewrites,
allowing static rewrites to map the route before comparing with the
local cache.

Middleware rewrites are dynamic and happening on the server, so we
can't send a manifest easily. This means that we need to propagate
the rewrites from the data requests into the components cache in
the router to make sure we have cache hits and don't miss.

This commit does exactly that and adds a test to verify that it works.

This fixes #37072 and fixes #31680

_Note:_ there's one thing that is somewhat an issue though: if the first
page the user lands on is a rewritten page, and will try to make a
shallow navigation to the same page--we will make a `fetch` request.
This is because we don't have any client cache of the `rewrite` we just
had.

## Bug

- [x] Related issues linked using `fixes #number`
- [x] Integration tests added
- [ ] Errors have helpful link attached, see `contributing.md`

packages/next/shared/lib/router/router.ts

@@ -1361,7 +1361,8 @@ export default class Router implements BaseRouter {
       }
 
       // shallow routing is only allowed for same page URL changes.
-      const isValidShallowRoute = options.shallow && nextState.route === route
+      const isValidShallowRoute =
+        options.shallow && nextState.route === (routeInfo.route ?? route)
 
       const shouldScroll = options.scroll ?? !isValidShallowRoute
       const resetScroll = shouldScroll ? { x: 0, y: 0 } : null
@@ -1524,7 +1525,7 @@ export default class Router implements BaseRouter {
   }
 
   async getRouteInfo({
-    route,
+    route: requestedRoute,
     pathname,
     query,
     as,
@@ -1542,6 +1543,14 @@ export default class Router implements BaseRouter {
     locale: string | undefined
     isPreview: boolean
   }) {
+    /**
+     * This `route` binding can change if there's a rewrite
+     * so we keep a reference to the original requested route
+     * so we can store the cache for it and avoid re-requesting every time
+     * for shallow routing purposes.
+     */
+    let route = requestedRoute
+
     try {
       let existingInfo: PrivateRouteInfo | undefined = this.components[route]
       if (routeProps.shallow && existingInfo && this.route === route) {
@@ -1593,7 +1602,11 @@ export default class Router implements BaseRouter {
         // Check again the cache with the new destination.
         existingInfo = this.components[route]
         if (routeProps.shallow && existingInfo && this.route === route) {
-          return existingInfo
+          // If we have a match with the current route due to rewrite,
+          // we can copy the existing information to the rewritten one.
+          // Then, we return the information along with the matched route.
+          this.components[requestedRoute] = { ...existingInfo, route }
+          return { ...existingInfo, route }
         }
 
         cachedRouteInfo =
@@ -1730,6 +1743,12 @@ export default class Router implements BaseRouter {
       routeInfo.route = route
       routeInfo.resolvedAs = resolvedAs
       this.components[route] = routeInfo
+
+      // If the route was rewritten in the process of fetching data,
+      // we update the cache to allow hitting the same data for shallow requests.
+      if (route !== requestedRoute) {
+        this.components[requestedRoute] = { ...routeInfo, route }
+      }
       return routeInfo
     } catch (err) {
       return this.handleRouteInfoError(

test/e2e/middleware-general/app/middleware.js

@@ -41,6 +41,11 @@ export async function middleware(request) {
     return NextResponse.next()
   }
 
+  if (url.pathname === '/sha') {
+    url.pathname = '/shallow'
+    return NextResponse.rewrite(url)
+  }
+
   if (url.pathname.startsWith('/fetch-user-agent-default')) {
     try {
       const apiRoute = new URL(url)

test/e2e/middleware-general/app/next.config.js

@@ -22,6 +22,10 @@ module.exports = {
         source: '/rewrite-2',
         destination: '/about/a',
       },
+      {
+        source: '/sha',
+        destination: '/shallow',
+      },
       {
         source: '/rewrite-3',
         destination: '/blog/middleware-rewrite',

test/e2e/middleware-general/app/pages/shallow.js

@@ -0,0 +1,43 @@
+import Link from 'next/link'
+import { useRouter } from 'next/router'
+
+export default function Shallow({ message }) {
+  const { pathname, query } = useRouter()
+  return (
+    <div>
+      <ul>
+        <li id="message-contents">{message}</li>
+        <li>
+          <Link href="/sha?hello=world" shallow>
+            <a id="shallow-link">Shallow link to ?hello=world</a>
+          </Link>
+        </li>
+        <li>
+          <Link href="/sha?hello=goodbye">
+            <a id="deep-link">Deep link to ?hello=goodbye</a>
+          </Link>
+        </li>
+        <li>
+          <h1 id="pathname">
+            Current path: <code>{pathname}</code>
+          </h1>
+        </li>
+        <li>
+          <h2 id="query" data-query-hello={query.hello}>
+            Current query: <code>{JSON.stringify(query)}</code>
+          </h2>
+        </li>
+      </ul>
+    </div>
+  )
+}
+
+let i = 0
+
+export const getServerSideProps = () => {
+  return {
+    props: {
+      message: `Random: ${++i}${Math.random()}`,
+    },
+  }
+}

test/e2e/middleware-general/test/index.test.ts

@@ -476,6 +476,34 @@ describe('Middleware Runtime', () => {
     await browser.waitForElementByCss('.refreshed')
     expect(await browser.eval('window.__SAME_PAGE')).toBeUndefined()
   })
+
+  it('allows shallow linking with middleware', async () => {
+    const browser = await webdriver(next.url, '/sha')
+    const getMessageContents = () =>
+      browser.elementById('message-contents').text()
+    const ssrMessage = await getMessageContents()
+    const requests: string[] = []
+
+    browser.on('request', (x) => {
+      requests.push(x.url())
+    })
+
+    browser.elementById('deep-link').click()
+    browser.waitForElementByCss('[data-query-hello="goodbye"]')
+    const deepLinkMessage = await getMessageContents()
+    expect(deepLinkMessage).not.toEqual(ssrMessage)
+
+    // Changing the route with a shallow link should not cause a server request
+    browser.elementById('shallow-link').click()
+    browser.waitForElementByCss('[data-query-hello="world"]')
+    expect(await getMessageContents()).toEqual(deepLinkMessage)
+
+    // Check that no server requests were made to ?hello=world,
+    // as it's a shallow request.
+    expect(requests).toEqual([
+      `${next.url}/_next/data/${next.buildId}/en/sha.json?hello=goodbye`,
+    ])
+  })
 })
 
 function readMiddlewareJSON(response) {