Prefetch SSG Data (#10127)

* Prefetch SSG Data

* Update packages/next/client/page-loader.js

Co-Authored-By: JJ Kasper <jj@jjsweb.site>

* Revert router.ts

* Revert link.tsx

* undo change

* mimmic existing

* simplify

* Prefetch href and asPath

* fix load

* dedupe prefetchAs

* Inject script tag on hover

* comment prefetchAs

* minify code

* introduce lazy files

* Add some breathing room

* correct default type

* Prefetch non-dynamic data

* Prefetch dynamic route data

* Fix size test

* Humanize code

* add tests

* Disable code

* Only generate modern version in modern mode

* Extract function helper

* add comments

* Filter out dynamic route to simplify manifest size

* add test

Co-authored-by: JJ Kasper <jj@jjsweb.site>

packages/next/build/index.ts

@@ -1,6 +1,7 @@
 import chalk from 'chalk'
 import ciEnvironment from 'ci-info'
 import crypto from 'crypto'
+import devalue from 'devalue'
 import escapeStringRegexp from 'escape-string-regexp'
 import findUp from 'find-up'
 import fs from 'fs'
@@ -28,6 +29,7 @@ import { recursiveReadDir } from '../lib/recursive-readdir'
 import { verifyTypeScriptSetup } from '../lib/verifyTypeScriptSetup'
 import {
   BUILD_MANIFEST,
+  CLIENT_STATIC_FILES_PATH,
   EXPORT_DETAIL,
   EXPORT_MARKER,
   PAGES_MANIFEST,
@@ -851,6 +853,11 @@ export default async function build(dir: string, conf = null): Promise<void> {
       JSON.stringify(prerenderManifest),
       'utf8'
     )
+    await generateClientSsgManifest(prerenderManifest, {
+      distDir,
+      buildId,
+      isModern: !!config.experimental.modern,
+    })
   } else {
     const prerenderManifest: PrerenderManifest = {
       version: 2,
@@ -863,6 +870,8 @@ export default async function build(dir: string, conf = null): Promise<void> {
       JSON.stringify(prerenderManifest),
       'utf8'
     )
+    // No need to call this fn as we already emitted a default SSG manifest:
+    // await generateClientSsgManifest(prerenderManifest, { distDir, buildId })
   }
 
   await fsWriteFile(
@@ -961,3 +970,36 @@ export default async function build(dir: string, conf = null): Promise<void> {
 
   await telemetry.flush()
 }
+
+function generateClientSsgManifest(
+  prerenderManifest: PrerenderManifest,
+  {
+    buildId,
+    distDir,
+    isModern,
+  }: { buildId: string; distDir: string; isModern: boolean }
+) {
+  const ssgPages: Set<string> = new Set<string>([
+    ...Object.entries(prerenderManifest.routes)
+      // Filter out dynamic routes
+      .filter(([, { srcRoute }]) => srcRoute == null)
+      .map(([route]) => route),
+    ...Object.keys(prerenderManifest.dynamicRoutes),
+  ])
+
+  const clientSsgManifestPaths = [
+    '_ssgManifest.js',
+    isModern && '_ssgManifest.module.js',
+  ]
+    .filter(Boolean)
+    .map(f => path.join(`${CLIENT_STATIC_FILES_PATH}/${buildId}`, f as string))
+  const clientSsgManifestContent = `self.__SSG_MANIFEST=${devalue(
+    ssgPages
+  )};self.__SSG_MANIFEST_CB&&self.__SSG_MANIFEST_CB()`
+  clientSsgManifestPaths.forEach(clientSsgManifestPath =>
+    fs.writeFileSync(
+      path.join(distDir, clientSsgManifestPath),
+      clientSsgManifestContent
+    )
+  )
+}

packages/next/build/webpack/plugins/build-manifest-plugin.ts

@@ -147,6 +147,23 @@ export default class BuildManifestPlugin {
           }
         }
 
+        // Add the runtime ssg manifest file as a lazy-loaded file dependency.
+        // We also stub this file out for development mode (when it is not
+        // generated).
+        const srcEmptySsgManifest = `self.__SSG_MANIFEST=new Set;self.__SSG_MANIFEST_CB&&self.__SSG_MANIFEST_CB()`
+
+        const ssgManifestPath = `${CLIENT_STATIC_FILES_PATH}/${this.buildId}/_ssgManifest.js`
+        assetMap.lowPriorityFiles.push(ssgManifestPath)
+        compilation.assets[ssgManifestPath] = new RawSource(srcEmptySsgManifest)
+
+        if (this.modern) {
+          const ssgManifestPathModern = `${CLIENT_STATIC_FILES_PATH}/${this.buildId}/_ssgManifest.module.js`
+          assetMap.lowPriorityFiles.push(ssgManifestPathModern)
+          compilation.assets[ssgManifestPathModern] = new RawSource(
+            srcEmptySsgManifest
+          )
+        }
+
         assetMap.pages = Object.keys(assetMap.pages)
           .sort()
           // eslint-disable-next-line

packages/next/client/page-loader.js

@@ -1,4 +1,8 @@
+import { parse } from 'url'
 import mitt from '../next-server/lib/mitt'
+import { isDynamicRoute } from './../next-server/lib/router/utils/is-dynamic'
+import { getRouteMatcher } from './../next-server/lib/router/utils/route-matcher'
+import { getRouteRegex } from './../next-server/lib/router/utils/route-regex'
 
 function hasRel(rel, link) {
   try {
@@ -18,6 +22,7 @@ const relPrefetch =
 
 const hasNoModule = 'noModule' in document.createElement('script')
 
+/** @param {string} route */
 function normalizeRoute(route) {
   if (route[0] !== '/') {
     throw new Error(`Route name should start with a "/", got "${route}"`)
@@ -62,6 +67,16 @@ export default class PageLoader {
         }
       })
     }
+    /** @type {Promise<Set<string>>} */
+    this.promisedSsgManifest = new Promise(resolve => {
+      if (window.__SSG_MANIFEST) {
+        resolve(window.__SSG_MANIFEST)
+      } else {
+        window.__SSG_MANIFEST_CB = () => {
+          resolve(window.__SSG_MANIFEST)
+        }
+      }
+    })
   }
 
   // Returns a promise for the dependencies for a particular route
@@ -76,6 +91,89 @@ export default class PageLoader {
     )
   }
 
+  /**
+   * @param {string} href the route href (file-system path)
+   * @param {string} asPath the URL as shown in browser (virtual path); used for dynamic routes
+   */
+  getDataHref(href, asPath) {
+    const getHrefForSlug = (/** @type string */ path) =>
+      `${this.assetPrefix}/_next/data/${this.buildId}${
+        path === '/' ? '/index' : path
+      }.json`
+
+    const { pathname: hrefPathname, query } = parse(href, true)
+    const { pathname: asPathname } = parse(asPath)
+
+    const route = normalizeRoute(hrefPathname)
+
+    let isDynamic = isDynamicRoute(route),
+      interpolatedRoute
+    if (isDynamic) {
+      const dynamicRegex = getRouteRegex(route)
+      const dynamicGroups = dynamicRegex.groups
+      const dynamicMatches =
+        // Try to match the dynamic route against the asPath
+        getRouteMatcher(dynamicRegex)(asPathname) ||
+        // Fall back to reading the values from the href
+        // TODO: should this take priority; also need to change in the router.
+        query
+
+      interpolatedRoute = route
+      if (
+        !Object.keys(dynamicGroups).every(param => {
+          let value = dynamicMatches[param]
+          const repeat = dynamicGroups[param].repeat
+
+          // support single-level catch-all
+          // TODO: more robust handling for user-error (passing `/`)
+          if (repeat && !Array.isArray(value)) value = [value]
+
+          return (
+            param in dynamicMatches &&
+            // Interpolate group into data URL if present
+            (interpolatedRoute = interpolatedRoute.replace(
+              `[${repeat ? '...' : ''}${param}]`,
+              repeat
+                ? value.map(encodeURIComponent).join('/')
+                : encodeURIComponent(value)
+            ))
+          )
+        })
+      ) {
+        interpolatedRoute = '' // did not satisfy all requirements
+
+        // n.b. We ignore this error because we handle warning for this case in
+        // development in the `<Link>` component directly.
+      }
+    }
+
+    return isDynamic
+      ? interpolatedRoute && getHrefForSlug(interpolatedRoute)
+      : getHrefForSlug(route)
+  }
+
+  /**
+   * @param {string} href the route href (file-system path)
+   * @param {string} asPath the URL as shown in browser (virtual path); used for dynamic routes
+   */
+  prefetchData(href, asPath) {
+    const { pathname: hrefPathname } = parse(href, true)
+    const route = normalizeRoute(hrefPathname)
+    return this.promisedSsgManifest.then(
+      (s, _dataHref) =>
+        // Check if the route requires a data file
+        s.has(route) &&
+        // Try to generate data href, noop when falsy
+        (_dataHref = this.getDataHref(href, asPath)) &&
+        // noop when data has already been prefetched (dedupe)
+        !document.querySelector(
+          `link[rel="${relPrefetch}"][href^="${_dataHref}"]`
+        ) &&
+        // Inject the `<link rel=prefetch>` tag for above computed `href`.
+        appendLink(_dataHref, relPrefetch, 'fetch')
+    )
+  }
+
   loadPage(route) {
     return this.loadPageScript(route).then(v => v.page)
   }
@@ -206,6 +304,10 @@ export default class PageLoader {
     register()
   }
 
+  /**
+   * @param {string} route
+   * @param {boolean} [isDependency]
+   */
   prefetch(route, isDependency) {
     // https://github.com/GoogleChromeLabs/quicklink/blob/453a661fa1fa940e2d2e044452398e38c67a98fb/src/index.mjs#L115-L118
     // License: Apache 2.0
@@ -215,6 +317,7 @@ export default class PageLoader {
       if (cn.saveData || /2g/.test(cn.effectiveType)) return Promise.resolve()
     }
 
+    /** @type {string} */
     let url
     if (isDependency) {
       url = route

packages/next/next-server/lib/router/router.ts

@@ -703,9 +703,12 @@ export default class Router implements BaseRouter {
         return
       }
 
+      Promise.all([
+        this.pageLoader.prefetchData(url, asPath),
         this.pageLoader[options.priority ? 'loadPage' : 'prefetch'](
           toRoute(pathname)
-      ).then(() => resolve(), reject)
+        ),
+      ]).then(() => resolve(), reject)
     })
   }
 

test/integration/preload-viewport/next.config.js

@@ -0,0 +1,5 @@
+module.exports = {
+  generateBuildId() {
+    return 'test-build'
+  },
+}

test/integration/preload-viewport/pages/ssg/basic.js

@@ -0,0 +1,5 @@
+export function getStaticProps() {
+  return { props: { message: 'hello world' } }
+}
+
+export default ({ message }) => <p id="content">{message}</p>

test/integration/preload-viewport/pages/ssg/catch-all/[...slug].js

@@ -0,0 +1,12 @@
+export function getStaticProps({ params }) {
+  return { props: { message: `hello ${params.slug.join(' ')}` } }
+}
+
+export function getStaticPaths() {
+  return {
+    paths: ['/ssg/catch-all/one', '/ssg/catch-all/one/two'],
+    fallback: true,
+  }
+}
+
+export default ({ message }) => <p id="content">{message || 'loading'}</p>

test/integration/preload-viewport/pages/ssg/dynamic-nested/[slug1]/[slug2].js

@@ -0,0 +1,12 @@
+export function getStaticProps({ params }) {
+  return { props: { message: `hello ${params.slug1} ${params.slug2}` } }
+}
+
+export function getStaticPaths() {
+  return {
+    paths: ['/ssg/dynamic-nested/one/two'],
+    fallback: true,
+  }
+}
+
+export default ({ message }) => <p id="content">{message || 'loading'}</p>

test/integration/preload-viewport/pages/ssg/dynamic/[slug].js

@@ -0,0 +1,9 @@
+export function getStaticProps({ params }) {
+  return { props: { message: `hello ${params.slug}` } }
+}
+
+export function getStaticPaths() {
+  return { paths: ['/ssg/dynamic/one'], fallback: true }
+}
+
+export default ({ message }) => <p id="content">{message || 'loading'}</p>

test/integration/preload-viewport/pages/ssg/fixture/index.js

@@ -0,0 +1,65 @@
+import Link from 'next/link'
+
+export default () => (
+  <main>
+    <h1>SSG Data Prefetch Fixtures</h1>
+    <p>
+      <Link href="/ssg/basic">
+        <a>Non-dynamic route</a>
+      </Link>
+      : this is a normal Next.js page that does not use dynamic routing.
+    </p>
+    <p>
+      <Link href="/ssg/dynamic/[slug]" as="/ssg/dynamic/one">
+        <a>Dynamic Route (one level) — Prerendered</a>
+      </Link>
+      : this is a Dynamic Page with a single dynamic segment that{' '}
+      <strong>was returned</strong> from <code>getStaticPaths</code>.<br />
+      <Link href="/ssg/dynamic/[slug]" as="/ssg/dynamic/two">
+        <a>Dynamic Route (one level) — Not Prerendered</a>
+      </Link>
+      : this is a Dynamic Page with a single dynamic segment that{' '}
+      <strong>was not returned</strong> from <code>getStaticPaths</code>.
+    </p>
+    <p>
+      <Link
+        href="/ssg/dynamic-nested/[slug1]/[slug2]"
+        as="/ssg/dynamic-nested/one/two"
+      >
+        <a>Multi Dynamic Route (two levels) — Prerendered</a>
+      </Link>
+      : this is a Dynamic Page with two dynamic segments that{' '}
+      <strong>were returned</strong> from <code>getStaticPaths</code>.<br />
+      <Link
+        href="/ssg/dynamic-nested/[slug1]/[slug2]"
+        as="/ssg/dynamic-nested/foo/bar"
+      >
+        <a>Multi Dynamic Route (two levels) — Not Prerendered</a>
+      </Link>
+      : this is a Dynamic Page with two dynamic segments that{' '}
+      <strong>were not returned</strong> from <code>getStaticPaths</code>.
+    </p>
+    <p>
+      <Link href="/ssg/catch-all/[...slug]" as="/ssg/catch-all/one">
+        <a>Catch-All Route (one level) — Prerendered</a>
+      </Link>
+      : this is a Catch-All Page with one segment that{' '}
+      <strong>was returned</strong> from <code>getStaticPaths</code>.<br />
+      <Link href="/ssg/catch-all/[...slug]" as="/ssg/catch-all/foo">
+        <a>Catch-All Route (one level) — Not Prerendered</a>
+      </Link>
+      : this is a Catch-All Page with one segment that{' '}
+      <strong>was not returned</strong> from <code>getStaticPaths</code>.<br />
+      <Link href="/ssg/catch-all/[...slug]" as="/ssg/catch-all/one/two">
+        <a>Catch-All Route (two levels) — Prerendered</a>
+      </Link>
+      : this is a Catch-All Page with two segments that{' '}
+      <strong>were returned</strong> from <code>getStaticPaths</code>.<br />
+      <Link href="/ssg/catch-all/[...slug]" as="/ssg/catch-all/foo/bar">
+        <a>Catch-All Route (two levels) — Not Prerendered</a>
+      </Link>
+      : this is a Catch-All Page with two segments that{' '}
+      <strong>were not returned</strong> from <code>getStaticPaths</code>.
+    </p>
+  </main>
+)

test/integration/preload-viewport/pages/ssg/fixture/mismatch.js

@@ -0,0 +1,89 @@
+import Link from 'next/link'
+
+export default () => (
+  <main>
+    <h1>Mismatched SSG Data Prefetch Fixtures</h1>
+    <p>
+      <Link href="/ssg/dynamic/[slug]?slug=one" as="/ssg/fixture/mismatch">
+        <a>Dynamic Route (one level) — Prerendered</a>
+      </Link>
+      : this is a Dynamic Page with a single dynamic segment that{' '}
+      <strong>was returned</strong> from <code>getStaticPaths</code>.<br />
+      <Link href="/ssg/dynamic/[slug]?slug=two" as="/ssg/fixture/mismatch">
+        <a>Dynamic Route (one level) — Not Prerendered</a>
+      </Link>
+      : this is a Dynamic Page with a single dynamic segment that{' '}
+      <strong>was not returned</strong> from <code>getStaticPaths</code>.
+    </p>
+    <p>
+      <Link
+        href={{
+          pathname: '/ssg/dynamic-nested/[slug1]/[slug2]',
+          query: { slug1: 'one', slug2: 'two' },
+        }}
+        as="/ssg/fixture/mismatch"
+      >
+        <a>Multi Dynamic Route (two levels) — Prerendered</a>
+      </Link>
+      : this is a Dynamic Page with two dynamic segments that{' '}
+      <strong>were returned</strong> from <code>getStaticPaths</code>.<br />
+      <Link
+        href={{
+          pathname: '/ssg/dynamic-nested/[slug1]/[slug2]',
+          query: { slug1: 'foo', slug2: 'bar' },
+        }}
+        as="/ssg/fixture/mismatch"
+      >
+        <a>Multi Dynamic Route (two levels) — Not Prerendered</a>
+      </Link>
+      : this is a Dynamic Page with two dynamic segments that{' '}
+      <strong>were not returned</strong> from <code>getStaticPaths</code>.
+    </p>
+    <p>
+      <Link
+        href={{
+          pathname: '/ssg/catch-all/[...slug]',
+          query: { slug: ['one'] },
+        }}
+        as="/ssg/fixture/mismatch"
+      >
+        <a>Catch-All Route (one level) — Prerendered</a>
+      </Link>
+      : this is a Catch-All Page with one segment that{' '}
+      <strong>was returned</strong> from <code>getStaticPaths</code>.<br />
+      <Link
+        href={{
+          pathname: '/ssg/catch-all/[...slug]',
+          query: { slug: ['foo'] },
+        }}
+        as="/ssg/fixture/mismatch"
+      >
+        <a>Catch-All Route (one level) — Not Prerendered</a>
+      </Link>
+      : this is a Catch-All Page with one segment that{' '}
+      <strong>was not returned</strong> from <code>getStaticPaths</code>.<br />
+      <Link
+        href={{
+          pathname: '/ssg/catch-all/[...slug]',
+          query: { slug: ['one', 'two'] },
+        }}
+        as="/ssg/fixture/mismatch"
+      >
+        <a>Catch-All Route (two levels) — Prerendered</a>
+      </Link>
+      : this is a Catch-All Page with two segments that{' '}
+      <strong>were returned</strong> from <code>getStaticPaths</code>.<br />
+      <Link
+        href={{
+          pathname: '/ssg/catch-all/[...slug]',
+          query: { slug: ['foo', 'bar'] },
+        }}
+        as="/ssg/fixture/mismatch"
+      >
+        <a>Catch-All Route (two levels) — Not Prerendered</a>
+      </Link>
+      : this is a Catch-All Page with two segments that{' '}
+      <strong>were not returned</strong> from <code>getStaticPaths</code>.
+    </p>
+  </main>
+)

test/integration/preload-viewport/test/index.test.js

@@ -9,6 +9,8 @@ import {
 } from 'next-test-utils'
 import webdriver from 'next-webdriver'
 import { join } from 'path'
+import { readFile } from 'fs-extra'
+import { parse } from 'url'
 
 jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 5
 
@@ -244,4 +246,78 @@ describe('Prefetching Links in viewport', () => {
     const calledPrefetch = await browser.eval(`window.calledPrefetch`)
     expect(calledPrefetch).toBe(true)
   })
+
+  it('should correctly omit pre-generaged dynamic pages from SSG manifest', async () => {
+    const content = await readFile(
+      join(appDir, '.next', 'static', 'test-build', '_ssgManifest.js'),
+      'utf8'
+    )
+
+    let self = {}
+    // eslint-disable-next-line no-eval
+    eval(content)
+    expect([...self.__SSG_MANIFEST].sort()).toMatchInlineSnapshot(`
+      Array [
+        "/ssg/basic",
+        "/ssg/catch-all/[...slug]",
+        "/ssg/dynamic-nested/[slug1]/[slug2]",
+        "/ssg/dynamic/[slug]",
+      ]
+    `)
+  })
+
+  it('should prefetch data files', async () => {
+    const browser = await webdriver(appPort, '/ssg/fixture')
+    await waitFor(2 * 1000) // wait for prefetching to occur
+
+    const links = await browser.elementsByCss('link[rel=prefetch][as=fetch]')
+
+    const hrefs = []
+    for (const link of links) {
+      const href = await link.getAttribute('href')
+      hrefs.push(href)
+    }
+    hrefs.sort()
+
+    expect(hrefs.map(href => parse(href).pathname)).toMatchInlineSnapshot(`
+      Array [
+        "/_next/data/test-build/ssg/basic.json",
+        "/_next/data/test-build/ssg/catch-all/foo.json",
+        "/_next/data/test-build/ssg/catch-all/foo/bar.json",
+        "/_next/data/test-build/ssg/catch-all/one.json",
+        "/_next/data/test-build/ssg/catch-all/one/two.json",
+        "/_next/data/test-build/ssg/dynamic-nested/foo/bar.json",
+        "/_next/data/test-build/ssg/dynamic-nested/one/two.json",
+        "/_next/data/test-build/ssg/dynamic/one.json",
+        "/_next/data/test-build/ssg/dynamic/two.json",
+      ]
+    `)
+  })
+
+  it('should prefetch data files when mismatched', async () => {
+    const browser = await webdriver(appPort, '/ssg/fixture/mismatch')
+    await waitFor(2 * 1000) // wait for prefetching to occur
+
+    const links = await browser.elementsByCss('link[rel=prefetch][as=fetch]')
+
+    const hrefs = []
+    for (const link of links) {
+      const href = await link.getAttribute('href')
+      hrefs.push(href)
+    }
+    hrefs.sort()
+
+    expect(hrefs.map(href => parse(href).pathname)).toMatchInlineSnapshot(`
+      Array [
+        "/_next/data/test-build/ssg/catch-all/foo.json",
+        "/_next/data/test-build/ssg/catch-all/foo/bar.json",
+        "/_next/data/test-build/ssg/catch-all/one.json",
+        "/_next/data/test-build/ssg/catch-all/one/two.json",
+        "/_next/data/test-build/ssg/dynamic-nested/foo/bar.json",
+        "/_next/data/test-build/ssg/dynamic-nested/one/two.json",
+        "/_next/data/test-build/ssg/dynamic/one.json",
+        "/_next/data/test-build/ssg/dynamic/two.json",
+      ]
+    `)
+  })
 })

test/integration/size-limit/test/index.test.js

@@ -80,7 +80,7 @@ describe('Production response size', () => {
     )
 
     // These numbers are without gzip compression!
-    const delta = responseSizesBytes - 231 * 1024
+    const delta = responseSizesBytes - 232 * 1024
     expect(delta).toBeLessThanOrEqual(1024) // don't increase size more than 1kb
     expect(delta).toBeGreaterThanOrEqual(-1024) // don't decrease size more than 1kb without updating target
   })
@@ -100,7 +100,7 @@ describe('Production response size', () => {
     )
 
     // These numbers are without gzip compression!
-    const delta = responseSizesBytes - 164 * 1024
+    const delta = responseSizesBytes - 165 * 1024
     expect(delta).toBeLessThanOrEqual(1024) // don't increase size more than 1kb
     expect(delta).toBeGreaterThanOrEqual(-1024) // don't decrease size more than 1kb without updating target
   })