Allow using ESM pkg with custom incremental cache (#59863)

Use dynamic import instead of require to load the incremental cache
handled, so when using ESM it will still work.

Updated the tests and merged them into new test suite, include 3 cases
of custom cache definition:
- CJS with `module.exports`
- CJS with `exports.default` with ESM mark
- ESM with `export default`

Closes NEXT-1924
Fixes #58509

packages/next/src/build/index.ts

@@ -163,6 +163,7 @@ import { formatManifest } from './manifests/formatter/format-manifest'
 import { getStartServerInfo, logStartInfo } from '../server/lib/app-info-log'
 import type { NextEnabledDirectories } from '../server/base-server'
 import { hasCustomExportOutput } from '../export/utils'
+import { interopDefault } from '../lib/interop-default'
 
 interface ExperimentalBypassForInfo {
   experimentalBypassFor?: RouteHas[]
@@ -1323,10 +1324,13 @@ export default async function build(
       if (config.experimental.staticWorkerRequestDeduping) {
         let CacheHandler
         if (incrementalCacheHandlerPath) {
-          CacheHandler = require(path.isAbsolute(incrementalCacheHandlerPath)
-            ? incrementalCacheHandlerPath
-            : path.join(dir, incrementalCacheHandlerPath))
-          CacheHandler = CacheHandler.default || CacheHandler
+          CacheHandler = interopDefault(
+            await import(
+              path.isAbsolute(incrementalCacheHandlerPath)
+                ? incrementalCacheHandlerPath
+                : path.join(dir, incrementalCacheHandlerPath)
+            )
+          )
         }
 
         const cacheInitialization = await initializeIncrementalCache({

packages/next/src/build/utils.ts

@@ -72,6 +72,7 @@ import { normalizeAppPath } from '../shared/lib/router/utils/app-paths'
 import { denormalizeAppPagePath } from '../shared/lib/page-path/denormalize-app-path'
 import { RouteKind } from '../server/future/route-kind'
 import { isAppRouteRouteModule } from '../server/future/route-modules/checks'
+import { interopDefault } from '../lib/interop-default'
 import type { PageExtensions } from './page-extensions-type'
 
 export type ROUTER_TYPE = 'pages' | 'app'
@@ -1308,10 +1309,13 @@ export async function buildAppStaticPaths({
   let CacheHandler: any
 
   if (incrementalCacheHandlerPath) {
-    CacheHandler = require(path.isAbsolute(incrementalCacheHandlerPath)
-      ? incrementalCacheHandlerPath
-      : path.join(dir, incrementalCacheHandlerPath))
-    CacheHandler = CacheHandler.default || CacheHandler
+    CacheHandler = interopDefault(
+      await import(
+        path.isAbsolute(incrementalCacheHandlerPath)
+          ? incrementalCacheHandlerPath
+          : path.join(dir, incrementalCacheHandlerPath)
+      )
+    )
   }
 
   const incrementalCache = new IncrementalCache({

packages/next/src/export/helpers/create-incremental-cache.ts

@@ -4,8 +4,9 @@ import path from 'path'
 import { IncrementalCache } from '../../server/lib/incremental-cache'
 import { hasNextSupport } from '../../telemetry/ci-info'
 import { nodeFs } from '../../server/lib/node-fs-methods'
+import { interopDefault } from '../../lib/interop-default'
 
-export function createIncrementalCache({
+export async function createIncrementalCache({
   incrementalCacheHandlerPath,
   isrMemoryCacheSize,
   fetchCacheKeyPrefix,
@@ -27,10 +28,15 @@ export function createIncrementalCache({
   // Custom cache handler overrides.
   let CacheHandler: any
   if (incrementalCacheHandlerPath) {
-    CacheHandler = require(path.isAbsolute(incrementalCacheHandlerPath)
-      ? incrementalCacheHandlerPath
-      : path.join(dir, incrementalCacheHandlerPath))
-    CacheHandler = CacheHandler.default || CacheHandler
+    CacheHandler = interopDefault(
+      (
+        await import(
+          path.isAbsolute(incrementalCacheHandlerPath)
+            ? incrementalCacheHandlerPath
+            : path.join(dir, incrementalCacheHandlerPath)
+        )
+      ).default
+    )
   }
 
   const incrementalCache = new IncrementalCache({

packages/next/src/export/worker.ts

@@ -220,7 +220,7 @@ async function exportPageImpl(
     // cache instance for this page.
     const incrementalCache =
       isAppDir && fetchCache
-        ? createIncrementalCache({
+        ? await createIncrementalCache({
             incrementalCacheHandlerPath,
             isrMemoryCacheSize,
             fetchCacheKeyPrefix,

packages/next/src/server/base-server.ts

@@ -387,7 +387,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
   protected abstract getIncrementalCache(options: {
     requestHeaders: Record<string, undefined | string | string[]>
     requestProtocol: 'http' | 'https'
-  }): import('./lib/incremental-cache').IncrementalCache
+  }): Promise<import('./lib/incremental-cache').IncrementalCache>
 
   protected abstract getResponseCache(options: {
     dev: boolean
@@ -1263,7 +1263,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
           protocol = parsedFullUrl.protocol as 'https:' | 'http:'
         } catch {}
 
-        const incrementalCache = this.getIncrementalCache({
+        const incrementalCache = await this.getIncrementalCache({
           requestHeaders: Object.assign({}, req.headers),
           requestProtocol: protocol.substring(0, protocol.length - 1) as
             | 'http'
@@ -2127,12 +2127,12 @@ export default abstract class Server<ServerOptions extends Options = Options> {
     // use existing incrementalCache instance if available
     const incrementalCache =
       (globalThis as any).__incrementalCache ||
-      this.getIncrementalCache({
+      (await this.getIncrementalCache({
         requestHeaders: Object.assign({}, req.headers),
         requestProtocol: protocol.substring(0, protocol.length - 1) as
           | 'http'
           | 'https',
-      })
+      }))
 
     const { routeModule } = components
 

packages/next/src/server/next-server.ts

@@ -100,11 +100,18 @@ import { RouteModuleLoader } from './future/helpers/module-loader/route-module-l
 import { loadManifest } from './load-manifest'
 import { lazyRenderAppPage } from './future/route-modules/app-page/module.render'
 import { lazyRenderPagesPage } from './future/route-modules/pages/module.render'
+import { interopDefault } from '../lib/interop-default'
 
 export * from './base-server'
 
 declare const __non_webpack_require__: NodeRequire
 
+// For module that can be both CJS or ESM
+const dynamicImportEsmDefault = process.env.NEXT_MINIMAL
+  ? __non_webpack_require__
+  : async (mod: string) => (await import(mod)).default
+
+// For module that will be compiled to CJS, e.g. instrument
 const dynamicRequire = process.env.NEXT_MINIMAL
   ? __non_webpack_require__
   : require
@@ -289,7 +296,7 @@ export default class NextNodeServer extends BaseServer {
     )
   }
 
-  protected getIncrementalCache({
+  protected async getIncrementalCache({
     requestHeaders,
     requestProtocol,
   }: {
@@ -301,12 +308,13 @@ export default class NextNodeServer extends BaseServer {
     const { incrementalCacheHandlerPath } = this.nextConfig.experimental
 
     if (incrementalCacheHandlerPath) {
-      CacheHandler = dynamicRequire(
-        isAbsolute(incrementalCacheHandlerPath)
-          ? incrementalCacheHandlerPath
-          : join(this.distDir, incrementalCacheHandlerPath)
+      CacheHandler = interopDefault(
+        await dynamicImportEsmDefault(
+          isAbsolute(incrementalCacheHandlerPath)
+            ? incrementalCacheHandlerPath
+            : join(this.distDir, incrementalCacheHandlerPath)
+        )
       )
-      CacheHandler = CacheHandler.default || CacheHandler
     }
 
     // incremental-cache is request specific

packages/next/src/server/web-server.ts

@@ -55,7 +55,7 @@ export default class NextWebServer extends BaseServer<WebServerOptions> {
     Object.assign(this.renderOpts, options.webServerConfig.extendRenderOpts)
   }
 
-  protected getIncrementalCache({
+  protected async getIncrementalCache({
     requestHeaders,
   }: {
     requestHeaders: IncrementalCache['requestHeaders']

test/e2e/app-dir/app-custom-cache-handler/app/layout.js

@@ -0,0 +1,12 @@
+export const metadata = {
+  title: 'Next.js',
+  description: 'Generated by Next.js',
+}
+
+export default function RootLayout({ children }) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  )
+}

test/e2e/app-dir/app-custom-cache-handler/app/page.js

@@ -0,0 +1,12 @@
+export default async function Page() {
+  const data = await fetch(
+    'https://next-data-api-endpoint.vercel.app/api/random?page'
+  ).then((res) => res.text())
+
+  return (
+    <>
+      <p id="page-data">{data}</p>
+      <p id="now">{Date.now()}</p>
+    </>
+  )
+}

test/e2e/app-dir/app-custom-cache-handler/cache-handler-cjs-default-export.js

@@ -0,0 +1,27 @@
+Object.defineProperty(exports, '__esModule', { value: true })
+
+const cache = new Map()
+
+const CacheHandler = /** @class */ (function () {
+  function CacheHandler(options) {
+    this.options = options
+    this.cache = cache
+    console.log('initialized custom cache-handler')
+    console.log('cache handler - cjs default export')
+  }
+  CacheHandler.prototype.get = function (key) {
+    console.log('cache-handler get', key)
+    return Promise.resolve(this.cache.get(key))
+  }
+  CacheHandler.prototype.set = function (key, data) {
+    console.log('cache-handler set', key)
+    this.cache.set(key, {
+      value: data,
+      lastModified: Date.now(),
+    })
+    return Promise.resolve(undefined)
+  }
+  return CacheHandler
+})()
+
+exports.default = CacheHandler

test/e2e/app-dir/app-custom-cache-handler/cache-handler-esm.js

@@ -0,0 +1,27 @@
+const cache = new Map()
+
+class CacheHandler {
+  constructor(options) {
+    this.options = options
+    this.cache = {}
+    console.log('initialized custom cache-handler')
+    console.log('cache handler - esm default export')
+  }
+
+  async get(key) {
+    console.log('key', key)
+    console.log('cache-handler get', key)
+    return cache.get(key)
+  }
+
+  async set(key, data) {
+    console.log('set key', key)
+    console.log('cache-handler set', key)
+    cache.set(key, {
+      value: data,
+      lastModified: Date.now(),
+    })
+  }
+}
+
+export default CacheHandler

test/e2e/app-dir/app-custom-cache-handler/cache-handler.js

@@ -0,0 +1,27 @@
+const cache = new Map()
+
+class CacheHandler {
+  constructor(options) {
+    this.options = options
+    this.cache = {}
+    console.log('initialized custom cache-handler')
+    console.log('cache handler - cjs module exports')
+  }
+
+  async get(key) {
+    console.log('key', key)
+    console.log('cache-handler get', key)
+    return cache.get(key)
+  }
+
+  async set(key, data) {
+    console.log('set key', key)
+    console.log('cache-handler set', key)
+    cache.set(key, {
+      value: data,
+      lastModified: Date.now(),
+    })
+  }
+}
+
+module.exports = CacheHandler

test/e2e/app-dir/app-custom-cache-handler/index.test.ts

@@ -0,0 +1,80 @@
+import { type NextInstance, createNextDescribe, FileRef } from 'e2e-utils'
+import { check } from 'next-test-utils'
+import fs from 'fs'
+
+const originalNextConfig = fs.readFileSync(
+  __dirname + '/next.config.js',
+  'utf8'
+)
+
+function runTests(
+  exportType: string,
+  { next, isNextDev }: { next: NextInstance; isNextDev: boolean }
+) {
+  describe(exportType, () => {
+    it('should have logs from cache-handler', async () => {
+      if (isNextDev) {
+        await next.fetch('/')
+      }
+      await check(() => {
+        expect(next.cliOutput).toContain('cache handler - ' + exportType)
+        expect(next.cliOutput).toContain('initialized custom cache-handler')
+        expect(next.cliOutput).toContain('cache-handler get')
+        expect(next.cliOutput).toContain('cache-handler set')
+        return 'success'
+      }, 'success')
+    })
+  })
+}
+
+createNextDescribe(
+  'app-dir - custom-cache-handler - cjs',
+  {
+    files: __dirname,
+    skipDeployment: true,
+    env: {
+      CUSTOM_CACHE_HANDLER: 'cache-handler.js',
+    },
+  },
+  ({ next, isNextDev }) => {
+    runTests('cjs module exports', { next, isNextDev })
+  }
+)
+
+createNextDescribe(
+  'app-dir - custom-cache-handler - cjs-default-export',
+  {
+    files: __dirname,
+    skipDeployment: true,
+    env: {
+      CUSTOM_CACHE_HANDLER: 'cache-handler-cjs-default-export.js',
+    },
+  },
+  ({ next, isNextDev }) => {
+    runTests('cjs default export', { next, isNextDev })
+  }
+)
+
+createNextDescribe(
+  'app-dir - custom-cache-handler - esm',
+  {
+    files: {
+      app: new FileRef(__dirname + '/app'),
+      'cache-handler-esm.js': new FileRef(__dirname + '/cache-handler-esm.js'),
+      'next.config.js': originalNextConfig.replace(
+        'module.exports = ',
+        'export default '
+      ),
+    },
+    skipDeployment: true,
+    packageJson: {
+      type: 'module',
+    },
+    env: {
+      CUSTOM_CACHE_HANDLER: 'cache-handler-esm.js',
+    },
+  },
+  ({ next, isNextDev }) => {
+    runTests('esm default export', { next, isNextDev })
+  }
+)

test/e2e/app-dir/app-custom-cache-handler/next.config.js

@@ -0,0 +1,6 @@
+module.exports = {
+  experimental: {
+    incrementalCacheHandlerPath:
+      process.cwd() + '/' + process.env.CUSTOM_CACHE_HANDLER,
+  },
+}

test/e2e/app-dir/app-static/app-static-custom-cache-handler-esm.test.ts

@@ -1,27 +0,0 @@
-import { createNextDescribe } from 'e2e-utils'
-import { join } from 'path'
-
-createNextDescribe(
-  'app-static-custom-cache-handler-esm',
-  {
-    files: __dirname,
-    env: {
-      CUSTOM_CACHE_HANDLER: join(
-        __dirname,
-        './cache-handler-default-export.js'
-      ),
-    },
-  },
-  ({ next, isNextStart }) => {
-    if (!isNextStart) {
-      it('should skip', () => {})
-      return
-    }
-
-    it('should have logs from cache-handler', async () => {
-      expect(next.cliOutput).toContain('initialized custom cache-handler')
-      expect(next.cliOutput).toContain('cache-handler get')
-      expect(next.cliOutput).toContain('cache-handler set')
-    })
-  }
-)

test/lib/next-modes/base.ts

@@ -233,11 +233,13 @@ export class NextInstance {
           ((global as any).isNextDeploy && !nextConfigFile)
         ) {
           const functions = []
-
+          const exportDeclare =
+            this.packageJson?.type === 'module'
+              ? 'export default'
+              : 'module.exports = '
           await fs.writeFile(
             path.join(this.testDir, 'next.config.js'),
-            `
-        module.exports = ` +
+            exportDeclare +
               JSON.stringify(
                 {
                   ...this.nextConfig,