add support for new URL() (#28940)

Currently `new URL()` for server assets is completely broken because of the `publicPath` that is used for them too. `new URL()` for SSR is broken on windows as it's using absolute urls on the windows filesystem. And `new URL()` is using an incorrect filename

* Place all `asset`s correctly in `/_next/static/media` with `[name].[hash:8][ext]`
* Added a separate runtime chunk for api entries, without `publicPath`
* Introduce separate layer for api entries, which uses server-side URLs.
* Otherwise new URL() will return a faked relative URL, that is identical in SSR and CSR
* Disables react-refresh for api entries

Fixes #27413



## Bug

- [ ] Related issues linked using `fixes #number`
- [x] Integration tests added
- [ ] Errors have helpful link attached, see `contributing.md`

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR.
- [ ] Related issues linked using `fixes #number`
- [x] Integration tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have helpful link attached, see `contributing.md`

## Documentation / Examples

- [ ] Make sure the linting passes

packages/next/build/entries.ts

@@ -69,6 +69,8 @@ export type WebpackEntrypoints = {
     | {
         import: string | string[]
         dependOn?: string | string[]
+        publicPath?: string
+        runtime?: string
       }
 }
 
@@ -179,3 +181,58 @@ export function createEntrypoints(
     server,
   }
 }
+
+export function finalizeEntrypoint(
+  name: string,
+  value: any,
+  isServer: boolean,
+  isWebpack5: boolean
+): any {
+  if (isWebpack5) {
+    if (isServer) {
+      const isApi = name.startsWith('pages/api/')
+      const runtime = isApi ? 'webpack-api-runtime' : 'webpack-runtime'
+      const layer = isApi ? 'api' : undefined
+      const publicPath = isApi ? '' : undefined
+      if (typeof value === 'object' && !Array.isArray(value)) {
+        return {
+          publicPath,
+          runtime,
+          layer,
+          ...value,
+        }
+      } else {
+        return {
+          import: value,
+          publicPath,
+          runtime,
+          layer,
+        }
+      }
+    } else {
+      if (
+        name !== 'polyfills' &&
+        name !== 'main' &&
+        name !== 'amp' &&
+        name !== 'react-refresh'
+      ) {
+        const dependOn =
+          name.startsWith('pages/') && name !== 'pages/_app'
+            ? 'pages/_app'
+            : 'main'
+        if (typeof value === 'object' && !Array.isArray(value)) {
+          return {
+            dependOn,
+            ...value,
+          }
+        } else {
+          return {
+            import: value,
+            dependOn,
+          }
+        }
+      }
+    }
+  }
+  return value
+}

packages/next/build/webpack-config.ts

@@ -28,7 +28,7 @@ import {
 } from '../shared/lib/constants'
 import { execOnce } from '../shared/lib/utils'
 import { NextConfigComplete } from '../server/config-shared'
-import { WebpackEntrypoints } from './entries'
+import { finalizeEntrypoint, WebpackEntrypoints } from './entries'
 import * as Log from './output/log'
 import { build as buildConfiguration } from './webpack/config'
 import { __overrideCssConfiguration } from './webpack/config/blocks/css/overrideCssConfiguration'
@@ -849,6 +849,20 @@ export default async function getBaseWebpackConfig(
 
   const emacsLockfilePattern = '**/.#*'
 
+  const codeCondition = {
+    test: /\.(tsx|ts|js|cjs|mjs|jsx)$/,
+    ...(config.experimental.externalDir
+      ? // Allowing importing TS/TSX files from outside of the root dir.
+        {}
+      : { include: [dir, ...babelIncludeRegexes] }),
+    exclude: (excludePath: string) => {
+      if (babelIncludeRegexes.some((r) => r.test(excludePath))) {
+        return false
+      }
+      return /node_modules/.test(excludePath)
+    },
+  }
+
   let webpackConfig: webpack.Configuration = {
     parallelism: Number(process.env.NEXT_WEBPACK_PARALLELISM) || undefined,
     externals: !isServer
@@ -946,9 +960,7 @@ export default async function getBaseWebpackConfig(
           : false
         : splitChunksConfig,
       runtimeChunk: isServer
-        ? isWebpack5 && !isLikeServerless
-          ? { name: 'webpack-runtime' }
-          : undefined
+        ? undefined
         : { name: CLIENT_STATIC_FILES_RUNTIME_WEBPACK },
       minimize: !(dev || isServer),
       minimizer: [
@@ -1080,26 +1092,49 @@ export default async function getBaseWebpackConfig(
                   fullySpecified: false,
                 },
               } as any,
+              {
+                test: /\.(js|cjs|mjs)$/,
+                issuerLayer: 'api',
+                parser: {
+                  // Switch back to normal URL handling
+                  url: true,
+                },
+              },
             ]
           : []),
         {
-          test: /\.(tsx|ts|js|mjs|jsx)$/,
-          ...(config.experimental.externalDir
-            ? // Allowing importing TS/TSX files from outside of the root dir.
-              {}
-            : { include: [dir, ...babelIncludeRegexes] }),
-          exclude: (excludePath: string) => {
-            if (babelIncludeRegexes.some((r) => r.test(excludePath))) {
-              return false
-            }
-            return /node_modules/.test(excludePath)
-          },
-          use: hasReactRefresh
-            ? [
-                require.resolve('@next/react-refresh-utils/loader'),
-                defaultLoaders.babel,
-              ]
-            : defaultLoaders.babel,
+          ...(isWebpack5
+            ? {
+                oneOf: [
+                  {
+                    ...codeCondition,
+                    issuerLayer: 'api',
+                    parser: {
+                      // Switch back to normal URL handling
+                      url: true,
+                    },
+                    use: defaultLoaders.babel,
+                  },
+                  {
+                    ...codeCondition,
+                    use: hasReactRefresh
+                      ? [
+                          require.resolve('@next/react-refresh-utils/loader'),
+                          defaultLoaders.babel,
+                        ]
+                      : defaultLoaders.babel,
+                  },
+                ],
+              }
+            : {
+                ...codeCondition,
+                use: hasReactRefresh
+                  ? [
+                      require.resolve('@next/react-refresh-utils/loader'),
+                      defaultLoaders.babel,
+                    ]
+                  : defaultLoaders.babel,
+              }),
         },
         ...(!config.images.disableStaticImages && isWebpack5
           ? [
@@ -1344,6 +1379,21 @@ export default async function getBaseWebpackConfig(
     // futureEmitAssets is on by default in webpack 5
     delete webpackConfig.output?.futureEmitAssets
 
+    webpackConfig.experiments = {
+      layers: true,
+    }
+
+    webpackConfig.module!.parser = {
+      javascript: {
+        url: 'relative',
+      },
+    }
+    webpackConfig.module!.generator = {
+      asset: {
+        filename: 'static/media/[name].[hash:8][ext]',
+      },
+    }
+
     if (isServer && dev) {
       // Enable building of client compilation before server compilation in development
       // @ts-ignore dependencies exists
@@ -1581,10 +1631,6 @@ export default async function getBaseWebpackConfig(
           exclude: fileLoaderExclude,
           issuer: fileLoaderExclude,
           type: 'asset/resource',
-          generator: {
-            publicPath: '/_next/',
-            filename: 'static/media/[name].[hash:8].[ext]',
-          },
         }
       : {
           loader: require.resolve('next/dist/compiled/file-loader'),
@@ -1871,32 +1917,13 @@ export default async function getBaseWebpackConfig(
       }
       delete entry['main.js']
 
-      if (isWebpack5 && !isServer) {
-        for (const name of Object.keys(entry)) {
-          if (
-            name === 'polyfills' ||
-            name === 'main' ||
-            name === 'amp' ||
-            name === 'react-refresh'
-          )
-            continue
-          const dependOn =
-            name.startsWith('pages/') && name !== 'pages/_app'
-              ? 'pages/_app'
-              : 'main'
-          const old = entry[name]
-          if (typeof old === 'object' && !Array.isArray(old)) {
-            entry[name] = {
-              dependOn,
-              ...old,
-            }
-          } else {
-            entry[name] = {
-              import: old,
-              dependOn,
-            }
-          }
-        }
+      for (const name of Object.keys(entry)) {
+        entry[name] = finalizeEntrypoint(
+          name,
+          entry[name],
+          isServer,
+          isWebpack5
+        )
       }
 
       return entry

packages/next/build/webpack/plugins/nextjs-require-cache-hot-reloader.ts

@@ -9,6 +9,8 @@ const originModules = [
   require.resolve('../../../server/load-components'),
 ]
 
+const RUNTIME_NAMES = ['webpack-runtime', 'webpack-api-runtime']
+
 function deleteCache(filePath: string) {
   try {
     filePath = realpathSync(filePath)
@@ -53,11 +55,13 @@ export class NextJsRequireCacheHotReloader implements webpack.Plugin {
       )
 
       compiler.hooks.afterEmit.tap(PLUGIN_NAME, (compilation) => {
-        const runtimeChunkPath = path.join(
-          compilation.outputOptions.path,
-          'webpack-runtime.js'
-        )
-        deleteCache(runtimeChunkPath)
+        RUNTIME_NAMES.forEach((name) => {
+          const runtimeChunkPath = path.join(
+            compilation.outputOptions.path,
+            `${name}.js`
+          )
+          deleteCache(runtimeChunkPath)
+        })
 
         // we need to make sure to clear all server entries from cache
         // since they can have a stale webpack-runtime cache

packages/next/build/webpack/plugins/pages-manifest-plugin.ts

@@ -35,7 +35,9 @@ export default class PagesManifestPlugin implements webpack.Plugin {
         .getFiles()
         .filter(
           (file: string) =>
-            !file.includes('webpack-runtime') && file.endsWith('.js')
+            !file.includes('webpack-runtime') &&
+            !file.includes('webpack-api-runtime') &&
+            file.endsWith('.js')
         )
 
       if (!isWebpack5 && files.length > 1) {

packages/next/server/dev/hot-reloader.ts

@@ -5,7 +5,11 @@ import { WebpackHotMiddleware } from './hot-middleware'
 import { join } from 'path'
 import { UrlObject } from 'url'
 import { webpack, isWebpack5 } from 'next/dist/compiled/webpack/webpack'
-import { createEntrypoints, createPagesMapping } from '../../build/entries'
+import {
+  createEntrypoints,
+  createPagesMapping,
+  finalizeEntrypoint,
+} from '../../build/entries'
 import { watchCompilers } from '../../build/output'
 import getBaseWebpackConfig from '../../build/webpack-config'
 import { API_ROUTE } from '../../lib/constants'
@@ -387,11 +391,17 @@ export default class HotReloader {
               absolutePagePath,
             }
 
-            entrypoints[
-              isClientCompilation ? clientBundlePath : serverBundlePath
-            ] = isClientCompilation
-              ? `next-client-pages-loader?${stringify(pageLoaderOpts)}!`
-              : absolutePagePath
+            const name = isClientCompilation
+              ? clientBundlePath
+              : serverBundlePath
+            entrypoints[name] = finalizeEntrypoint(
+              name,
+              isClientCompilation
+                ? `next-client-pages-loader?${stringify(pageLoaderOpts)}!`
+                : absolutePagePath,
+              !isClientCompilation,
+              isWebpack5
+            )
           })
         )
 

packages/next/types/webpack.d.ts

@@ -145,6 +145,9 @@ declare module 'webpack' {
       parallelism?: number
       /** Optimization options */
       optimization?: Options.Optimization
+      experiments?: {
+        layers: boolean
+      }
     }
 
     interface Entry {
@@ -293,6 +296,12 @@ declare module 'webpack' {
       strictExportPresence?: boolean
       /** An array of rules applied for modules. */
       rules: RuleSetRule[]
+      parser?: {
+        javascript?: any
+      }
+      generator?: {
+        asset?: any
+      }
     }
 
     interface Resolve {

test/integration/server-asset-modules/my-data.json

@@ -0,0 +1,3 @@
+{
+  "message": "hello world"
+}

test/integration/server-asset-modules/next.config.js

@@ -0,0 +1 @@
+module.exports = {}

test/integration/server-asset-modules/pages/api/test.js

@@ -0,0 +1,6 @@
+import * as fs from 'fs/promises'
+export default async (req, res) => {
+  const fileUrl = new URL('../../my-data.json', import.meta.url)
+  const content = await fs.readFile(fileUrl, { encoding: 'utf-8' })
+  res.json(JSON.parse(content))
+}

test/integration/server-asset-modules/test/index.test.js

@@ -0,0 +1,80 @@
+/* eslint-env jest */
+
+import fs from 'fs-extra'
+import {
+  fetchViaHTTP,
+  findPort,
+  killApp,
+  launchApp,
+  nextBuild,
+  nextStart,
+} from 'next-test-utils'
+import { join } from 'path'
+
+jest.setTimeout(1000 * 60 * 2)
+
+let app
+let appPort
+const appDir = join(__dirname, '../')
+
+function runTests() {
+  it('should enable reading local files in api routes', async () => {
+    const res = await fetchViaHTTP(appPort, '/api/test', null, {})
+    expect(res.status).toEqual(200)
+    const content = await res.json()
+    expect(content).toHaveProperty('message', 'hello world')
+  })
+}
+
+const nextConfig = join(appDir, 'next.config.js')
+
+describe('serverside asset modules', () => {
+  describe('dev mode', () => {
+    beforeAll(async () => {
+      appPort = await findPort()
+      app = await launchApp(appDir, appPort)
+    })
+    afterAll(() => killApp(app))
+
+    runTests()
+  })
+
+  describe('production mode', () => {
+    beforeAll(async () => {
+      const curConfig = await fs.readFile(nextConfig, 'utf8')
+
+      if (curConfig.includes('target')) {
+        await fs.writeFile(nextConfig, `module.exports = {}`)
+      }
+      await nextBuild(appDir)
+
+      appPort = await findPort()
+      app = await nextStart(appDir, appPort)
+    })
+    afterAll(() => killApp(app))
+
+    runTests()
+  })
+
+  describe('serverless mode', () => {
+    let origNextConfig
+
+    beforeAll(async () => {
+      origNextConfig = await fs.readFile(nextConfig, 'utf8')
+      await fs.writeFile(
+        nextConfig,
+        `module.exports = { target: 'serverless' }`
+      )
+
+      await nextBuild(appDir)
+
+      appPort = await findPort()
+      app = await nextStart(appDir, appPort)
+    })
+    afterAll(async () => {
+      await fs.writeFile(nextConfig, origNextConfig)
+      await killApp(app)
+    })
+    runTests()
+  })
+})

test/integration/url/pages/api/basename.js

@@ -0,0 +1,7 @@
+import path from 'path'
+
+const img = new URL('../../public/vercel.png', import.meta.url)
+
+export default (req, res) => {
+  res.json({ basename: path.posix.basename(img.pathname) })
+}

test/integration/url/pages/api/size.js

@@ -0,0 +1,7 @@
+import fs from 'fs'
+
+const img = new URL('../../public/vercel.png', import.meta.url)
+
+export default (req, res) => {
+  res.json({ size: fs.readFileSync(img).length })
+}

test/integration/url/pages/ssg.js

@@ -0,0 +1,15 @@
+export async function getStaticProps() {
+  return {
+    props: {
+      url: new URL('../public/vercel.png', import.meta.url).pathname,
+    },
+  }
+}
+
+export default function Index({ url }) {
+  return (
+    <div>
+      Hello {new URL('../public/vercel.png', import.meta.url).pathname}+{url}
+    </div>
+  )
+}

test/integration/url/pages/ssr.js

@@ -0,0 +1,15 @@
+export function getServerSideProps() {
+  return {
+    props: {
+      url: new URL('../public/vercel.png', import.meta.url).pathname,
+    },
+  }
+}
+
+export default function Index({ url }) {
+  return (
+    <div>
+      Hello {new URL('../public/vercel.png', import.meta.url).pathname}+{url}
+    </div>
+  )
+}

test/integration/url/pages/static.js

@@ -0,0 +1,9 @@
+const url = new URL('../public/vercel.png', import.meta.url).pathname
+
+export default function Index(props) {
+  return (
+    <div>
+      Hello {new URL('../public/vercel.png', import.meta.url).pathname}+{url}
+    </div>
+  )
+}

test/integration/url/test/index.test.js

@@ -0,0 +1,82 @@
+/* eslint-disable no-loop-func */
+/* eslint-env jest */
+
+import fs from 'fs-extra'
+import { join } from 'path'
+import {
+  nextBuild,
+  findPort,
+  nextStart,
+  killApp,
+  renderViaHTTP,
+  fetchViaHTTP,
+  launchApp,
+  getBrowserBodyText,
+  check,
+} from 'next-test-utils'
+import webdriver from 'next-webdriver'
+
+jest.setTimeout(1000 * 60 * 2)
+const appDir = join(__dirname, '../')
+
+for (const dev of [false, true]) {
+  describe(`Handle new URL asset references in next ${
+    dev ? 'dev' : 'build'
+  }`, () => {
+    let appPort
+    let app
+    beforeAll(async () => {
+      await fs.remove(join(appDir, '.next'))
+      if (dev) {
+        appPort = await findPort()
+        app = await launchApp(appDir, appPort)
+      } else {
+        await nextBuild(appDir)
+        appPort = await findPort()
+        app = await nextStart(appDir, appPort)
+      }
+    })
+    afterAll(() => killApp(app))
+
+    const expectedServer =
+      /Hello <!-- -->\/_next\/static\/media\/vercel\.[0-9a-f]{8}\.png<!-- -->\+<!-- -->\/_next\/static\/media\/vercel\.[0-9a-f]{8}\.png/
+    const expectedClient = new RegExp(
+      expectedServer.source.replace(/<!-- -->/g, '')
+    )
+
+    for (const page of ['/static', '/ssr', '/ssg']) {
+      it(`should render the ${page} page`, async () => {
+        const html = await renderViaHTTP(appPort, page)
+        expect(html).toMatch(expectedServer)
+      })
+
+      it(`should client-render the ${page} page`, async () => {
+        let browser
+        try {
+          browser = await webdriver(appPort, page)
+          await check(() => getBrowserBodyText(browser), expectedClient)
+        } finally {
+          await browser.close()
+        }
+      })
+    }
+
+    it('should respond on size api', async () => {
+      const data = await fetchViaHTTP(appPort, '/api/size').then(
+        (res) => res.ok && res.json()
+      )
+
+      expect(data).toEqual({ size: 30079 })
+    })
+
+    it('should respond on basename api', async () => {
+      const data = await fetchViaHTTP(appPort, '/api/basename').then(
+        (res) => res.ok && res.json()
+      )
+
+      expect(data).toEqual({
+        basename: expect.stringMatching(/^vercel\.[0-9a-f]{8}\.png$/),
+      })
+    })
+  })
+}