[edge] allow importing blob assets (#38492)

* [edge] allow importing blob assets

* Fix test

* extract to a new file, to make it easier to read and review

* Use webpack asset discovery and transform with a loader

* fix tests

* don't prefix assets

* use emitFile

* rename assets to blobs to be more specific

* rename blobs to assets and use webpack's hashing algo

* Dedupe correctly

* Add a Node.js dep test

* Update packages/next/server/next-server.ts

Co-authored-by: Tobias Koppers <tobias.koppers@googlemail.com>

* [code review] test remote URL fetches

* [code review] use `import type` for type-only imports

* Update packages/next/server/next-server.ts

Co-authored-by: Tobias Koppers <tobias.koppers@googlemail.com>

* Apply suggestions from code review

Co-authored-by: JJ Kasper <jj@jjsweb.site>

Co-authored-by: Tobias Koppers <tobias.koppers@googlemail.com>
Co-authored-by: JJ Kasper <jj@jjsweb.site>

packages/next/build/webpack-config.ts

@@ -1272,6 +1272,7 @@ export default async function getBaseWebpackConfig(
         'next-middleware-loader',
         'next-edge-function-loader',
         'next-edge-ssr-loader',
+        'next-middleware-asset-loader',
         'next-middleware-wasm-loader',
         'next-app-loader',
       ].reduce((alias, loader) => {
@@ -1769,6 +1770,16 @@ export default async function getBaseWebpackConfig(
       type: 'javascript/auto',
       resourceQuery: /module/i,
     })
+    webpack5Config.module?.rules?.unshift({
+      dependency: 'url',
+      loader: 'next-middleware-asset-loader',
+      type: 'javascript/auto',
+      layer: 'edge-asset',
+    })
+    webpack5Config.module?.rules?.unshift({
+      issuerLayer: 'edge-asset',
+      type: 'asset/source',
+    })
   }
 
   webpack5Config.experiments = {

packages/next/build/webpack/loaders/get-module-build-info.ts

@@ -10,7 +10,8 @@ export function getModuleBuildInfo(webpackModule: webpack5.Module) {
     nextEdgeApiFunction?: EdgeMiddlewareMeta
     nextEdgeSSR?: EdgeSSRMeta
     nextUsedEnvVars?: Set<string>
-    nextWasmMiddlewareBinding?: WasmBinding
+    nextWasmMiddlewareBinding?: AssetBinding
+    nextAssetMiddlewareBinding?: AssetBinding
     usingIndirectEval?: boolean | Set<string>
     route?: RouteMeta
     importLocByPath?: Map<string, any>
@@ -32,7 +33,7 @@ export interface EdgeSSRMeta {
   page: string
 }
 
-export interface WasmBinding {
+export interface AssetBinding {
   filePath: string
   name: string
 }

packages/next/build/webpack/loaders/next-middleware-asset-loader.ts

@@ -0,0 +1,19 @@
+import loaderUtils from 'next/dist/compiled/loader-utils3'
+import { getModuleBuildInfo } from './get-module-build-info'
+
+export default function MiddlewareAssetLoader(this: any, source: Buffer) {
+  const name = loaderUtils.interpolateName(this, '[name].[hash].[ext]', {
+    context: this.rootContext,
+    content: source,
+  })
+  const filePath = `edge-chunks/asset_${name}`
+  const buildInfo = getModuleBuildInfo(this._module)
+  buildInfo.nextAssetMiddlewareBinding = {
+    filePath: `server/${filePath}`,
+    name,
+  }
+  this.emitFile(filePath, source)
+  return `module.exports = ${JSON.stringify(`blob:${name}`)}`
+}
+
+export const raw = true

packages/next/build/webpack/plugins/middleware-plugin.ts

@@ -1,5 +1,8 @@
-import type { EdgeMiddlewareMeta } from '../loaders/get-module-build-info'
-import type { EdgeSSRMeta, WasmBinding } from '../loaders/get-module-build-info'
+import type {
+  AssetBinding,
+  EdgeMiddlewareMeta,
+} from '../loaders/get-module-build-info'
+import type { EdgeSSRMeta } from '../loaders/get-module-build-info'
 import { getNamedMiddlewareRegex } from '../../../shared/lib/router/utils/route-regex'
 import { getModuleBuildInfo } from '../loaders/get-module-build-info'
 import { getSortedRoutes } from '../../../shared/lib/router/utils'
@@ -14,13 +17,14 @@ import {
   NEXT_CLIENT_SSR_ENTRY_SUFFIX,
 } from '../../../shared/lib/constants'
 
-interface EdgeFunctionDefinition {
+export interface EdgeFunctionDefinition {
   env: string[]
   files: string[]
   name: string
   page: string
   regexp: string
-  wasm?: WasmBinding[]
+  wasm?: AssetBinding[]
+  assets?: AssetBinding[]
 }
 
 export interface MiddlewareManifest {
@@ -35,7 +39,8 @@ interface EntryMetadata {
   edgeApiFunction?: EdgeMiddlewareMeta
   edgeSSR?: EdgeSSRMeta
   env: Set<string>
-  wasmBindings: Set<WasmBinding>
+  wasmBindings: Map<string, string>
+  assetBindings: Map<string, string>
 }
 
 const NAME = 'MiddlewarePlugin'
@@ -410,7 +415,8 @@ function getExtractMetadata(params: {
 
       const entryMetadata: EntryMetadata = {
         env: new Set<string>(),
-        wasmBindings: new Set<WasmBinding>(),
+        wasmBindings: new Map(),
+        assetBindings: new Map(),
       }
 
       for (const entryModule of entryModules) {
@@ -479,7 +485,17 @@ function getExtractMetadata(params: {
          * append it to the entry wasm bindings.
          */
         if (buildInfo?.nextWasmMiddlewareBinding) {
-          entryMetadata.wasmBindings.add(buildInfo.nextWasmMiddlewareBinding)
+          entryMetadata.wasmBindings.set(
+            buildInfo.nextWasmMiddlewareBinding.name,
+            buildInfo.nextWasmMiddlewareBinding.filePath
+          )
+        }
+
+        if (buildInfo?.nextAssetMiddlewareBinding) {
+          entryMetadata.assetBindings.set(
+            buildInfo.nextAssetMiddlewareBinding.name,
+            buildInfo.nextAssetMiddlewareBinding.filePath
+          )
         }
 
         /**
@@ -557,7 +573,14 @@ function getCreateAssets(params: {
         name: entrypoint.name,
         page: page,
         regexp,
-        wasm: Array.from(metadata.wasmBindings),
+        wasm: Array.from(metadata.wasmBindings, ([name, filePath]) => ({
+          name,
+          filePath,
+        })),
+        assets: Array.from(metadata.assetBindings, ([name, filePath]) => ({
+          name,
+          filePath,
+        })),
       }
 
       if (metadata.edgeApiFunction || metadata.edgeSSR) {

packages/next/server/body-streams.ts

@@ -7,7 +7,7 @@ type BodyStream = ReadableStream<Uint8Array>
 /**
  * Creates a ReadableStream from a Node.js HTTP request
  */
-export function requestToBodyStream(request: IncomingMessage): BodyStream {
+export function requestToBodyStream(request: Readable): BodyStream {
   const transform = new Primitives.TransformStream<Uint8Array, Uint8Array>({
     start(controller) {
       request.on('data', (chunk) => controller.enqueue(chunk))

packages/next/server/next-server.ts

@@ -1121,6 +1121,12 @@ export default class NextNodeServer extends BaseServer {
         ...binding,
         filePath: join(this.distDir, binding.filePath),
       })),
+      assets: (pageInfo.assets ?? []).map((binding) => {
+        return {
+          ...binding,
+          filePath: join(this.distDir, binding.filePath),
+        }
+      }),
     }
   }
 
@@ -1223,10 +1229,11 @@ export default class NextNodeServer extends BaseServer {
         }
 
         result = await run({
+          distDir: this.distDir,
           name: middlewareInfo.name,
           paths: middlewareInfo.paths,
           env: middlewareInfo.env,
-          wasm: middlewareInfo.wasm,
+          edgeFunctionEntry: middlewareInfo,
           request: {
             headers: params.request.headers,
             method,
@@ -1552,10 +1559,11 @@ export default class NextNodeServer extends BaseServer {
     const nodeReq = params.req as NodeNextRequest
 
     const result = await run({
+      distDir: this.distDir,
       name: middlewareInfo.name,
       paths: middlewareInfo.paths,
       env: middlewareInfo.env,
-      wasm: middlewareInfo.wasm,
+      edgeFunctionEntry: middlewareInfo,
       request: {
         headers: params.req.headers,
         method: params.req.method,

packages/next/server/web/sandbox/context.ts

@@ -1,5 +1,5 @@
 import type { Primitives } from 'next/dist/compiled/@edge-runtime/primitives'
-import type { WasmBinding } from '../../../build/webpack/loaders/get-module-build-info'
+import type { AssetBinding } from '../../../build/webpack/loaders/get-module-build-info'
 import {
   decorateServerError,
   getServerError,
@@ -9,6 +9,8 @@ import { EdgeRuntime } from 'next/dist/compiled/edge-runtime'
 import { readFileSync, promises as fs } from 'fs'
 import { validateURL } from '../utils'
 import { pick } from '../../../lib/pick'
+import { fetchInlineAsset } from './fetch-inline-assets'
+import type { EdgeFunctionDefinition } from '../../../build/webpack/plugins/middleware-plugin'
 
 const WEBPACK_HASH_REGEX =
   /__webpack_require__\.h = function\(\) \{ return "[0-9a-f]+"; \}/g
@@ -48,7 +50,8 @@ interface ModuleContextOptions {
   onWarning: (warn: Error) => void
   useCache: boolean
   env: string[]
-  wasm: WasmBinding[]
+  distDir: string
+  edgeFunctionEntry: Pick<EdgeFunctionDefinition, 'assets' | 'wasm'>
 }
 
 const pendingModuleCaches = new Map<string, Promise<ModuleContext>>()
@@ -103,7 +106,7 @@ export async function getModuleContext(options: ModuleContextOptions) {
 async function createModuleContext(options: ModuleContextOptions) {
   const warnedEvals = new Set<string>()
   const warnedWasmCodegens = new Set<string>()
-  const wasm = await loadWasm(options.wasm)
+  const wasm = await loadWasm(options.edgeFunctionEntry.wasm ?? [])
   const runtime = new EdgeRuntime({
     codeGeneration:
       process.env.NODE_ENV !== 'production'
@@ -197,7 +200,17 @@ Learn More: https://nextjs.org/docs/messages/middleware-dynamic-wasm-compilation
         }
 
       const __fetch = context.fetch
-      context.fetch = (input: RequestInfo, init: RequestInit = {}) => {
+      context.fetch = async (input: RequestInfo, init: RequestInit = {}) => {
+        const assetResponse = await fetchInlineAsset({
+          input,
+          assets: options.edgeFunctionEntry.assets,
+          distDir: options.distDir,
+          context,
+        })
+        if (assetResponse) {
+          return assetResponse
+        }
+
         init.headers = new Headers(init.headers ?? {})
         const prevs =
           init.headers.get(`x-middleware-subrequest`)?.split(':') || []
@@ -270,7 +283,7 @@ Learn More: https://nextjs.org/docs/messages/middleware-dynamic-wasm-compilation
 }
 
 async function loadWasm(
-  wasm: WasmBinding[]
+  wasm: AssetBinding[]
 ): Promise<Record<string, WebAssembly.Module>> {
   const modules: Record<string, WebAssembly.Module> = {}
 

packages/next/server/web/sandbox/fetch-inline-assets.ts

@@ -0,0 +1,39 @@
+import { createReadStream, promises as fs } from 'fs'
+import path from 'path'
+import { requestToBodyStream } from '../../body-streams'
+import type { EdgeFunctionDefinition } from '../../../build/webpack/plugins/middleware-plugin'
+
+/**
+ * Short-circuits the `fetch` function
+ * to return a stream for a given asset, if a user used `new URL("file", import.meta.url)`.
+ * This allows to embed assets in Edge Runtime.
+ */
+export async function fetchInlineAsset(options: {
+  input: RequestInfo
+  distDir: string
+  assets: EdgeFunctionDefinition['assets']
+  context: { Response: any }
+}): Promise<Response | undefined> {
+  const inputString = String(options.input)
+  if (!inputString.startsWith('blob:')) {
+    return
+  }
+
+  const hash = inputString.replace('blob:', '')
+  const asset = options.assets?.find((x) => x.name === hash)
+  if (!asset) {
+    return
+  }
+
+  const filePath = path.resolve(options.distDir, asset.filePath)
+
+  const fileIsReadable = await fs.access(filePath).then(
+    () => true,
+    () => false
+  )
+
+  if (fileIsReadable) {
+    const readStream = createReadStream(filePath)
+    return new options.context.Response(requestToBodyStream(readStream))
+  }
+}

packages/next/server/web/sandbox/sandbox.ts

@@ -1,7 +1,7 @@
 import type { RequestData, FetchEventResult } from '../types'
-import type { WasmBinding } from '../../../build/webpack/loaders/get-module-build-info'
 import { getServerError } from 'next/dist/compiled/@next/react-dev-overlay/dist/middleware'
 import { getModuleContext } from './context'
+import { EdgeFunctionDefinition } from '../../../build/webpack/plugins/middleware-plugin'
 
 export const ErrorSource = Symbol('SandboxError')
 
@@ -12,7 +12,8 @@ type RunnerFn = (params: {
   paths: string[]
   request: RequestData
   useCache: boolean
-  wasm: WasmBinding[]
+  edgeFunctionEntry: Pick<EdgeFunctionDefinition, 'wasm' | 'assets'>
+  distDir: string
 }) => Promise<FetchEventResult>
 
 export const run = withTaggedErrors(async (params) => {
@@ -21,7 +22,8 @@ export const run = withTaggedErrors(async (params) => {
     onWarning: params.onWarning,
     useCache: params.useCache !== false,
     env: params.env,
-    wasm: params.wasm,
+    edgeFunctionEntry: params.edgeFunctionEntry,
+    distDir: params.distDir,
   })
 
   for (const paramPath of params.paths) {

test/e2e/edge-compiler-can-import-blob-assets/app/node_modules/my-pkg/hello/world.json

@@ -0,0 +1 @@
+{ "i am": "a node dependency" }

test/e2e/edge-compiler-can-import-blob-assets/app/pages/api/edge.js

@@ -0,0 +1,60 @@
+export const config = { runtime: 'experimental-edge' }
+
+/**
+ * @param {import('next/server').NextRequest} req
+ */
+export default async (req) => {
+  const handlerName = req.nextUrl.searchParams.get('handler')
+  const handler = handlers.get(handlerName) || defaultHandler
+  return handler()
+}
+
+/**
+ * @type {Map<string, () => Promise<Response>>}
+ */
+const handlers = new Map([
+  [
+    'text-file',
+    async () => {
+      const url = new URL('../../src/text-file.txt', import.meta.url)
+      return fetch(url)
+    },
+  ],
+  [
+    'image-file',
+    async () => {
+      const url = new URL('../../src/vercel.png', import.meta.url)
+      return fetch(url)
+    },
+  ],
+  [
+    'from-node-module',
+    async () => {
+      const url = new URL('my-pkg/hello/world.json', import.meta.url)
+      return fetch(url)
+    },
+  ],
+  [
+    'remote-full',
+    async () => {
+      const url = new URL('https://example.vercel.sh')
+      const response = await fetch(url)
+      const headers = new Headers(response.headers)
+      headers.delete('content-encoding')
+      return new Response(response.body, { headers, status: response.status })
+    },
+  ],
+  [
+    'remote-with-base',
+    async () => {
+      const url = new URL('/', 'https://example.vercel.sh')
+      const response = await fetch(url)
+      const headers = new Headers(response.headers)
+      headers.delete('content-encoding')
+      return new Response(response.body, { headers, status: response.status })
+    },
+  ],
+])
+
+const defaultHandler = async () =>
+  new Response('Invalid handler', { status: 400 })

test/e2e/edge-compiler-can-import-blob-assets/app/src/text-file.txt

@@ -0,0 +1 @@
+Hello, from text-file.txt!

test/e2e/edge-compiler-can-import-blob-assets/index.test.ts

@@ -0,0 +1,97 @@
+import { createNext, FileRef } from 'e2e-utils'
+import { NextInstance } from 'test/lib/next-modes/base'
+import { fetchViaHTTP, renderViaHTTP } from 'next-test-utils'
+import path from 'path'
+import { promises as fs } from 'fs'
+import { readJson } from 'fs-extra'
+import type { MiddlewareManifest } from 'next/build/webpack/plugins/middleware-plugin'
+
+describe('Edge Compiler can import asset assets', () => {
+  let next: NextInstance
+
+  beforeAll(async () => {
+    next = await createNext({
+      files: new FileRef(path.join(__dirname, './app')),
+    })
+  })
+  afterAll(() => next.destroy())
+
+  it('allows to fetch a remote URL', async () => {
+    const response = await fetchViaHTTP(next.url, '/api/edge', {
+      handler: 'remote-full',
+    })
+    expect(await response.text()).toContain('Example Domain')
+  })
+
+  it('allows to fetch a remote URL with a path and basename', async () => {
+    const response = await fetchViaHTTP(
+      next.url,
+      '/api/edge',
+      {
+        handler: 'remote-with-base',
+      },
+      {
+        compress: true,
+      }
+    )
+    expect(await response.text()).toContain('Example Domain')
+  })
+
+  it('allows to fetch text assets', async () => {
+    const html = await renderViaHTTP(next.url, '/api/edge', {
+      handler: 'text-file',
+    })
+    expect(html).toContain('Hello, from text-file.txt!')
+  })
+
+  it('allows to fetch image assets', async () => {
+    const response = await fetchViaHTTP(next.url, '/api/edge', {
+      handler: 'image-file',
+    })
+    const buffer: Buffer = await response.buffer()
+    const image = await fs.readFile(
+      path.join(__dirname, './app/src/vercel.png')
+    )
+    expect(buffer.equals(image)).toBeTrue()
+  })
+
+  it('allows to assets from node_modules', async () => {
+    const response = await fetchViaHTTP(next.url, '/api/edge', {
+      handler: 'from-node-module',
+    })
+    const json = await response.json()
+    expect(json).toEqual({
+      'i am': 'a node dependency',
+    })
+  })
+
+  it('extracts all the assets from the bundle', async () => {
+    const manifestPath = path.join(
+      next.testDir,
+      '.next/server/middleware-manifest.json'
+    )
+    const manifest: MiddlewareManifest = await readJson(manifestPath)
+    const orderedAssets = manifest.functions['/api/edge'].assets.sort(
+      (a, z) => {
+        return String(a.name).localeCompare(z.name)
+      }
+    )
+
+    expect(orderedAssets).toMatchObject([
+      {
+        name: expect.stringMatching(/^text-file\.[0-9a-f]{16}\.txt$/),
+        filePath: expect.stringMatching(
+          /^server\/edge-chunks\/asset_text-file/
+        ),
+      },
+      {
+        name: expect.stringMatching(/^vercel\.[0-9a-f]{16}\.png$/),
+        filePath: expect.stringMatching(/^server\/edge-chunks\/asset_vercel/),
+      },
+      {
+        name: expect.stringMatching(/^world\.[0-9a-f]{16}\.json/),
+        filePath: expect.stringMatching(/^server\/edge-chunks\/asset_world/),
+      },
+    ])
+  })
+})

test/e2e/middleware-general/test/index.test.ts

@@ -114,6 +114,7 @@ describe('Middleware Runtime', () => {
             page: '/',
             regexp: '^/.*$',
             wasm: [],
+            assets: [],
           },
         })
       })

test/e2e/middleware-trailing-slash/test/index.test.ts

@@ -61,6 +61,7 @@ describe('Middleware Runtime trailing slash', () => {
             page: '/',
             regexp: '^/.*$',
             wasm: [],
+            assets: [],
           },
         })
       })