[edge] allow importing blob assets (#38492)
* [edge] allow importing blob assets * Fix test * extract to a new file, to make it easier to read and review * Use webpack asset discovery and transform with a loader * fix tests * don't prefix assets * use emitFile * rename assets to blobs to be more specific * rename blobs to assets and use webpack's hashing algo * Dedupe correctly * Add a Node.js dep test * Update packages/next/server/next-server.ts Co-authored-by: Tobias Koppers <tobias.koppers@googlemail.com> * [code review] test remote URL fetches * [code review] use `import type` for type-only imports * Update packages/next/server/next-server.ts Co-authored-by: Tobias Koppers <tobias.koppers@googlemail.com> * Apply suggestions from code review Co-authored-by: JJ Kasper <jj@jjsweb.site> Co-authored-by: Tobias Koppers <tobias.koppers@googlemail.com> Co-authored-by: JJ Kasper <jj@jjsweb.site>
This commit is contained in:
parent
9b312dbbe0
commit
20486c159d
16 changed files with 298 additions and 21 deletions
|
@ -1272,6 +1272,7 @@ export default async function getBaseWebpackConfig(
|
|||
'next-middleware-loader',
|
||||
'next-edge-function-loader',
|
||||
'next-edge-ssr-loader',
|
||||
'next-middleware-asset-loader',
|
||||
'next-middleware-wasm-loader',
|
||||
'next-app-loader',
|
||||
].reduce((alias, loader) => {
|
||||
|
@ -1769,6 +1770,16 @@ export default async function getBaseWebpackConfig(
|
|||
type: 'javascript/auto',
|
||||
resourceQuery: /module/i,
|
||||
})
|
||||
webpack5Config.module?.rules?.unshift({
|
||||
dependency: 'url',
|
||||
loader: 'next-middleware-asset-loader',
|
||||
type: 'javascript/auto',
|
||||
layer: 'edge-asset',
|
||||
})
|
||||
webpack5Config.module?.rules?.unshift({
|
||||
issuerLayer: 'edge-asset',
|
||||
type: 'asset/source',
|
||||
})
|
||||
}
|
||||
|
||||
webpack5Config.experiments = {
|
||||
|
|
|
@ -10,7 +10,8 @@ export function getModuleBuildInfo(webpackModule: webpack5.Module) {
|
|||
nextEdgeApiFunction?: EdgeMiddlewareMeta
|
||||
nextEdgeSSR?: EdgeSSRMeta
|
||||
nextUsedEnvVars?: Set<string>
|
||||
nextWasmMiddlewareBinding?: WasmBinding
|
||||
nextWasmMiddlewareBinding?: AssetBinding
|
||||
nextAssetMiddlewareBinding?: AssetBinding
|
||||
usingIndirectEval?: boolean | Set<string>
|
||||
route?: RouteMeta
|
||||
importLocByPath?: Map<string, any>
|
||||
|
@ -32,7 +33,7 @@ export interface EdgeSSRMeta {
|
|||
page: string
|
||||
}
|
||||
|
||||
export interface WasmBinding {
|
||||
export interface AssetBinding {
|
||||
filePath: string
|
||||
name: string
|
||||
}
|
||||
|
|
|
@ -0,0 +1,19 @@
|
|||
import loaderUtils from 'next/dist/compiled/loader-utils3'
|
||||
import { getModuleBuildInfo } from './get-module-build-info'
|
||||
|
||||
export default function MiddlewareAssetLoader(this: any, source: Buffer) {
|
||||
const name = loaderUtils.interpolateName(this, '[name].[hash].[ext]', {
|
||||
context: this.rootContext,
|
||||
content: source,
|
||||
})
|
||||
const filePath = `edge-chunks/asset_${name}`
|
||||
const buildInfo = getModuleBuildInfo(this._module)
|
||||
buildInfo.nextAssetMiddlewareBinding = {
|
||||
filePath: `server/${filePath}`,
|
||||
name,
|
||||
}
|
||||
this.emitFile(filePath, source)
|
||||
return `module.exports = ${JSON.stringify(`blob:${name}`)}`
|
||||
}
|
||||
|
||||
export const raw = true
|
|
@ -1,5 +1,8 @@
|
|||
import type { EdgeMiddlewareMeta } from '../loaders/get-module-build-info'
|
||||
import type { EdgeSSRMeta, WasmBinding } from '../loaders/get-module-build-info'
|
||||
import type {
|
||||
AssetBinding,
|
||||
EdgeMiddlewareMeta,
|
||||
} from '../loaders/get-module-build-info'
|
||||
import type { EdgeSSRMeta } from '../loaders/get-module-build-info'
|
||||
import { getNamedMiddlewareRegex } from '../../../shared/lib/router/utils/route-regex'
|
||||
import { getModuleBuildInfo } from '../loaders/get-module-build-info'
|
||||
import { getSortedRoutes } from '../../../shared/lib/router/utils'
|
||||
|
@ -14,13 +17,14 @@ import {
|
|||
NEXT_CLIENT_SSR_ENTRY_SUFFIX,
|
||||
} from '../../../shared/lib/constants'
|
||||
|
||||
interface EdgeFunctionDefinition {
|
||||
export interface EdgeFunctionDefinition {
|
||||
env: string[]
|
||||
files: string[]
|
||||
name: string
|
||||
page: string
|
||||
regexp: string
|
||||
wasm?: WasmBinding[]
|
||||
wasm?: AssetBinding[]
|
||||
assets?: AssetBinding[]
|
||||
}
|
||||
|
||||
export interface MiddlewareManifest {
|
||||
|
@ -35,7 +39,8 @@ interface EntryMetadata {
|
|||
edgeApiFunction?: EdgeMiddlewareMeta
|
||||
edgeSSR?: EdgeSSRMeta
|
||||
env: Set<string>
|
||||
wasmBindings: Set<WasmBinding>
|
||||
wasmBindings: Map<string, string>
|
||||
assetBindings: Map<string, string>
|
||||
}
|
||||
|
||||
const NAME = 'MiddlewarePlugin'
|
||||
|
@ -410,7 +415,8 @@ function getExtractMetadata(params: {
|
|||
|
||||
const entryMetadata: EntryMetadata = {
|
||||
env: new Set<string>(),
|
||||
wasmBindings: new Set<WasmBinding>(),
|
||||
wasmBindings: new Map(),
|
||||
assetBindings: new Map(),
|
||||
}
|
||||
|
||||
for (const entryModule of entryModules) {
|
||||
|
@ -479,7 +485,17 @@ function getExtractMetadata(params: {
|
|||
* append it to the entry wasm bindings.
|
||||
*/
|
||||
if (buildInfo?.nextWasmMiddlewareBinding) {
|
||||
entryMetadata.wasmBindings.add(buildInfo.nextWasmMiddlewareBinding)
|
||||
entryMetadata.wasmBindings.set(
|
||||
buildInfo.nextWasmMiddlewareBinding.name,
|
||||
buildInfo.nextWasmMiddlewareBinding.filePath
|
||||
)
|
||||
}
|
||||
|
||||
if (buildInfo?.nextAssetMiddlewareBinding) {
|
||||
entryMetadata.assetBindings.set(
|
||||
buildInfo.nextAssetMiddlewareBinding.name,
|
||||
buildInfo.nextAssetMiddlewareBinding.filePath
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -557,7 +573,14 @@ function getCreateAssets(params: {
|
|||
name: entrypoint.name,
|
||||
page: page,
|
||||
regexp,
|
||||
wasm: Array.from(metadata.wasmBindings),
|
||||
wasm: Array.from(metadata.wasmBindings, ([name, filePath]) => ({
|
||||
name,
|
||||
filePath,
|
||||
})),
|
||||
assets: Array.from(metadata.assetBindings, ([name, filePath]) => ({
|
||||
name,
|
||||
filePath,
|
||||
})),
|
||||
}
|
||||
|
||||
if (metadata.edgeApiFunction || metadata.edgeSSR) {
|
||||
|
|
|
@ -7,7 +7,7 @@ type BodyStream = ReadableStream<Uint8Array>
|
|||
/**
|
||||
* Creates a ReadableStream from a Node.js HTTP request
|
||||
*/
|
||||
export function requestToBodyStream(request: IncomingMessage): BodyStream {
|
||||
export function requestToBodyStream(request: Readable): BodyStream {
|
||||
const transform = new Primitives.TransformStream<Uint8Array, Uint8Array>({
|
||||
start(controller) {
|
||||
request.on('data', (chunk) => controller.enqueue(chunk))
|
||||
|
|
|
@ -1121,6 +1121,12 @@ export default class NextNodeServer extends BaseServer {
|
|||
...binding,
|
||||
filePath: join(this.distDir, binding.filePath),
|
||||
})),
|
||||
assets: (pageInfo.assets ?? []).map((binding) => {
|
||||
return {
|
||||
...binding,
|
||||
filePath: join(this.distDir, binding.filePath),
|
||||
}
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1223,10 +1229,11 @@ export default class NextNodeServer extends BaseServer {
|
|||
}
|
||||
|
||||
result = await run({
|
||||
distDir: this.distDir,
|
||||
name: middlewareInfo.name,
|
||||
paths: middlewareInfo.paths,
|
||||
env: middlewareInfo.env,
|
||||
wasm: middlewareInfo.wasm,
|
||||
edgeFunctionEntry: middlewareInfo,
|
||||
request: {
|
||||
headers: params.request.headers,
|
||||
method,
|
||||
|
@ -1552,10 +1559,11 @@ export default class NextNodeServer extends BaseServer {
|
|||
const nodeReq = params.req as NodeNextRequest
|
||||
|
||||
const result = await run({
|
||||
distDir: this.distDir,
|
||||
name: middlewareInfo.name,
|
||||
paths: middlewareInfo.paths,
|
||||
env: middlewareInfo.env,
|
||||
wasm: middlewareInfo.wasm,
|
||||
edgeFunctionEntry: middlewareInfo,
|
||||
request: {
|
||||
headers: params.req.headers,
|
||||
method: params.req.method,
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
import type { Primitives } from 'next/dist/compiled/@edge-runtime/primitives'
|
||||
import type { WasmBinding } from '../../../build/webpack/loaders/get-module-build-info'
|
||||
import type { AssetBinding } from '../../../build/webpack/loaders/get-module-build-info'
|
||||
import {
|
||||
decorateServerError,
|
||||
getServerError,
|
||||
|
@ -9,6 +9,8 @@ import { EdgeRuntime } from 'next/dist/compiled/edge-runtime'
|
|||
import { readFileSync, promises as fs } from 'fs'
|
||||
import { validateURL } from '../utils'
|
||||
import { pick } from '../../../lib/pick'
|
||||
import { fetchInlineAsset } from './fetch-inline-assets'
|
||||
import type { EdgeFunctionDefinition } from '../../../build/webpack/plugins/middleware-plugin'
|
||||
|
||||
const WEBPACK_HASH_REGEX =
|
||||
/__webpack_require__\.h = function\(\) \{ return "[0-9a-f]+"; \}/g
|
||||
|
@ -48,7 +50,8 @@ interface ModuleContextOptions {
|
|||
onWarning: (warn: Error) => void
|
||||
useCache: boolean
|
||||
env: string[]
|
||||
wasm: WasmBinding[]
|
||||
distDir: string
|
||||
edgeFunctionEntry: Pick<EdgeFunctionDefinition, 'assets' | 'wasm'>
|
||||
}
|
||||
|
||||
const pendingModuleCaches = new Map<string, Promise<ModuleContext>>()
|
||||
|
@ -103,7 +106,7 @@ export async function getModuleContext(options: ModuleContextOptions) {
|
|||
async function createModuleContext(options: ModuleContextOptions) {
|
||||
const warnedEvals = new Set<string>()
|
||||
const warnedWasmCodegens = new Set<string>()
|
||||
const wasm = await loadWasm(options.wasm)
|
||||
const wasm = await loadWasm(options.edgeFunctionEntry.wasm ?? [])
|
||||
const runtime = new EdgeRuntime({
|
||||
codeGeneration:
|
||||
process.env.NODE_ENV !== 'production'
|
||||
|
@ -197,7 +200,17 @@ Learn More: https://nextjs.org/docs/messages/middleware-dynamic-wasm-compilation
|
|||
}
|
||||
|
||||
const __fetch = context.fetch
|
||||
context.fetch = (input: RequestInfo, init: RequestInit = {}) => {
|
||||
context.fetch = async (input: RequestInfo, init: RequestInit = {}) => {
|
||||
const assetResponse = await fetchInlineAsset({
|
||||
input,
|
||||
assets: options.edgeFunctionEntry.assets,
|
||||
distDir: options.distDir,
|
||||
context,
|
||||
})
|
||||
if (assetResponse) {
|
||||
return assetResponse
|
||||
}
|
||||
|
||||
init.headers = new Headers(init.headers ?? {})
|
||||
const prevs =
|
||||
init.headers.get(`x-middleware-subrequest`)?.split(':') || []
|
||||
|
@ -270,7 +283,7 @@ Learn More: https://nextjs.org/docs/messages/middleware-dynamic-wasm-compilation
|
|||
}
|
||||
|
||||
async function loadWasm(
|
||||
wasm: WasmBinding[]
|
||||
wasm: AssetBinding[]
|
||||
): Promise<Record<string, WebAssembly.Module>> {
|
||||
const modules: Record<string, WebAssembly.Module> = {}
|
||||
|
||||
|
|
39
packages/next/server/web/sandbox/fetch-inline-assets.ts
Normal file
39
packages/next/server/web/sandbox/fetch-inline-assets.ts
Normal file
|
@ -0,0 +1,39 @@
|
|||
import { createReadStream, promises as fs } from 'fs'
|
||||
import path from 'path'
|
||||
import { requestToBodyStream } from '../../body-streams'
|
||||
import type { EdgeFunctionDefinition } from '../../../build/webpack/plugins/middleware-plugin'
|
||||
|
||||
/**
|
||||
* Short-circuits the `fetch` function
|
||||
* to return a stream for a given asset, if a user used `new URL("file", import.meta.url)`.
|
||||
* This allows to embed assets in Edge Runtime.
|
||||
*/
|
||||
export async function fetchInlineAsset(options: {
|
||||
input: RequestInfo
|
||||
distDir: string
|
||||
assets: EdgeFunctionDefinition['assets']
|
||||
context: { Response: any }
|
||||
}): Promise<Response | undefined> {
|
||||
const inputString = String(options.input)
|
||||
if (!inputString.startsWith('blob:')) {
|
||||
return
|
||||
}
|
||||
|
||||
const hash = inputString.replace('blob:', '')
|
||||
const asset = options.assets?.find((x) => x.name === hash)
|
||||
if (!asset) {
|
||||
return
|
||||
}
|
||||
|
||||
const filePath = path.resolve(options.distDir, asset.filePath)
|
||||
|
||||
const fileIsReadable = await fs.access(filePath).then(
|
||||
() => true,
|
||||
() => false
|
||||
)
|
||||
|
||||
if (fileIsReadable) {
|
||||
const readStream = createReadStream(filePath)
|
||||
return new options.context.Response(requestToBodyStream(readStream))
|
||||
}
|
||||
}
|
|
@ -1,7 +1,7 @@
|
|||
import type { RequestData, FetchEventResult } from '../types'
|
||||
import type { WasmBinding } from '../../../build/webpack/loaders/get-module-build-info'
|
||||
import { getServerError } from 'next/dist/compiled/@next/react-dev-overlay/dist/middleware'
|
||||
import { getModuleContext } from './context'
|
||||
import { EdgeFunctionDefinition } from '../../../build/webpack/plugins/middleware-plugin'
|
||||
|
||||
export const ErrorSource = Symbol('SandboxError')
|
||||
|
||||
|
@ -12,7 +12,8 @@ type RunnerFn = (params: {
|
|||
paths: string[]
|
||||
request: RequestData
|
||||
useCache: boolean
|
||||
wasm: WasmBinding[]
|
||||
edgeFunctionEntry: Pick<EdgeFunctionDefinition, 'wasm' | 'assets'>
|
||||
distDir: string
|
||||
}) => Promise<FetchEventResult>
|
||||
|
||||
export const run = withTaggedErrors(async (params) => {
|
||||
|
@ -21,7 +22,8 @@ export const run = withTaggedErrors(async (params) => {
|
|||
onWarning: params.onWarning,
|
||||
useCache: params.useCache !== false,
|
||||
env: params.env,
|
||||
wasm: params.wasm,
|
||||
edgeFunctionEntry: params.edgeFunctionEntry,
|
||||
distDir: params.distDir,
|
||||
})
|
||||
|
||||
for (const paramPath of params.paths) {
|
||||
|
|
1
test/e2e/edge-compiler-can-import-blob-assets/app/node_modules/my-pkg/hello/world.json
generated
vendored
Normal file
1
test/e2e/edge-compiler-can-import-blob-assets/app/node_modules/my-pkg/hello/world.json
generated
vendored
Normal file
|
@ -0,0 +1 @@
|
|||
{ "i am": "a node dependency" }
|
|
@ -0,0 +1,60 @@
|
|||
export const config = { runtime: 'experimental-edge' }
|
||||
|
||||
/**
|
||||
* @param {import('next/server').NextRequest} req
|
||||
*/
|
||||
export default async (req) => {
|
||||
const handlerName = req.nextUrl.searchParams.get('handler')
|
||||
const handler = handlers.get(handlerName) || defaultHandler
|
||||
return handler()
|
||||
}
|
||||
|
||||
/**
|
||||
* @type {Map<string, () => Promise<Response>>}
|
||||
*/
|
||||
const handlers = new Map([
|
||||
[
|
||||
'text-file',
|
||||
async () => {
|
||||
const url = new URL('../../src/text-file.txt', import.meta.url)
|
||||
return fetch(url)
|
||||
},
|
||||
],
|
||||
[
|
||||
'image-file',
|
||||
async () => {
|
||||
const url = new URL('../../src/vercel.png', import.meta.url)
|
||||
return fetch(url)
|
||||
},
|
||||
],
|
||||
[
|
||||
'from-node-module',
|
||||
async () => {
|
||||
const url = new URL('my-pkg/hello/world.json', import.meta.url)
|
||||
return fetch(url)
|
||||
},
|
||||
],
|
||||
[
|
||||
'remote-full',
|
||||
async () => {
|
||||
const url = new URL('https://example.vercel.sh')
|
||||
const response = await fetch(url)
|
||||
const headers = new Headers(response.headers)
|
||||
headers.delete('content-encoding')
|
||||
return new Response(response.body, { headers, status: response.status })
|
||||
},
|
||||
],
|
||||
[
|
||||
'remote-with-base',
|
||||
async () => {
|
||||
const url = new URL('/', 'https://example.vercel.sh')
|
||||
const response = await fetch(url)
|
||||
const headers = new Headers(response.headers)
|
||||
headers.delete('content-encoding')
|
||||
return new Response(response.body, { headers, status: response.status })
|
||||
},
|
||||
],
|
||||
])
|
||||
|
||||
const defaultHandler = async () =>
|
||||
new Response('Invalid handler', { status: 400 })
|
|
@ -0,0 +1 @@
|
|||
Hello, from text-file.txt!
|
BIN
test/e2e/edge-compiler-can-import-blob-assets/app/src/vercel.png
Normal file
BIN
test/e2e/edge-compiler-can-import-blob-assets/app/src/vercel.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 29 KiB |
97
test/e2e/edge-compiler-can-import-blob-assets/index.test.ts
Normal file
97
test/e2e/edge-compiler-can-import-blob-assets/index.test.ts
Normal file
|
@ -0,0 +1,97 @@
|
|||
import { createNext, FileRef } from 'e2e-utils'
|
||||
import { NextInstance } from 'test/lib/next-modes/base'
|
||||
import { fetchViaHTTP, renderViaHTTP } from 'next-test-utils'
|
||||
import path from 'path'
|
||||
import { promises as fs } from 'fs'
|
||||
import { readJson } from 'fs-extra'
|
||||
import type { MiddlewareManifest } from 'next/build/webpack/plugins/middleware-plugin'
|
||||
|
||||
describe('Edge Compiler can import asset assets', () => {
|
||||
let next: NextInstance
|
||||
|
||||
beforeAll(async () => {
|
||||
next = await createNext({
|
||||
files: new FileRef(path.join(__dirname, './app')),
|
||||
})
|
||||
})
|
||||
afterAll(() => next.destroy())
|
||||
|
||||
it('allows to fetch a remote URL', async () => {
|
||||
const response = await fetchViaHTTP(next.url, '/api/edge', {
|
||||
handler: 'remote-full',
|
||||
})
|
||||
expect(await response.text()).toContain('Example Domain')
|
||||
})
|
||||
|
||||
it('allows to fetch a remote URL with a path and basename', async () => {
|
||||
const response = await fetchViaHTTP(
|
||||
next.url,
|
||||
'/api/edge',
|
||||
{
|
||||
handler: 'remote-with-base',
|
||||
},
|
||||
{
|
||||
compress: true,
|
||||
}
|
||||
)
|
||||
expect(await response.text()).toContain('Example Domain')
|
||||
})
|
||||
|
||||
it('allows to fetch text assets', async () => {
|
||||
const html = await renderViaHTTP(next.url, '/api/edge', {
|
||||
handler: 'text-file',
|
||||
})
|
||||
expect(html).toContain('Hello, from text-file.txt!')
|
||||
})
|
||||
|
||||
it('allows to fetch image assets', async () => {
|
||||
const response = await fetchViaHTTP(next.url, '/api/edge', {
|
||||
handler: 'image-file',
|
||||
})
|
||||
const buffer: Buffer = await response.buffer()
|
||||
const image = await fs.readFile(
|
||||
path.join(__dirname, './app/src/vercel.png')
|
||||
)
|
||||
expect(buffer.equals(image)).toBeTrue()
|
||||
})
|
||||
|
||||
it('allows to assets from node_modules', async () => {
|
||||
const response = await fetchViaHTTP(next.url, '/api/edge', {
|
||||
handler: 'from-node-module',
|
||||
})
|
||||
const json = await response.json()
|
||||
expect(json).toEqual({
|
||||
'i am': 'a node dependency',
|
||||
})
|
||||
})
|
||||
|
||||
it('extracts all the assets from the bundle', async () => {
|
||||
const manifestPath = path.join(
|
||||
next.testDir,
|
||||
'.next/server/middleware-manifest.json'
|
||||
)
|
||||
const manifest: MiddlewareManifest = await readJson(manifestPath)
|
||||
const orderedAssets = manifest.functions['/api/edge'].assets.sort(
|
||||
(a, z) => {
|
||||
return String(a.name).localeCompare(z.name)
|
||||
}
|
||||
)
|
||||
|
||||
expect(orderedAssets).toMatchObject([
|
||||
{
|
||||
name: expect.stringMatching(/^text-file\.[0-9a-f]{16}\.txt$/),
|
||||
filePath: expect.stringMatching(
|
||||
/^server\/edge-chunks\/asset_text-file/
|
||||
),
|
||||
},
|
||||
{
|
||||
name: expect.stringMatching(/^vercel\.[0-9a-f]{16}\.png$/),
|
||||
filePath: expect.stringMatching(/^server\/edge-chunks\/asset_vercel/),
|
||||
},
|
||||
{
|
||||
name: expect.stringMatching(/^world\.[0-9a-f]{16}\.json/),
|
||||
filePath: expect.stringMatching(/^server\/edge-chunks\/asset_world/),
|
||||
},
|
||||
])
|
||||
})
|
||||
})
|
|
@ -114,6 +114,7 @@ describe('Middleware Runtime', () => {
|
|||
page: '/',
|
||||
regexp: '^/.*$',
|
||||
wasm: [],
|
||||
assets: [],
|
||||
},
|
||||
})
|
||||
})
|
||||
|
|
|
@ -61,6 +61,7 @@ describe('Middleware Runtime trailing slash', () => {
|
|||
page: '/',
|
||||
regexp: '^/.*$',
|
||||
wasm: [],
|
||||
assets: [],
|
||||
},
|
||||
})
|
||||
})
|
||||
|
|
Loading…
Reference in a new issue