diff --git a/packages/next/src/client/app-index.tsx b/packages/next/src/client/app-index.tsx index f59ee8507b..9058c233bf 100644 --- a/packages/next/src/client/app-index.tsx +++ b/packages/next/src/client/app-index.tsx @@ -43,7 +43,7 @@ const appElement: HTMLElement | Document | null = document const encoder = new TextEncoder() -let initialServerDataBuffer: string[] | undefined = undefined +let initialServerDataBuffer: (string | Uint8Array)[] | undefined = undefined let initialServerDataWriter: ReadableStreamDefaultController | undefined = undefined let initialServerDataLoaded = false @@ -56,6 +56,7 @@ function nextServerDataCallback( | [isBootStrap: 0] | [isNotBootstrap: 1, responsePartial: string] | [isFormState: 2, formState: any] + | [isBinary: 3, responseBase64Partial: string] ): void { if (seg[0] === 0) { initialServerDataBuffer = [] @@ -70,6 +71,22 @@ function nextServerDataCallback( } } else if (seg[0] === 2) { initialFormStateData = seg[1] + } else if (seg[0] === 3) { + if (!initialServerDataBuffer) + throw new Error('Unexpected server data: missing bootstrap script.') + + // Decode the base64 string back to binary data. + const binaryString = atob(seg[1]) + const decodedChunk = new Uint8Array(binaryString.length) + for (var i = 0; i < binaryString.length; i++) { + decodedChunk[i] = binaryString.charCodeAt(i) + } + + if (initialServerDataWriter) { + initialServerDataWriter.enqueue(decodedChunk) + } else { + initialServerDataBuffer.push(decodedChunk) + } } } @@ -84,7 +101,7 @@ function nextServerDataCallback( function nextServerDataRegisterWriter(ctr: ReadableStreamDefaultController) { if (initialServerDataBuffer) { initialServerDataBuffer.forEach((val) => { - ctr.enqueue(encoder.encode(val)) + ctr.enqueue(typeof val === 'string' ? encoder.encode(val) : val) }) if (initialServerDataLoaded && !initialServerDataFlushed) { ctr.close() diff --git a/packages/next/src/server/app-render/app-render.tsx b/packages/next/src/server/app-render/app-render.tsx index 23facb23c4..95ec13aa14 100644 --- a/packages/next/src/server/app-render/app-render.tsx +++ b/packages/next/src/server/app-render/app-render.tsx @@ -398,7 +398,7 @@ function createFlightDataResolver(ctx: AppRenderContext) { // Generate the flight data and as soon as it can, convert it into a string. const promise = generateFlight(ctx) .then(async (result) => ({ - flightData: await result.toUnchunkedString(true), + flightData: await result.toUnchunkedBuffer(true), })) // Otherwise if it errored, return the error. .catch((err) => ({ err })) diff --git a/packages/next/src/server/app-render/use-flight-response.tsx b/packages/next/src/server/app-render/use-flight-response.tsx index 6069a71aff..86831a58bd 100644 --- a/packages/next/src/server/app-render/use-flight-response.tsx +++ b/packages/next/src/server/app-render/use-flight-response.tsx @@ -9,6 +9,7 @@ const isEdgeRuntime = process.env.NEXT_RUNTIME === 'edge' const INLINE_FLIGHT_PAYLOAD_BOOTSTRAP = 0 const INLINE_FLIGHT_PAYLOAD_DATA = 1 const INLINE_FLIGHT_PAYLOAD_FORM_STATE = 2 +const INLINE_FLIGHT_PAYLOAD_BINARY = 3 const flightResponses = new WeakMap, Promise>() const encoder = new TextEncoder() @@ -96,10 +97,8 @@ export function createInlinedDataReadableStream( ? `` + `${scriptStart}self.__next_f.push(${htmlInlinedData})` ) ) } diff --git a/packages/next/src/server/base-server.ts b/packages/next/src/server/base-server.ts index 48a5e45e64..efdfd1a103 100644 --- a/packages/next/src/server/base-server.ts +++ b/packages/next/src/server/base-server.ts @@ -13,6 +13,8 @@ import type { ParsedUrlQuery } from 'querystring' import type { RenderOptsPartial as PagesRenderOptsPartial } from './render' import type { RenderOptsPartial as AppRenderOptsPartial } from './app-render/types' import type { + CachedAppPageValue, + CachedPageValue, ResponseCacheBase, ResponseCacheEntry, ResponseGenerator, @@ -2543,16 +2545,29 @@ export default abstract class Server< return null } + if (isAppPath) { + return { + value: { + kind: 'APP_PAGE', + html: result, + headers, + rscData: metadata.flightData, + postponed: metadata.postponed, + status: res.statusCode, + } satisfies CachedAppPageValue, + revalidate: metadata.revalidate, + } + } + // We now have a valid HTML result that we can return to the user. return { value: { kind: 'PAGE', html: result, - pageData: metadata.pageData ?? metadata.flightData, - postponed: metadata.postponed, + pageData: metadata.pageData, headers, - status: isAppPath ? res.statusCode : undefined, - }, + status: res.statusCode, + } satisfies CachedPageValue, revalidate: metadata.revalidate, } } @@ -2665,7 +2680,6 @@ export default abstract class Server< value: { kind: 'PAGE', html: RenderResult.fromStatic(html), - postponed: undefined, status: undefined, headers: undefined, pageData: {}, @@ -2734,7 +2748,7 @@ export default abstract class Server< } const didPostpone = - cacheEntry.value?.kind === 'PAGE' && + cacheEntry.value?.kind === 'APP_PAGE' && typeof cacheEntry.value.postponed === 'string' if ( @@ -2901,7 +2915,11 @@ export default abstract class Server< } else if (isAppPath) { // If the request has a postponed state and it's a resume request we // should error. - if (cachedData.postponed && minimalPostponed) { + if ( + cachedData.kind === 'APP_PAGE' && + cachedData.postponed && + minimalPostponed + ) { throw new Error( 'Invariant: postponed state should not be present on a resume request' ) @@ -2949,7 +2967,11 @@ export default abstract class Server< } // Mark that the request did postpone if this is a data request. - if (cachedData.postponed && isRSCRequest) { + if ( + cachedData.kind === 'APP_PAGE' && + cachedData.postponed && + isRSCRequest + ) { res.setHeader(NEXT_DID_POSTPONE_HEADER, '1') } @@ -2960,8 +2982,15 @@ export default abstract class Server< if (isDataReq && !isPreviewMode) { // If this is a dynamic RSC request, then stream the response. if (isDynamicRSCRequest) { - if (cachedData.pageData) { - throw new Error('Invariant: Expected pageData to be undefined') + if (cachedData.kind !== 'APP_PAGE') { + console.error({ url: req.url, pathname }, cachedData) + throw new Error( + `Invariant: expected cache data kind of APP_PAGE got ${cachedData.kind}` + ) + } + + if (cachedData.rscData) { + throw new Error('Invariant: Expected rscData to be undefined') } if (cachedData.postponed) { @@ -2980,9 +3009,15 @@ export default abstract class Server< } } - if (typeof cachedData.pageData !== 'string') { + if (cachedData.kind !== 'APP_PAGE') { throw new Error( - `Invariant: expected pageData to be a string, got ${typeof cachedData.pageData}` + `Invariant: expected cached data to be APP_PAGE got ${cachedData.kind}` + ) + } + + if (!Buffer.isBuffer(cachedData.rscData)) { + throw new Error( + `Invariant: expected rscData to be a Buffer, got ${typeof cachedData.rscData}` ) } @@ -2990,7 +3025,7 @@ export default abstract class Server< // data. return { type: 'rsc', - body: RenderResult.fromStatic(cachedData.pageData), + body: RenderResult.fromStatic(cachedData.rscData), revalidate: cacheEntry.revalidate, } } @@ -3001,7 +3036,10 @@ export default abstract class Server< // If there's no postponed state, we should just serve the HTML. This // should also be the case for a resume request because it's completed // as a server render (rather than a static render). - if (!cachedData.postponed || this.minimalMode) { + if ( + !(cachedData.kind === 'APP_PAGE' && cachedData.postponed) || + this.minimalMode + ) { return { type: 'html', body, @@ -3030,7 +3068,7 @@ export default abstract class Server< throw new Error('Invariant: expected a result to be returned') } - if (result.value?.kind !== 'PAGE') { + if (result.value?.kind !== 'APP_PAGE') { throw new Error( `Invariant: expected a page response, got ${result.value?.kind}` ) @@ -3056,6 +3094,11 @@ export default abstract class Server< revalidate: 0, } } else if (isDataReq) { + if (cachedData.kind !== 'PAGE') { + throw new Error( + `Invariant: expected cached data to be PAGE got ${cachedData.kind}` + ) + } return { type: 'json', body: RenderResult.fromStatic(JSON.stringify(cachedData.pageData)), diff --git a/packages/next/src/server/lib/incremental-cache/fetch-cache.ts b/packages/next/src/server/lib/incremental-cache/fetch-cache.ts index 6ecb7048ed..55d77c0f22 100644 --- a/packages/next/src/server/lib/incremental-cache/fetch-cache.ts +++ b/packages/next/src/server/lib/incremental-cache/fetch-cache.ts @@ -109,7 +109,10 @@ export default class FetchCache implements CacheHandler { } // rough estimate of size of cache value return ( - value.html.length + (JSON.stringify(value.pageData)?.length || 0) + value.html.length + + (JSON.stringify( + value.kind === 'APP_PAGE' ? value.rscData : value.pageData + )?.length || 0) ) }, }) diff --git a/packages/next/src/server/lib/incremental-cache/file-system-cache.ts b/packages/next/src/server/lib/incremental-cache/file-system-cache.ts index 2e8400da74..379917ea2e 100644 --- a/packages/next/src/server/lib/incremental-cache/file-system-cache.ts +++ b/packages/next/src/server/lib/incremental-cache/file-system-cache.ts @@ -77,7 +77,10 @@ export default class FileSystemCache implements CacheHandler { } // rough estimate of size of cache value return ( - value.html.length + (JSON.stringify(value.pageData)?.length || 0) + value.html.length + + (JSON.stringify( + value.kind === 'APP_PAGE' ? value.rscData : value.pageData + )?.length || 0) ) }, }) @@ -250,23 +253,6 @@ export default class FileSystemCache implements CacheHandler { } } } else { - const pageData = isAppPath - ? await this.fs.readFile( - this.getFilePath( - `${key}${ - this.isAppPPREnabled ? RSC_PREFETCH_SUFFIX : RSC_SUFFIX - }`, - 'app' - ), - 'utf8' - ) - : JSON.parse( - await this.fs.readFile( - this.getFilePath(`${key}${NEXT_DATA_SUFFIX}`, 'pages'), - 'utf8' - ) - ) - let meta: RouteMetadata | undefined if (isAppPath) { @@ -280,16 +266,44 @@ export default class FileSystemCache implements CacheHandler { } catch {} } - data = { - lastModified: mtime.getTime(), - value: { - kind: 'PAGE', - html: fileData, - pageData, - postponed: meta?.postponed, - headers: meta?.headers, - status: meta?.status, - }, + if (isAppPath) { + const rscData = await this.fs.readFile( + this.getFilePath( + `${key}${ + this.isAppPPREnabled ? RSC_PREFETCH_SUFFIX : RSC_SUFFIX + }`, + 'app' + ) + ) + data = { + lastModified: mtime.getTime(), + value: { + kind: 'APP_PAGE', + html: fileData, + rscData, + postponed: meta?.postponed, + headers: meta?.headers, + status: meta?.status, + }, + } + } else { + const pageData = JSON.parse( + await this.fs.readFile( + this.getFilePath(`${key}${NEXT_DATA_SUFFIX}`, 'pages'), + 'utf8' + ) + ) + + data = { + lastModified: mtime.getTime(), + value: { + kind: 'PAGE', + html: fileData, + pageData, + headers: meta?.headers, + status: meta?.status, + }, + } } } @@ -301,7 +315,7 @@ export default class FileSystemCache implements CacheHandler { } } - if (data?.value?.kind === 'PAGE') { + if (data?.value?.kind === 'APP_PAGE' || data?.value?.kind === 'PAGE') { let cacheTags: undefined | string[] const tagsHeader = data.value.headers?.[NEXT_CACHE_TAGS_HEADER] @@ -385,8 +399,8 @@ export default class FileSystemCache implements CacheHandler { return } - if (data?.kind === 'PAGE') { - const isAppPath = typeof data.pageData === 'string' + if (data?.kind === 'PAGE' || data?.kind === 'APP_PAGE') { + const isAppPath = 'rscData' in data const htmlPath = this.getFilePath( `${key}.html`, isAppPath ? 'app' : 'pages' @@ -405,14 +419,14 @@ export default class FileSystemCache implements CacheHandler { }`, isAppPath ? 'app' : 'pages' ), - isAppPath ? data.pageData : JSON.stringify(data.pageData) + isAppPath ? data.rscData : JSON.stringify(data.pageData) ) if (data.headers || data.status) { const meta: RouteMetadata = { headers: data.headers, status: data.status, - postponed: data.postponed, + postponed: isAppPath ? data.postponed : undefined, } await this.fs.writeFile( diff --git a/packages/next/src/server/render-result.ts b/packages/next/src/server/render-result.ts index 57bd89695d..448a043179 100644 --- a/packages/next/src/server/render-result.ts +++ b/packages/next/src/server/render-result.ts @@ -4,7 +4,9 @@ import type { FetchMetrics } from './base-http' import { chainStreams, + streamFromBuffer, streamFromString, + streamToBuffer, streamToString, } from './stream-utils/node-web-streams-helper' import { isAbortError, pipeToNodeResponse } from './pipe-readable' @@ -12,7 +14,7 @@ import { isAbortError, pipeToNodeResponse } from './pipe-readable' type ContentTypeOption = string | undefined export type AppPageRenderResultMetadata = { - flightData?: string + flightData?: Buffer revalidate?: Revalidate staticBailoutInfo?: { stack?: string @@ -50,6 +52,7 @@ export type RenderResultResponse = | ReadableStream[] | ReadableStream | string + | Buffer | null export type RenderResultOptions< @@ -89,7 +92,7 @@ export default class RenderResult< * @param value the static response value * @returns a new RenderResult instance */ - public static fromStatic(value: string) { + public static fromStatic(value: string | Buffer) { return new RenderResult(value, { metadata: {} }) } @@ -125,6 +128,26 @@ export default class RenderResult< return typeof this.response !== 'string' } + public toUnchunkedBuffer(stream?: false): Buffer + public toUnchunkedBuffer(stream: true): Promise + public toUnchunkedBuffer(stream = false): Promise | Buffer { + if (this.response === null) { + throw new Error('Invariant: null responses cannot be unchunked') + } + + if (typeof this.response !== 'string') { + if (!stream) { + throw new Error( + 'Invariant: dynamic responses cannot be unchunked. This is a bug in Next.js' + ) + } + + return streamToBuffer(this.readable) + } + + return Buffer.from(this.response) + } + /** * Returns the response if it is a string. If the page was dynamic, this will * return a promise if the `stream` option is true, or it will throw an error. @@ -164,6 +187,10 @@ export default class RenderResult< throw new Error('Invariant: static responses cannot be streamed') } + if (Buffer.isBuffer(this.response)) { + return streamFromBuffer(this.response) + } + // If the response is an array of streams, then chain them together. if (Array.isArray(this.response)) { return chainStreams(...this.response) @@ -191,6 +218,8 @@ export default class RenderResult< responses = [streamFromString(this.response)] } else if (Array.isArray(this.response)) { responses = this.response + } else if (Buffer.isBuffer(this.response)) { + responses = [streamFromBuffer(this.response)] } else { responses = [this.response] } diff --git a/packages/next/src/server/response-cache/types.ts b/packages/next/src/server/response-cache/types.ts index 4f098edf99..e8492bb3f4 100644 --- a/packages/next/src/server/response-cache/types.ts +++ b/packages/next/src/server/response-cache/types.ts @@ -40,12 +40,22 @@ export interface CachedRedirectValue { props: Object } -interface CachedPageValue { +export interface CachedAppPageValue { + kind: 'APP_PAGE' + // this needs to be a RenderResult so since renderResponse + // expects that type instead of a string + html: RenderResult + rscData: Buffer | undefined + status: number | undefined + postponed: string | undefined + headers: OutgoingHttpHeaders | undefined +} + +export interface CachedPageValue { kind: 'PAGE' // this needs to be a RenderResult so since renderResponse // expects that type instead of a string html: RenderResult - postponed: string | undefined pageData: Object status: number | undefined headers: OutgoingHttpHeaders | undefined @@ -69,13 +79,23 @@ export interface CachedImageValue { isStale?: boolean } -interface IncrementalCachedPageValue { +export interface IncrementalCachedAppPageValue { + kind: 'APP_PAGE' + // this needs to be a string since the cache expects to store + // the string value + html: string + rscData: Buffer | undefined + headers: OutgoingHttpHeaders | undefined + postponed: string | undefined + status: number | undefined +} + +export interface IncrementalCachedPageValue { kind: 'PAGE' // this needs to be a string since the cache expects to store // the string value html: string pageData: Object - postponed: string | undefined headers: OutgoingHttpHeaders | undefined status: number | undefined } @@ -92,6 +112,7 @@ export type IncrementalCacheEntry = { export type IncrementalCacheValue = | CachedRedirectValue | IncrementalCachedPageValue + | IncrementalCachedAppPageValue | CachedImageValue | CachedFetchValue | CachedRouteValue @@ -99,6 +120,7 @@ export type IncrementalCacheValue = export type ResponseCacheValue = | CachedRedirectValue | CachedPageValue + | CachedAppPageValue | CachedImageValue | CachedRouteValue diff --git a/packages/next/src/server/response-cache/utils.ts b/packages/next/src/server/response-cache/utils.ts index 174d6fa619..18ec64c1dd 100644 --- a/packages/next/src/server/response-cache/utils.ts +++ b/packages/next/src/server/response-cache/utils.ts @@ -12,12 +12,20 @@ export async function fromResponseCacheEntry( ? { kind: 'PAGE', html: await cacheEntry.value.html.toUnchunkedString(true), - postponed: cacheEntry.value.postponed, pageData: cacheEntry.value.pageData, headers: cacheEntry.value.headers, status: cacheEntry.value.status, } - : cacheEntry.value, + : cacheEntry.value?.kind === 'APP_PAGE' + ? { + kind: 'APP_PAGE', + html: await cacheEntry.value.html.toUnchunkedString(true), + postponed: cacheEntry.value.postponed, + rscData: cacheEntry.value.rscData, + headers: cacheEntry.value.headers, + status: cacheEntry.value.status, + } + : cacheEntry.value, } } @@ -42,10 +50,18 @@ export async function toResponseCacheEntry( kind: 'PAGE', html: RenderResult.fromStatic(response.value.html), pageData: response.value.pageData, - postponed: response.value.postponed, headers: response.value.headers, status: response.value.status, } - : response.value, + : response.value?.kind === 'APP_PAGE' + ? { + kind: 'APP_PAGE', + html: RenderResult.fromStatic(response.value.html), + rscData: response.value.rscData, + headers: response.value.headers, + status: response.value.status, + postponed: response.value.postponed, + } + : response.value, } } diff --git a/packages/next/src/server/stream-utils/node-web-streams-helper.ts b/packages/next/src/server/stream-utils/node-web-streams-helper.ts index a3d4ae8b92..8b051257c5 100644 --- a/packages/next/src/server/stream-utils/node-web-streams-helper.ts +++ b/packages/next/src/server/stream-utils/node-web-streams-helper.ts @@ -73,6 +73,28 @@ export function streamFromString(str: string): ReadableStream { }) } +export function streamFromBuffer(chunk: Buffer): ReadableStream { + return new ReadableStream({ + start(controller) { + controller.enqueue(chunk) + controller.close() + }, + }) +} + +export async function streamToBuffer( + stream: ReadableStream +): Promise { + const buffers: Buffer[] = [] + + // @ts-expect-error TypeScript gets this wrong (https://nodejs.org/api/webstreams.html#async-iteration) + for await (const chunk of stream) { + buffers.push(chunk) + } + + return Buffer.concat(buffers) +} + export async function streamToString( stream: ReadableStream ): Promise { diff --git a/test/e2e/app-dir/binary/app/client.js b/test/e2e/app-dir/binary/app/client.js new file mode 100644 index 0000000000..37f29ac669 --- /dev/null +++ b/test/e2e/app-dir/binary/app/client.js @@ -0,0 +1,19 @@ +'use client' + +import { useEffect, useState } from 'react' + +export function Client({ binary, arbitrary }) { + const [hydrated, setHydrated] = useState(false) + + useEffect(() => { + setHydrated(true) + }, []) + + return ( + <> +
utf8 binary: {new TextDecoder().decode(binary)}
+
arbitrary binary: {String(arbitrary)}
+
hydrated: {String(hydrated)}
+ + ) +} diff --git a/test/e2e/app-dir/binary/app/layout.js b/test/e2e/app-dir/binary/app/layout.js new file mode 100644 index 0000000000..8525f5f8c0 --- /dev/null +++ b/test/e2e/app-dir/binary/app/layout.js @@ -0,0 +1,12 @@ +export const metadata = { + title: 'Next.js', + description: 'Generated by Next.js', +} + +export default function RootLayout({ children }) { + return ( + + {children} + + ) +} diff --git a/test/e2e/app-dir/binary/app/page.js b/test/e2e/app-dir/binary/app/page.js new file mode 100644 index 0000000000..5bbd22eede --- /dev/null +++ b/test/e2e/app-dir/binary/app/page.js @@ -0,0 +1,8 @@ +import { Client } from './client' + +export default function Page() { + const binaryData = new Uint8Array([104, 101, 108, 108, 111]) + const nonUtf8BinaryData = new Uint8Array([0xff, 0, 1, 2, 3]) + + return +} diff --git a/test/e2e/app-dir/binary/next.config.js b/test/e2e/app-dir/binary/next.config.js new file mode 100644 index 0000000000..5b7ed7e24f --- /dev/null +++ b/test/e2e/app-dir/binary/next.config.js @@ -0,0 +1,6 @@ +module.exports = { + experimental: { + // This ensures that we're running the experimental React. + taint: true, + }, +} diff --git a/test/e2e/app-dir/binary/rsc-binary.test.ts b/test/e2e/app-dir/binary/rsc-binary.test.ts new file mode 100644 index 0000000000..4f29a0a3d5 --- /dev/null +++ b/test/e2e/app-dir/binary/rsc-binary.test.ts @@ -0,0 +1,32 @@ +import { nextTestSetup } from 'e2e-utils' +import { check } from 'next-test-utils' + +describe('RSC binary serialization', () => { + const { next, skipped } = nextTestSetup({ + files: __dirname, + skipDeployment: true, + dependencies: { + react: '19.0.0-beta-4508873393-20240430', + 'react-dom': '19.0.0-beta-4508873393-20240430', + 'server-only': 'latest', + }, + }) + if (skipped) return + + afterEach(async () => { + await next.stop() + }) + + it('should correctly encode/decode binaries and hydrate', async function () { + const browser = await next.browser('/') + await check(async () => { + const content = await browser.elementByCss('body').text() + + return content.includes('utf8 binary: hello') && + content.includes('arbitrary binary: 255,0,1,2,3') && + content.includes('hydrated: true') + ? 'success' + : 'fail' + }, 'success') + }) +})