archive/rsnext
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix broken HTML inlining of non UTF-8 decodable binary data from Flight payload (#65664)

Browse source 
This PR ensures that any arbitrary binary data can be passed across the
RSC boundary, especially when inlined in HTML. While the Flight payloads
in RSC requests (`text/x-component`) already work, it's a different case
when we inline them directly in HTML as that's required to be a valid
string in UTF-8.

So instead of always inlining the UTF-8 decoded chunk (`new
TextDecoder('utf-8')`), we fallback non-decodable chunks to base64 and
send as a special item in `__next_f` so we can safely change it back to
a binary typed array.

---------

Co-authored-by: JJ Kasper <jj@jjsweb.site>
This commit is contained in:
Shu Ding 2024-05-15 15:30:46 -07:00 committed by GitHub
parent d9ce336a13
commit a34d909877
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
15 changed files with 344 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
packages/next/src/client/app-index.tsx
View file

@ -43,7 +43,7 @@ const appElement: HTMLElement | Document | null = document
const encoder = new TextEncoder()
let initialServerDataBuffer: string[] | undefined = undefined
let initialServerDataBuffer: (string | Uint8Array)[] | undefined = undefined
let initialServerDataWriter: ReadableStreamDefaultController | undefined =
undefined
let initialServerDataLoaded = false
@ -56,6 +56,7 @@ function nextServerDataCallback(
| [isBootStrap: 0]
| [isNotBootstrap: 1, responsePartial: string]
| [isFormState: 2, formState: any]
| [isBinary: 3, responseBase64Partial: string]
): void {
if (seg[0] === 0) {
initialServerDataBuffer = []
@ -70,6 +71,22 @@ function nextServerDataCallback(
}
} else if (seg[0] === 2) {
initialFormStateData = seg[1]
} else if (seg[0] === 3) {
if (!initialServerDataBuffer)
throw new Error('Unexpected server data: missing bootstrap script.')
// Decode the base64 string back to binary data.
const binaryString = atob(seg[1])
const decodedChunk = new Uint8Array(binaryString.length)
for (var i = 0; i < binaryString.length; i++) {
decodedChunk[i] = binaryString.charCodeAt(i)
}
if (initialServerDataWriter) {
initialServerDataWriter.enqueue(decodedChunk)
} else {
initialServerDataBuffer.push(decodedChunk)
}
}
}
@ -84,7 +101,7 @@ function nextServerDataCallback(
function nextServerDataRegisterWriter(ctr: ReadableStreamDefaultController) {
if (initialServerDataBuffer) {
initialServerDataBuffer.forEach((val) => {
ctr.enqueue(encoder.encode(val))
ctr.enqueue(typeof val === 'string' ? encoder.encode(val) : val)
})
if (initialServerDataLoaded && !initialServerDataFlushed) {
ctr.close()

2
packages/next/src/server/app-render/app-render.tsx
View file

@ -398,7 +398,7 @@ function createFlightDataResolver(ctx: AppRenderContext) {
// Generate the flight data and as soon as it can, convert it into a string.
const promise = generateFlight(ctx)
.then(async (result) => ({
flightData: await result.toUnchunkedString(true),
flightData: await result.toUnchunkedBuffer(true),
}))
// Otherwise if it errored, return the error.
.catch((err) => ({ err }))

53
packages/next/src/server/app-render/use-flight-response.tsx
View file

@ -9,6 +9,7 @@ const isEdgeRuntime = process.env.NEXT_RUNTIME === 'edge'
const INLINE_FLIGHT_PAYLOAD_BOOTSTRAP = 0
const INLINE_FLIGHT_PAYLOAD_DATA = 1
const INLINE_FLIGHT_PAYLOAD_FORM_STATE = 2
const INLINE_FLIGHT_PAYLOAD_BINARY = 3
const flightResponses = new WeakMap<BinaryStreamOf<any>, Promise<any>>()
const encoder = new TextEncoder()
@ -96,10 +97,8 @@ export function createInlinedDataReadableStream(
? `<script nonce=${JSON.stringify(nonce)}>`
: '<script>'
const decoder = new TextDecoder('utf-8', { fatal: true })
const decoderOptions = { stream: true }
const flightReader = flightStream.getReader()
const decoder = new TextDecoder('utf-8', { fatal: true })
const readable = new ReadableStream({
type: 'bytes',
@ -114,15 +113,26 @@ export function createInlinedDataReadableStream(
async pull(controller) {
try {
const { done, value } = await flightReader.read()
if (done) {
const tail = decoder.decode(value, { stream: false })
if (tail.length) {
writeFlightDataInstruction(controller, startScriptTag, tail)
if (value) {
try {
const decodedString = decoder.decode(value, { stream: !done })
// The chunk cannot be decoded as valid UTF-8 string as it might
// have arbitrary binary data.
writeFlightDataInstruction(
controller,
startScriptTag,
decodedString
)
} catch {
// The chunk cannot be decoded as valid UTF-8 string.
writeFlightDataInstruction(controller, startScriptTag, value)
}
}
if (done) {
controller.close()
} else {
const chunkAsString = decoder.decode(value, decoderOptions)
writeFlightDataInstruction(controller, startScriptTag, chunkAsString)
}
} catch (error) {
// There was a problem in the upstream reader or during decoding or enqueuing
@ -154,13 +164,28 @@ function writeInitialInstructions(
function writeFlightDataInstruction(
controller: ReadableStreamDefaultController,
scriptStart: string,
chunkAsString: string
chunk: string | Uint8Array
) {
let htmlInlinedData: string
if (typeof chunk === 'string') {
htmlInlinedData = htmlEscapeJsonString(
JSON.stringify([INLINE_FLIGHT_PAYLOAD_DATA, chunk])
)
} else {
// The chunk cannot be embedded as a UTF-8 string in the script tag.
// Instead let's inline it in base64.
// Credits to Devon Govett (devongovett) for the technique.
// https://github.com/devongovett/rsc-html-stream
const base64 = btoa(String.fromCodePoint(...chunk))
htmlInlinedData = htmlEscapeJsonString(
JSON.stringify([INLINE_FLIGHT_PAYLOAD_BINARY, base64])
)
}
controller.enqueue(
encoder.encode(
`${scriptStart}self.__next_f.push(${htmlEscapeJsonString(
JSON.stringify([INLINE_FLIGHT_PAYLOAD_DATA, chunkAsString])
)})</script>`
`${scriptStart}self.__next_f.push(${htmlInlinedData})</script>`
)
)
}

73
packages/next/src/server/base-server.ts
View file

@ -13,6 +13,8 @@ import type { ParsedUrlQuery } from 'querystring'
import type { RenderOptsPartial as PagesRenderOptsPartial } from './render'
import type { RenderOptsPartial as AppRenderOptsPartial } from './app-render/types'
import type {
CachedAppPageValue,
CachedPageValue,
ResponseCacheBase,
ResponseCacheEntry,
ResponseGenerator,
@ -2543,16 +2545,29 @@ export default abstract class Server<
return null
}
if (isAppPath) {
return {
value: {
kind: 'APP_PAGE',
html: result,
headers,
rscData: metadata.flightData,
postponed: metadata.postponed,
status: res.statusCode,
} satisfies CachedAppPageValue,
revalidate: metadata.revalidate,
}
}
// We now have a valid HTML result that we can return to the user.
return {
value: {
kind: 'PAGE',
html: result,
pageData: metadata.pageData ?? metadata.flightData,
postponed: metadata.postponed,
pageData: metadata.pageData,
headers,
status: isAppPath ? res.statusCode : undefined,
},
status: res.statusCode,
} satisfies CachedPageValue,
revalidate: metadata.revalidate,
}
}
@ -2665,7 +2680,6 @@ export default abstract class Server<
value: {
kind: 'PAGE',
html: RenderResult.fromStatic(html),
postponed: undefined,
status: undefined,
headers: undefined,
pageData: {},
@ -2734,7 +2748,7 @@ export default abstract class Server<
}
const didPostpone =
cacheEntry.value?.kind === 'PAGE' &&
cacheEntry.value?.kind === 'APP_PAGE' &&
typeof cacheEntry.value.postponed === 'string'
if (
@ -2901,7 +2915,11 @@ export default abstract class Server<
} else if (isAppPath) {
// If the request has a postponed state and it's a resume request we
// should error.
if (cachedData.postponed && minimalPostponed) {
if (
cachedData.kind === 'APP_PAGE' &&
cachedData.postponed &&
minimalPostponed
) {
throw new Error(
'Invariant: postponed state should not be present on a resume request'
)
@ -2949,7 +2967,11 @@ export default abstract class Server<
}
// Mark that the request did postpone if this is a data request.
if (cachedData.postponed && isRSCRequest) {
if (
cachedData.kind === 'APP_PAGE' &&
cachedData.postponed &&
isRSCRequest
) {
res.setHeader(NEXT_DID_POSTPONE_HEADER, '1')
}
@ -2960,8 +2982,15 @@ export default abstract class Server<
if (isDataReq && !isPreviewMode) {
// If this is a dynamic RSC request, then stream the response.
if (isDynamicRSCRequest) {
if (cachedData.pageData) {
throw new Error('Invariant: Expected pageData to be undefined')
if (cachedData.kind !== 'APP_PAGE') {
console.error({ url: req.url, pathname }, cachedData)
throw new Error(
`Invariant: expected cache data kind of APP_PAGE got ${cachedData.kind}`
)
}
if (cachedData.rscData) {
throw new Error('Invariant: Expected rscData to be undefined')
}
if (cachedData.postponed) {
@ -2980,9 +3009,15 @@ export default abstract class Server<
}
}
if (typeof cachedData.pageData !== 'string') {
if (cachedData.kind !== 'APP_PAGE') {
throw new Error(
`Invariant: expected pageData to be a string, got ${typeof cachedData.pageData}`
`Invariant: expected cached data to be APP_PAGE got ${cachedData.kind}`
)
}
if (!Buffer.isBuffer(cachedData.rscData)) {
throw new Error(
`Invariant: expected rscData to be a Buffer, got ${typeof cachedData.rscData}`
)
}
@ -2990,7 +3025,7 @@ export default abstract class Server<
// data.
return {
type: 'rsc',
body: RenderResult.fromStatic(cachedData.pageData),
body: RenderResult.fromStatic(cachedData.rscData),
revalidate: cacheEntry.revalidate,
}
}
@ -3001,7 +3036,10 @@ export default abstract class Server<
// If there's no postponed state, we should just serve the HTML. This
// should also be the case for a resume request because it's completed
// as a server render (rather than a static render).
if (!cachedData.postponed || this.minimalMode) {
if (
!(cachedData.kind === 'APP_PAGE' && cachedData.postponed) ||
this.minimalMode
) {
return {
type: 'html',
body,
@ -3030,7 +3068,7 @@ export default abstract class Server<
throw new Error('Invariant: expected a result to be returned')
}
if (result.value?.kind !== 'PAGE') {
if (result.value?.kind !== 'APP_PAGE') {
throw new Error(
`Invariant: expected a page response, got ${result.value?.kind}`
)
@ -3056,6 +3094,11 @@ export default abstract class Server<
revalidate: 0,
}
} else if (isDataReq) {
if (cachedData.kind !== 'PAGE') {
throw new Error(
`Invariant: expected cached data to be PAGE got ${cachedData.kind}`
)
}
return {
type: 'json',
body: RenderResult.fromStatic(JSON.stringify(cachedData.pageData)),

5
packages/next/src/server/lib/incremental-cache/fetch-cache.ts
View file

@ -109,7 +109,10 @@ export default class FetchCache implements CacheHandler {
}
// rough estimate of size of cache value
return (
value.html.length + (JSON.stringify(value.pageData)?.length || 0)
value.html.length +
(JSON.stringify(
value.kind === 'APP_PAGE' ? value.rscData : value.pageData
)?.length || 0)
)
},
})

80
packages/next/src/server/lib/incremental-cache/file-system-cache.ts
View file

@ -77,7 +77,10 @@ export default class FileSystemCache implements CacheHandler {
}
// rough estimate of size of cache value
return (
value.html.length + (JSON.stringify(value.pageData)?.length || 0)
value.html.length +
(JSON.stringify(
value.kind === 'APP_PAGE' ? value.rscData : value.pageData
)?.length || 0)
)
},
})
@ -250,23 +253,6 @@ export default class FileSystemCache implements CacheHandler {
}
}
} else {
const pageData = isAppPath
? await this.fs.readFile(
this.getFilePath(
`${key}${
this.isAppPPREnabled ? RSC_PREFETCH_SUFFIX : RSC_SUFFIX
}`,
'app'
),
'utf8'
)
: JSON.parse(
await this.fs.readFile(
this.getFilePath(`${key}${NEXT_DATA_SUFFIX}`, 'pages'),
'utf8'
)
)
let meta: RouteMetadata | undefined
if (isAppPath) {
@ -280,16 +266,44 @@ export default class FileSystemCache implements CacheHandler {
} catch {}
}
data = {
lastModified: mtime.getTime(),
value: {
kind: 'PAGE',
html: fileData,
pageData,
postponed: meta?.postponed,
headers: meta?.headers,
status: meta?.status,
},
if (isAppPath) {
const rscData = await this.fs.readFile(
this.getFilePath(
`${key}${
this.isAppPPREnabled ? RSC_PREFETCH_SUFFIX : RSC_SUFFIX
}`,
'app'
)
)
data = {
lastModified: mtime.getTime(),
value: {
kind: 'APP_PAGE',
html: fileData,
rscData,
postponed: meta?.postponed,
headers: meta?.headers,
status: meta?.status,
},
}
} else {
const pageData = JSON.parse(
await this.fs.readFile(
this.getFilePath(`${key}${NEXT_DATA_SUFFIX}`, 'pages'),
'utf8'
)
)
data = {
lastModified: mtime.getTime(),
value: {
kind: 'PAGE',
html: fileData,
pageData,
headers: meta?.headers,
status: meta?.status,
},
}
}
}
@ -301,7 +315,7 @@ export default class FileSystemCache implements CacheHandler {
}
}
if (data?.value?.kind === 'PAGE') {
if (data?.value?.kind === 'APP_PAGE' || data?.value?.kind === 'PAGE') {
let cacheTags: undefined | string[]
const tagsHeader = data.value.headers?.[NEXT_CACHE_TAGS_HEADER]
@ -385,8 +399,8 @@ export default class FileSystemCache implements CacheHandler {
return
}
if (data?.kind === 'PAGE') {
const isAppPath = typeof data.pageData === 'string'
if (data?.kind === 'PAGE' || data?.kind === 'APP_PAGE') {
const isAppPath = 'rscData' in data
const htmlPath = this.getFilePath(
`${key}.html`,
isAppPath ? 'app' : 'pages'
@ -405,14 +419,14 @@ export default class FileSystemCache implements CacheHandler {
}`,
isAppPath ? 'app' : 'pages'
),
isAppPath ? data.pageData : JSON.stringify(data.pageData)
isAppPath ? data.rscData : JSON.stringify(data.pageData)
)
if (data.headers || data.status) {
const meta: RouteMetadata = {
headers: data.headers,
status: data.status,
postponed: data.postponed,
postponed: isAppPath ? data.postponed : undefined,
}
await this.fs.writeFile(

33
packages/next/src/server/render-result.ts
View file

@ -4,7 +4,9 @@ import type { FetchMetrics } from './base-http'
import {
chainStreams,
streamFromBuffer,
streamFromString,
streamToBuffer,
streamToString,
} from './stream-utils/node-web-streams-helper'
import { isAbortError, pipeToNodeResponse } from './pipe-readable'
@ -12,7 +14,7 @@ import { isAbortError, pipeToNodeResponse } from './pipe-readable'
type ContentTypeOption = string | undefined
export type AppPageRenderResultMetadata = {
flightData?: string
flightData?: Buffer
revalidate?: Revalidate
staticBailoutInfo?: {
stack?: string
@ -50,6 +52,7 @@ export type RenderResultResponse =
| ReadableStream<Uint8Array>[]
| ReadableStream<Uint8Array>
| string
| Buffer
| null
export type RenderResultOptions<
@ -89,7 +92,7 @@ export default class RenderResult<
* @param value the static response value
* @returns a new RenderResult instance
*/
public static fromStatic(value: string) {
public static fromStatic(value: string | Buffer) {
return new RenderResult<StaticRenderResultMetadata>(value, { metadata: {} })
}
@ -125,6 +128,26 @@ export default class RenderResult<
return typeof this.response !== 'string'
}
public toUnchunkedBuffer(stream?: false): Buffer
public toUnchunkedBuffer(stream: true): Promise<Buffer>
public toUnchunkedBuffer(stream = false): Promise<Buffer> | Buffer {
if (this.response === null) {
throw new Error('Invariant: null responses cannot be unchunked')
}
if (typeof this.response !== 'string') {
if (!stream) {
throw new Error(
'Invariant: dynamic responses cannot be unchunked. This is a bug in Next.js'
)
}
return streamToBuffer(this.readable)
}
return Buffer.from(this.response)
}
/**
* Returns the response if it is a string. If the page was dynamic, this will
* return a promise if the `stream` option is true, or it will throw an error.
@ -164,6 +187,10 @@ export default class RenderResult<
throw new Error('Invariant: static responses cannot be streamed')
}
if (Buffer.isBuffer(this.response)) {
return streamFromBuffer(this.response)
}
// If the response is an array of streams, then chain them together.
if (Array.isArray(this.response)) {
return chainStreams(...this.response)
@ -191,6 +218,8 @@ export default class RenderResult<
responses = [streamFromString(this.response)]
} else if (Array.isArray(this.response)) {
responses = this.response
} else if (Buffer.isBuffer(this.response)) {
responses = [streamFromBuffer(this.response)]
} else {
responses = [this.response]
}

30
packages/next/src/server/response-cache/types.ts
View file

@ -40,12 +40,22 @@ export interface CachedRedirectValue {
props: Object
}
interface CachedPageValue {
export interface CachedAppPageValue {
kind: 'APP_PAGE'
// this needs to be a RenderResult so since renderResponse
// expects that type instead of a string
html: RenderResult
rscData: Buffer | undefined
status: number | undefined
postponed: string | undefined
headers: OutgoingHttpHeaders | undefined
}
export interface CachedPageValue {
kind: 'PAGE'
// this needs to be a RenderResult so since renderResponse
// expects that type instead of a string
html: RenderResult
postponed: string | undefined
pageData: Object
status: number | undefined
headers: OutgoingHttpHeaders | undefined
@ -69,13 +79,23 @@ export interface CachedImageValue {
isStale?: boolean
}
interface IncrementalCachedPageValue {
export interface IncrementalCachedAppPageValue {
kind: 'APP_PAGE'
// this needs to be a string since the cache expects to store
// the string value
html: string
rscData: Buffer | undefined
headers: OutgoingHttpHeaders | undefined
postponed: string | undefined
status: number | undefined
}
export interface IncrementalCachedPageValue {
kind: 'PAGE'
// this needs to be a string since the cache expects to store
// the string value
html: string
pageData: Object
postponed: string | undefined
headers: OutgoingHttpHeaders | undefined
status: number | undefined
}
@ -92,6 +112,7 @@ export type IncrementalCacheEntry = {
export type IncrementalCacheValue =
| CachedRedirectValue
| IncrementalCachedPageValue
| IncrementalCachedAppPageValue
| CachedImageValue
| CachedFetchValue
| CachedRouteValue
@ -99,6 +120,7 @@ export type IncrementalCacheValue =
export type ResponseCacheValue =
| CachedRedirectValue
| CachedPageValue
| CachedAppPageValue
| CachedImageValue
| CachedRouteValue

24
packages/next/src/server/response-cache/utils.ts
View file

@ -12,12 +12,20 @@ export async function fromResponseCacheEntry(
? {
kind: 'PAGE',
html: await cacheEntry.value.html.toUnchunkedString(true),
postponed: cacheEntry.value.postponed,
pageData: cacheEntry.value.pageData,
headers: cacheEntry.value.headers,
status: cacheEntry.value.status,
}
: cacheEntry.value,
: cacheEntry.value?.kind === 'APP_PAGE'
? {
kind: 'APP_PAGE',
html: await cacheEntry.value.html.toUnchunkedString(true),
postponed: cacheEntry.value.postponed,
rscData: cacheEntry.value.rscData,
headers: cacheEntry.value.headers,
status: cacheEntry.value.status,
}
: cacheEntry.value,
}
}
@ -42,10 +50,18 @@ export async function toResponseCacheEntry(
kind: 'PAGE',
html: RenderResult.fromStatic(response.value.html),
pageData: response.value.pageData,
postponed: response.value.postponed,
headers: response.value.headers,
status: response.value.status,
}
: response.value,
: response.value?.kind === 'APP_PAGE'
? {
kind: 'APP_PAGE',
html: RenderResult.fromStatic(response.value.html),
rscData: response.value.rscData,
headers: response.value.headers,
status: response.value.status,
postponed: response.value.postponed,
}
: response.value,
}
}

22
packages/next/src/server/stream-utils/node-web-streams-helper.ts
View file

@ -73,6 +73,28 @@ export function streamFromString(str: string): ReadableStream<Uint8Array> {
})
}
export function streamFromBuffer(chunk: Buffer): ReadableStream<Uint8Array> {
return new ReadableStream({
start(controller) {
controller.enqueue(chunk)
controller.close()
},
})
}
export async function streamToBuffer(
stream: ReadableStream<Uint8Array>
): Promise<Buffer> {
const buffers: Buffer[] = []
// @ts-expect-error TypeScript gets this wrong (https://nodejs.org/api/webstreams.html#async-iteration)
for await (const chunk of stream) {
buffers.push(chunk)
}
return Buffer.concat(buffers)
}
export async function streamToString(
stream: ReadableStream<Uint8Array>
): Promise<string> {

19
test/e2e/app-dir/binary/app/client.js Normal file
View file

@ -0,0 +1,19 @@
'use client'
import { useEffect, useState } from 'react'
export function Client({ binary, arbitrary }) {
const [hydrated, setHydrated] = useState(false)
useEffect(() => {
setHydrated(true)
}, [])
return (
<>
<div>utf8 binary: {new TextDecoder().decode(binary)}</div>
<div>arbitrary binary: {String(arbitrary)}</div>
<div>hydrated: {String(hydrated)}</div>
</>
)
}

12
test/e2e/app-dir/binary/app/layout.js Normal file
View file

@ -0,0 +1,12 @@
export const metadata = {
title: 'Next.js',
description: 'Generated by Next.js',
}
export default function RootLayout({ children }) {
return (
<html lang="en">
<body>{children}</body>
</html>
)
}

8
test/e2e/app-dir/binary/app/page.js Normal file
View file

@ -0,0 +1,8 @@
import { Client } from './client'
export default function Page() {
const binaryData = new Uint8Array([104, 101, 108, 108, 111])
const nonUtf8BinaryData = new Uint8Array([0xff, 0, 1, 2, 3])
return <Client binary={binaryData} arbitrary={nonUtf8BinaryData} />
}

6
test/e2e/app-dir/binary/next.config.js Normal file
View file

@ -0,0 +1,6 @@
module.exports = {
experimental: {
// This ensures that we're running the experimental React.
taint: true,
},
}

32
test/e2e/app-dir/binary/rsc-binary.test.ts Normal file
View file

@ -0,0 +1,32 @@
import { nextTestSetup } from 'e2e-utils'
import { check } from 'next-test-utils'
describe('RSC binary serialization', () => {
const { next, skipped } = nextTestSetup({
files: __dirname,
skipDeployment: true,
dependencies: {
react: '19.0.0-beta-4508873393-20240430',
'react-dom': '19.0.0-beta-4508873393-20240430',
'server-only': 'latest',
},
})
if (skipped) return
afterEach(async () => {
await next.stop()
})
it('should correctly encode/decode binaries and hydrate', async function () {
const browser = await next.browser('/')
await check(async () => {
const content = await browser.elementByCss('body').text()
return content.includes('utf8 binary: hello') &&
content.includes('arbitrary binary: 255,0,1,2,3') &&
content.includes('hydrated: true')
? 'success'
: 'fail'
}, 'success')
})
})