Fix running server with Polyfilled fetch (#32368)

**Note**: This PR is applying again changes landed #31935 that were reverted from an investigation.

This PR fixes #30398

By default Next will polyfill some fetch APIs (Request, Response, Header and fetch) only if fetch is not found in the global scope in certain entry points. If we have a custom server which is adding a global fetch (and only fetch) at the very top then the rest of APIs will not be polyfilled.

This PR adds a test on the custom server where we can add a custom polyfill for fetch with an env variable. This reproduces the issue since next-server.js will be required without having a polyfill for Response which makes it fail on requiring NextResponse. Then we remove the code that checks for subrequests to happen within the **sandbox** so that we don't need to polyfill `next-server` anymore.

The we also introduce an improvement on how we handle relative requests. Since #31858 introduced a `port` and `hostname` options for the server, we can always pass absolute URLs to the Middleware so we can always use the original `nextUrl` to pass it to fetch. This brings a lot of simplification for `NextURL` since we don't have to consider relative URLs no more.

## Bug

- [x] Related issues linked using `fixes #number`
- [x] Integration tests added
- [x] Errors have helpful link attached, see `contributing.md`
This commit is contained in:
Javi Velasco 2021-12-13 19:30:24 +01:00 committed by GitHub
parent c658fd3276
commit 59f7676966
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 286 additions and 164 deletions

View file

@ -91,10 +91,10 @@ import { parseNextUrl } from '../shared/lib/router/utils/parse-next-url'
import isError from '../lib/is-error'
import { getMiddlewareInfo } from './require'
import { MIDDLEWARE_ROUTE } from '../lib/constants'
import { NextResponse } from './web/spec-extension/response'
import { run } from './web/sandbox'
import { addRequestMeta, getRequestMeta } from './request-meta'
import { toNodeHeaders } from './web/utils'
import { relativizeURL } from '../shared/lib/router/utils/relativize-url'
const getCustomRouteMatcher = pathMatch(true)
@ -379,7 +379,13 @@ export default abstract class Server {
parsedUrl.query = parseQs(parsedUrl.query)
}
addRequestMeta(req, '__NEXT_INIT_URL', req.url)
// When there are hostname and port we build an absolute URL
const initUrl =
this.hostname && this.port
? `http://${this.hostname}:${this.port}${req.url}`
: req.url
addRequestMeta(req, '__NEXT_INIT_URL', initUrl)
addRequestMeta(req, '__NEXT_INIT_QUERY', { ...parsedUrl.query })
const url = parseNextUrl({
@ -673,6 +679,14 @@ export default abstract class Server {
}): Promise<FetchEventResult | null> {
this.middlewareBetaWarning()
// For middleware to "fetch" we must always provide an absolute URL
const url = getRequestMeta(params.request, '__NEXT_INIT_URL')!
if (!url.startsWith('http')) {
throw new Error(
'To use middleware you must provide a `hostname` and `port` to the Next.js Server'
)
}
const page: { name?: string; params?: { [key: string]: string } } = {}
if (await this.hasPage(params.parsedUrl.pathname)) {
page.name = params.parsedUrl.pathname
@ -687,8 +701,6 @@ export default abstract class Server {
}
}
const subreq = params.request.headers[`x-middleware-subrequest`]
const subrequests = typeof subreq === 'string' ? subreq.split(':') : []
const allHeaders = new Headers()
let result: FetchEventResult | null = null
@ -708,14 +720,6 @@ export default abstract class Server {
serverless: this._isLikeServerless,
})
if (subrequests.includes(middlewareInfo.name)) {
result = {
response: NextResponse.next(),
waitUntil: Promise.resolve(),
}
continue
}
result = await run({
name: middlewareInfo.name,
paths: middlewareInfo.paths,
@ -727,7 +731,7 @@ export default abstract class Server {
i18n: this.nextConfig.i18n,
trailingSlash: this.nextConfig.trailingSlash,
},
url: getRequestMeta(params.request, '__NEXT_INIT_URL')!,
url: url,
page: page,
},
useCache: !this.nextConfig.experimental.concurrentFeatures,
@ -1185,9 +1189,13 @@ export default abstract class Server {
type: 'route',
name: 'middleware catchall',
fn: async (req, res, _params, parsed) => {
const fullUrl = getRequestMeta(req, '__NEXT_INIT_URL')
if (!this.middleware?.length) {
return { finished: false }
}
const initUrl = getRequestMeta(req, '__NEXT_INIT_URL')!
const parsedUrl = parseNextUrl({
url: fullUrl,
url: initUrl,
headers: req.headers,
nextConfig: {
basePath: this.nextConfig.basePath,
@ -1226,6 +1234,18 @@ export default abstract class Server {
return { finished: true }
}
if (result.response.headers.has('x-middleware-rewrite')) {
const value = result.response.headers.get('x-middleware-rewrite')!
const rel = relativizeURL(value, initUrl)
result.response.headers.set('x-middleware-rewrite', rel)
}
if (result.response.headers.has('Location')) {
const value = result.response.headers.get('Location')!
const rel = relativizeURL(value, initUrl)
result.response.headers.set('Location', rel)
}
if (
!result.response.headers.has('x-middleware-rewrite') &&
!result.response.headers.has('x-middleware-next') &&

View file

@ -1,8 +1,9 @@
import type { NextMiddleware, RequestData, FetchEventResult } from './types'
import type { RequestInit } from './spec-extension/request'
import { DeprecationError } from './error'
import { fromNodeHeaders } from './utils'
import { NextFetchEvent } from './spec-extension/fetch-event'
import { NextRequest, RequestInit } from './spec-extension/request'
import { NextRequest } from './spec-extension/request'
import { NextResponse } from './spec-extension/response'
import { waitUntilSymbol } from './spec-compliant/fetch-event'
@ -11,13 +12,9 @@ export async function adapter(params: {
page: string
request: RequestData
}): Promise<FetchEventResult> {
const url = params.request.url.startsWith('/')
? `https://${params.request.headers.host}${params.request.url}`
: params.request.url
const request = new NextRequestHint({
page: params.page,
input: url,
input: params.request.url,
init: {
geo: params.request.geo,
headers: fromNodeHeaders(params.request.headers),

View file

@ -1,66 +1,78 @@
import type { PathLocale } from '../../shared/lib/i18n/normalize-locale-path'
import type { DomainLocale, I18NConfig } from '../config-shared'
import { getLocaleMetadata } from '../../shared/lib/i18n/get-locale-metadata'
import cookie from 'next/dist/compiled/cookie'
import { replaceBasePath } from '../router'
/**
* TODO
*
* - Add comments to the URLNext API.
* - Move internals to be using symbols for its shape.
* - Make sure logging does not show any implementation details.
* - Include in the event payload the nextJS configuration
*/
import cookie from 'next/dist/compiled/cookie'
interface Options {
base?: string | URL
basePath?: string
headers?: { [key: string]: string | string[] | undefined }
i18n?: I18NConfig | null
trailingSlash?: boolean
}
const REGEX_LOCALHOST_HOSTNAME =
/(?!^https?:\/\/)(127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|::1)/
const Internal = Symbol('NextURLInternal')
export class NextURL extends URL {
private _basePath: string
private _locale?: {
defaultLocale: string
domain?: DomainLocale
locale: string
path: PathLocale
redirect?: string
trailingSlash?: boolean
export class NextURL {
[Internal]: {
url: URL
options: Options
basePath: string
locale?: {
defaultLocale: string
domain?: DomainLocale
locale: string
path: PathLocale
redirect?: string
trailingSlash?: boolean
}
}
private _options: Options
private _url: URL
constructor(input: string, options: Options = {}) {
const url = createWHATWGURL(input)
super(url)
this._options = options
this._basePath = ''
this._url = url
constructor(input: string | URL, base?: string | URL, opts?: Options)
constructor(input: string | URL, opts?: Options)
constructor(
input: string | URL,
baseOrOpts?: string | URL | Options,
opts?: Options
) {
let base: undefined | string | URL
let options: Options
if (
(typeof baseOrOpts === 'object' && 'pathname' in baseOrOpts) ||
typeof baseOrOpts === 'string'
) {
base = baseOrOpts
options = opts || {}
} else {
options = opts || baseOrOpts || {}
}
this[Internal] = {
url: parseURL(input, base ?? options.base),
options: options,
basePath: '',
}
this.analyzeUrl()
}
get absolute() {
return this._url.hostname !== 'localhost'
}
private analyzeUrl() {
const { headers = {}, basePath, i18n } = this[Internal].options
analyzeUrl() {
const { headers = {}, basePath, i18n } = this._options
if (basePath && this._url.pathname.startsWith(basePath)) {
this._url.pathname = replaceBasePath(this._url.pathname, basePath)
this._basePath = basePath
if (basePath && this[Internal].url.pathname.startsWith(basePath)) {
this[Internal].url.pathname = replaceBasePath(
this[Internal].url.pathname,
basePath
)
this[Internal].basePath = basePath
} else {
this._basePath = ''
this[Internal].basePath = ''
}
if (i18n) {
this._locale = getLocaleMetadata({
this[Internal].locale = getLocaleMetadata({
cookies: () => {
const value = headers['cookie']
return value
@ -73,154 +85,156 @@ export class NextURL extends URL {
i18n: i18n,
},
url: {
hostname: this._url.hostname || null,
pathname: this._url.pathname,
hostname: this[Internal].url.hostname || null,
pathname: this[Internal].url.pathname,
},
})
if (this._locale?.path.detectedLocale) {
this._url.pathname = this._locale.path.pathname
if (this[Internal].locale?.path.detectedLocale) {
this[Internal].url.pathname = this[Internal].locale!.path.pathname
}
}
}
formatPathname() {
const { i18n } = this._options
let pathname = this._url.pathname
private formatPathname() {
const { i18n } = this[Internal].options
let pathname = this[Internal].url.pathname
if (this._locale?.locale && i18n?.defaultLocale !== this._locale?.locale) {
pathname = `/${this._locale?.locale}${pathname}`
if (
this[Internal].locale?.locale &&
i18n?.defaultLocale !== this[Internal].locale?.locale
) {
pathname = `/${this[Internal].locale?.locale}${pathname}`
}
if (this._basePath) {
pathname = `${this._basePath}${pathname}`
if (this[Internal].basePath) {
pathname = `${this[Internal].basePath}${pathname}`
}
return pathname
}
get locale() {
if (!this._locale) {
throw new TypeError(`The URL is not configured with i18n`)
}
return this._locale.locale
public get locale() {
return this[Internal].locale?.locale ?? ''
}
set locale(locale: string) {
if (!this._locale) {
throw new TypeError(`The URL is not configured with i18n`)
public set locale(locale: string) {
if (
!this[Internal].locale ||
!this[Internal].options.i18n?.locales.includes(locale)
) {
throw new TypeError(
`The NextURL configuration includes no locale "${locale}"`
)
}
this._locale.locale = locale
this[Internal].locale!.locale = locale
}
get defaultLocale() {
return this._locale?.defaultLocale
return this[Internal].locale?.defaultLocale
}
get domainLocale() {
return this._locale?.domain
return this[Internal].locale?.domain
}
get searchParams() {
return this._url.searchParams
return this[Internal].url.searchParams
}
get host() {
return this.absolute ? this._url.host : ''
return this[Internal].url.host
}
set host(value: string) {
this._url.host = value
this[Internal].url.host = value
}
get hostname() {
return this.absolute ? this._url.hostname : ''
return this[Internal].url.hostname
}
set hostname(value: string) {
this._url.hostname = value || 'localhost'
this[Internal].url.hostname = value
}
get port() {
return this.absolute ? this._url.port : ''
return this[Internal].url.port
}
set port(value: string) {
this._url.port = value
this[Internal].url.port = value
}
get protocol() {
return this.absolute ? this._url.protocol : ''
return this[Internal].url.protocol
}
set protocol(value: string) {
this._url.protocol = value
this[Internal].url.protocol = value
}
get href() {
const pathname = this.formatPathname()
return this.absolute
? `${this.protocol}//${this.host}${pathname}${this._url.search}`
: `${pathname}${this._url.search}`
return `${this.protocol}//${this.host}${pathname}${this[Internal].url.search}`
}
set href(url: string) {
this._url = createWHATWGURL(url)
this[Internal].url = parseURL(url)
this.analyzeUrl()
}
get origin() {
return this.absolute ? this._url.origin : ''
return this[Internal].url.origin
}
get pathname() {
return this._url.pathname
return this[Internal].url.pathname
}
set pathname(value: string) {
this._url.pathname = value
this[Internal].url.pathname = value
}
get hash() {
return this._url.hash
return this[Internal].url.hash
}
set hash(value: string) {
this._url.hash = value
this[Internal].url.hash = value
}
get search() {
return this._url.search
return this[Internal].url.search
}
set search(value: string) {
this._url.search = value
this[Internal].url.search = value
}
get password() {
return this._url.password
return this[Internal].url.password
}
set password(value: string) {
this._url.password = value
this[Internal].url.password = value
}
get username() {
return this._url.username
return this[Internal].url.username
}
set username(value: string) {
this._url.username = value
this[Internal].url.username = value
}
get basePath() {
return this._basePath
return this[Internal].basePath
}
set basePath(value: string) {
this._basePath = value.startsWith('/') ? value : `/${value}`
this[Internal].basePath = value.startsWith('/') ? value : `/${value}`
}
toString() {
@ -232,13 +246,12 @@ export class NextURL extends URL {
}
}
function createWHATWGURL(url: string) {
url = url.replace(REGEX_LOCALHOST_HOSTNAME, 'localhost')
return isRelativeURL(url)
? new URL(url.replace(/^\/+/, '/'), new URL('https://localhost'))
: new URL(url)
}
const REGEX_LOCALHOST_HOSTNAME =
/(?!^https?:\/\/)(127(?:\.(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)){3}|::1|localhost)/
function isRelativeURL(url: string) {
return url.startsWith('/')
function parseURL(url: string | URL, base?: string | URL) {
return new URL(
String(url).replace(REGEX_LOCALHOST_HOSTNAME, 'localhost'),
base && String(base).replace(REGEX_LOCALHOST_HOSTNAME, 'localhost')
)
}

View file

@ -18,6 +18,19 @@ export async function run(params: {
runInContext(paramPath)
}
const subreq = params.request.headers[`x-middleware-subrequest`]
const subrequests = typeof subreq === 'string' ? subreq.split(':') : []
if (subrequests.includes(params.name)) {
return {
waitUntil: Promise.resolve(),
response: new context.Response(null, {
headers: {
'x-middleware-next': '1',
},
}),
}
}
return context._ENTRIES[`middleware_${params.name}`].default({
request: params.request,
})

View file

@ -0,0 +1,13 @@
/**
* Given a URL as a string and a base URL it will make the URL relative
* if the parsed protocol and host is the same as the one in the base
* URL. Otherwise it returns the same URL string.
*/
export function relativizeURL(url: string | string, base: string | URL) {
const baseURL = typeof base === 'string' ? new URL(base) : base
const relative = new URL(url, base)
const origin = `${baseURL.protocol}//${baseURL.host}`
return `${relative.protocol}//${relative.host}` === origin
? relative.toString().replace(origin, '')
: relative.toString()
}

View file

@ -1,3 +1,7 @@
if (process.env.POLYFILL_FETCH) {
global.fetch = require('node-fetch').default
}
const http = require('http')
const next = require('next')

View file

@ -195,4 +195,14 @@ describe('Custom Server', () => {
}
)
})
describe('with a custom fetch polyfill', () => {
beforeAll(() => startServer({ POLYFILL_FETCH: 'true' }))
afterAll(() => killApp(server))
it('should serve internal file from render', async () => {
const data = await renderViaHTTP(appPort, '/static/hello.txt')
expect(data).toMatch(/hello world/)
})
})
})

View file

@ -0,0 +1,7 @@
import { NextResponse } from 'next/server'
export function middleware() {
const response = NextResponse.next()
response.headers.set('x-dynamic-path', 'true')
return response
}

View file

@ -54,9 +54,7 @@ export async function middleware(request) {
}
if (url.pathname.endsWith('/root-subrequest')) {
return fetch(
`http://${request.headers.get('host')}/interface/root-subrequest`
)
return fetch(url)
}
return new Response(null, {

View file

@ -57,11 +57,11 @@ export async function middleware(request, ev) {
ev.waitUntil(
(async () => {
writer.write(encoder.encode('this is a streamed '.repeat(10)))
await sleep(2000)
await sleep(200)
writer.write(encoder.encode('after 2 seconds '.repeat(10)))
await sleep(2000)
await sleep(200)
writer.write(encoder.encode('after 4 seconds '.repeat(10)))
await sleep(2000)
await sleep(200)
writer.close()
})()
)

View file

@ -447,6 +447,14 @@ function interfaceTests(locale = '') {
const element = await browser.elementByCss('.title')
expect(await element.text()).toEqual('Dynamic route')
})
it(`${locale} allows subrequests without infinite loops`, async () => {
const res = await fetchViaHTTP(
context.appPort,
`/interface/root-subrequest`
)
expect(res.headers.get('x-dynamic-path')).toBe('true')
})
}
function getCookieFromResponse(res, cookieName) {

View file

@ -1,74 +1,70 @@
/* eslint-env jest */
import { NextURL } from 'next/dist/server/web/next-url'
it('has the right shape', () => {
const parsed = new NextURL('/about?param1=value1')
// TODO Make NextURL extend URL
it.skip('has the right shape and prototype', () => {
const parsed = new NextURL('/about?param1=value1', 'http://127.0.0.1')
expect(parsed).toBeInstanceOf(URL)
})
it('allows to format relative urls', async () => {
const parsed = new NextURL('/about?param1=value1')
it('allows to the pathname', async () => {
const parsed = new NextURL('/about?param1=value1', 'http://127.0.0.1:3000')
expect(parsed.basePath).toEqual('')
expect(parsed.hostname).toEqual('')
expect(parsed.host).toEqual('')
expect(parsed.href).toEqual('/about?param1=value1')
expect(parsed.hostname).toEqual('localhost')
expect(parsed.host).toEqual('localhost:3000')
expect(parsed.href).toEqual('http://localhost:3000/about?param1=value1')
parsed.pathname = '/hihi'
expect(parsed.href).toEqual('/hihi?param1=value1')
expect(parsed.href).toEqual('http://localhost:3000/hihi?param1=value1')
})
it('allows to change the host of a relative url', () => {
const parsed = new NextURL('/about?param1=value1')
expect(parsed.hostname).toEqual('')
expect(parsed.host).toEqual('')
expect(parsed.href).toEqual('/about?param1=value1')
it('allows to change the host', () => {
const parsed = new NextURL('/about?param1=value1', 'http://127.0.0.1')
expect(parsed.hostname).toEqual('localhost')
expect(parsed.host).toEqual('localhost')
expect(parsed.href).toEqual('http://localhost/about?param1=value1')
parsed.hostname = 'foo.com'
expect(parsed.hostname).toEqual('foo.com')
expect(parsed.host).toEqual('foo.com')
expect(parsed.href).toEqual('https://foo.com/about?param1=value1')
expect(parsed.toString()).toEqual('https://foo.com/about?param1=value1')
expect(parsed.href).toEqual('http://foo.com/about?param1=value1')
expect(parsed.toString()).toEqual('http://foo.com/about?param1=value1')
})
it('allows to change the hostname of a relative url', () => {
const url = new NextURL('/example')
url.hostname = 'foo.com'
it('does noop changing to an invalid hostname', () => {
const url = new NextURL('https://foo.com/example')
url.hostname = ''
expect(url.toString()).toEqual('https://foo.com/example')
})
it('allows to remove the hostname of an absolute url', () => {
const url = new NextURL('https://foo.com/example')
url.hostname = ''
expect(url.toString()).toEqual('/example')
})
it('allows to change the whole href of an absolute url', () => {
it('allows to change the whole href', () => {
const url = new NextURL('https://localhost.com/foo')
expect(url.hostname).toEqual('localhost.com')
expect(url.protocol).toEqual('https:')
expect(url.host).toEqual('localhost.com')
url.href = '/foo'
expect(url.hostname).toEqual('')
expect(url.protocol).toEqual('')
expect(url.host).toEqual('')
url.href = 'http://foo.com/bar'
expect(url.hostname).toEqual('foo.com')
expect(url.protocol).toEqual('http:')
expect(url.host).toEqual('foo.com')
})
it('allows to update search params', () => {
const url = new NextURL('/example')
const url = new NextURL('/example', 'http://localhost.com')
url.searchParams.set('foo', 'bar')
expect(url.search).toEqual('?foo=bar')
expect(url.toString()).toEqual('/example?foo=bar')
expect(url.toString()).toEqual('http://localhost.com/example?foo=bar')
})
it('parses and formats the basePath', () => {
const url = new NextURL('/root/example', {
base: 'http://127.0.0.1',
basePath: '/root',
})
expect(url.basePath).toEqual('/root')
expect(url.pathname).toEqual('/example')
expect(url.toString()).toEqual('/root/example')
expect(url.toString()).toEqual('http://localhost/root/example')
const url2 = new NextURL('https://foo.com/root/bar', {
basePath: '/root',
@ -89,14 +85,47 @@ it('parses and formats the basePath', () => {
expect(url3.basePath).toEqual('')
url3.href = '/root/example'
expect(url.basePath).toEqual('/root')
expect(url.pathname).toEqual('/example')
expect(url.toString()).toEqual('/root/example')
url3.href = 'http://localhost.com/root/example'
expect(url3.basePath).toEqual('/root')
expect(url3.pathname).toEqual('/example')
expect(url3.toString()).toEqual('http://localhost.com/root/example')
})
it('allows to get empty locale when there is no locale', () => {
const url = new NextURL('https://localhost:3000/foo')
expect(url.locale).toEqual('')
})
it('doesnt allow to set an unexisting locale', () => {
const url = new NextURL('https://localhost:3000/foo')
let error: Error | null = null
try {
url.locale = 'foo'
} catch (err) {
error = err
}
expect(error).toBeInstanceOf(TypeError)
expect(error.message).toEqual(
'The NextURL configuration includes no locale "foo"'
)
})
it('always get a default locale', () => {
const url = new NextURL('/bar', {
base: 'http://127.0.0.1',
i18n: {
defaultLocale: 'en',
locales: ['en', 'es', 'fr'],
},
})
expect(url.locale).toEqual('en')
})
it('parses and formats the default locale', () => {
const url = new NextURL('/es/bar', {
base: 'http://127.0.0.1',
basePath: '/root',
i18n: {
defaultLocale: 'en',
@ -105,19 +134,19 @@ it('parses and formats the default locale', () => {
})
expect(url.locale).toEqual('es')
expect(url.toString()).toEqual('/es/bar')
expect(url.toString()).toEqual('http://localhost/es/bar')
url.basePath = '/root'
expect(url.locale).toEqual('es')
expect(url.toString()).toEqual('/root/es/bar')
expect(url.toString()).toEqual('http://localhost/root/es/bar')
url.locale = 'en'
expect(url.locale).toEqual('en')
expect(url.toString()).toEqual('/root/bar')
expect(url.toString()).toEqual('http://localhost/root/bar')
url.locale = 'fr'
expect(url.locale).toEqual('fr')
expect(url.toString()).toEqual('/root/fr/bar')
expect(url.toString()).toEqual('http://localhost/root/fr/bar')
})
it('consider 127.0.0.1 and variations as localhost', () => {
@ -131,3 +160,13 @@ it('consider 127.0.0.1 and variations as localhost', () => {
expect(new NextURL('https://127.0.1.0:3000/hello')).toStrictEqual(httpsUrl)
expect(new NextURL('https://::1:3000/hello')).toStrictEqual(httpsUrl)
})
it('allows to change the port', () => {
const url = new NextURL('https://localhost:3000/foo')
url.port = '3001'
expect(url.href).toEqual('https://localhost:3001/foo')
url.port = '80'
expect(url.href).toEqual('https://localhost:80/foo')
url.port = ''
expect(url.href).toEqual('https://localhost/foo')
})