feat(next-swc): try to fallback native bindings with MODULE_NOT_FOUND (#52667)
### What? closes WEB-1287. This PR is a stopgap workaround for https://github.com/npm/cli/issues/4828. There is ongoing discussion & RFC, but it is unclear when we can have those. Until then, PR tries to attempt to load native bindings by manually downloading binaries if original attempt fails with MODULE_NOT_FOUND. The implementation basically reuses most piece of existing wasm fallback; differences are it tries to all possible triples instead, and also try only for MODULE_NOT_FOUND. Other errors are treated as legit error from installed binary, do not attempt to re-download.
This commit is contained in:
parent
81dd7f8077
commit
7ce663ed52
3 changed files with 235 additions and 147 deletions
|
@ -7,7 +7,7 @@ import * as Log from '../output/log'
|
|||
import { getParserOptions } from './options'
|
||||
import { eventSwcLoadFailure } from '../../telemetry/events/swc-load-failure'
|
||||
import { patchIncorrectLockfile } from '../../lib/patch-incorrect-lockfile'
|
||||
import { downloadWasmSwc } from '../../lib/download-wasm-swc'
|
||||
import { downloadWasmSwc, downloadNativeNextSwc } from '../../lib/download-swc'
|
||||
import { spawn } from 'child_process'
|
||||
import { NextConfigComplete, TurboLoaderItem } from '../../server/config-shared'
|
||||
import { isDeepStrictEqual } from 'util'
|
||||
|
@ -88,6 +88,8 @@ let pendingBindings: any
|
|||
let swcTraceFlushGuard: any
|
||||
let swcHeapProfilerFlushGuard: any
|
||||
let swcCrashReporterFlushGuard: any
|
||||
let downloadNativeBindingsPromise: Promise<void> | undefined = undefined
|
||||
|
||||
export const lockfilePatchPromise: { cur?: Promise<void> } = {}
|
||||
|
||||
export interface Binding {
|
||||
|
@ -155,9 +157,31 @@ export async function loadBindings(): Promise<Binding> {
|
|||
}
|
||||
}
|
||||
|
||||
// Trickle down loading `fallback` bindings:
|
||||
//
|
||||
// - First, try to load native bindings installed in node_modules.
|
||||
// - If that fails with `ERR_MODULE_NOT_FOUND`, treat it as case of https://github.com/npm/cli/issues/4828
|
||||
// that host system where generated package lock is not matching to the guest system running on, try to manually
|
||||
// download corresponding target triple and load it. This won't be triggered if native bindings are failed to load
|
||||
// with other reasons than `ERR_MODULE_NOT_FOUND`.
|
||||
// - Lastly, falls back to wasm binding where possible.
|
||||
try {
|
||||
return resolve(loadNative(isCustomTurbopack))
|
||||
} catch (a) {
|
||||
if (
|
||||
Array.isArray(a) &&
|
||||
a.every((m) => m.includes('it was not installed'))
|
||||
) {
|
||||
let fallbackBindings = await tryLoadNativeWithFallback(
|
||||
attempts,
|
||||
isCustomTurbopack
|
||||
)
|
||||
|
||||
if (fallbackBindings) {
|
||||
return resolve(fallbackBindings)
|
||||
}
|
||||
}
|
||||
|
||||
attempts = attempts.concat(a)
|
||||
}
|
||||
|
||||
|
@ -177,6 +201,33 @@ export async function loadBindings(): Promise<Binding> {
|
|||
return pendingBindings
|
||||
}
|
||||
|
||||
async function tryLoadNativeWithFallback(
|
||||
attempts: Array<string>,
|
||||
isCustomTurbopack: boolean
|
||||
) {
|
||||
const nativeBindingsDirectory = path.join(
|
||||
path.dirname(require.resolve('next/package.json')),
|
||||
'next-swc-fallback'
|
||||
)
|
||||
|
||||
if (!downloadNativeBindingsPromise) {
|
||||
downloadNativeBindingsPromise = downloadNativeNextSwc(
|
||||
nextVersion,
|
||||
nativeBindingsDirectory,
|
||||
triples.map((triple: any) => triple.platformArchABI)
|
||||
)
|
||||
}
|
||||
await downloadNativeBindingsPromise
|
||||
|
||||
try {
|
||||
let bindings = loadNative(isCustomTurbopack, nativeBindingsDirectory)
|
||||
return bindings
|
||||
} catch (a: any) {
|
||||
attempts.concat(a)
|
||||
}
|
||||
return undefined
|
||||
}
|
||||
|
||||
async function tryLoadWasmWithFallback(
|
||||
attempts: any,
|
||||
isCustomTurbopack: boolean
|
||||
|
@ -776,7 +827,7 @@ async function loadWasm(importPath = '', isCustomTurbopack: boolean) {
|
|||
throw attempts
|
||||
}
|
||||
|
||||
function loadNative(isCustomTurbopack = false) {
|
||||
function loadNative(isCustomTurbopack = false, importPath?: string) {
|
||||
if (nativeBindings) {
|
||||
return nativeBindings
|
||||
}
|
||||
|
@ -794,10 +845,18 @@ function loadNative(isCustomTurbopack = false) {
|
|||
|
||||
if (!bindings) {
|
||||
for (const triple of triples) {
|
||||
let pkg = `@next/swc-${triple.platformArchABI}`
|
||||
let pkg = importPath
|
||||
? path.join(
|
||||
importPath,
|
||||
`@next/swc-${triple.platformArchABI}`,
|
||||
`next-swc.${triple.platformArchABI}.node`
|
||||
)
|
||||
: `@next/swc-${triple.platformArchABI}`
|
||||
try {
|
||||
bindings = require(pkg)
|
||||
checkVersionMismatch(require(`${pkg}/package.json`))
|
||||
if (!importPath) {
|
||||
checkVersionMismatch(require(`${pkg}/package.json`))
|
||||
}
|
||||
break
|
||||
} catch (e: any) {
|
||||
if (e?.code === 'MODULE_NOT_FOUND') {
|
||||
|
|
172
packages/next/src/lib/download-swc.ts
Normal file
172
packages/next/src/lib/download-swc.ts
Normal file
|
@ -0,0 +1,172 @@
|
|||
import os from 'os'
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import * as Log from '../build/output/log'
|
||||
import tar from 'next/dist/compiled/tar'
|
||||
const { fetch } = require('next/dist/compiled/undici') as {
|
||||
fetch: typeof global.fetch
|
||||
}
|
||||
const { WritableStream } = require('node:stream/web') as {
|
||||
WritableStream: typeof global.WritableStream
|
||||
}
|
||||
import { fileExists } from './file-exists'
|
||||
import { getRegistry } from './helpers/get-registry'
|
||||
|
||||
const MAX_VERSIONS_TO_CACHE = 8
|
||||
|
||||
// get platform specific cache directory adapted from playwright's handling
|
||||
// https://github.com/microsoft/playwright/blob/7d924470d397975a74a19184c136b3573a974e13/packages/playwright-core/src/utils/registry.ts#L141
|
||||
async function getCacheDirectory() {
|
||||
let result
|
||||
const envDefined = process.env['NEXT_SWC_PATH']
|
||||
|
||||
if (envDefined) {
|
||||
result = envDefined
|
||||
} else {
|
||||
let systemCacheDirectory
|
||||
if (process.platform === 'linux') {
|
||||
systemCacheDirectory =
|
||||
process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache')
|
||||
} else if (process.platform === 'darwin') {
|
||||
systemCacheDirectory = path.join(os.homedir(), 'Library', 'Caches')
|
||||
} else if (process.platform === 'win32') {
|
||||
systemCacheDirectory =
|
||||
process.env.LOCALAPPDATA || path.join(os.homedir(), 'AppData', 'Local')
|
||||
} else {
|
||||
/// Attempt to use generic tmp location for un-handled platform
|
||||
if (!systemCacheDirectory) {
|
||||
for (const dir of [
|
||||
path.join(os.homedir(), '.cache'),
|
||||
path.join(os.tmpdir()),
|
||||
]) {
|
||||
if (await fileExists(dir)) {
|
||||
systemCacheDirectory = dir
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!systemCacheDirectory) {
|
||||
console.error(new Error('Unsupported platform: ' + process.platform))
|
||||
process.exit(0)
|
||||
}
|
||||
}
|
||||
result = path.join(systemCacheDirectory, 'next-swc')
|
||||
}
|
||||
|
||||
if (!path.isAbsolute(result)) {
|
||||
// It is important to resolve to the absolute path:
|
||||
// - for unzipping to work correctly;
|
||||
// - so that registry directory matches between installation and execution.
|
||||
// INIT_CWD points to the root of `npm/yarn install` and is probably what
|
||||
// the user meant when typing the relative path.
|
||||
result = path.resolve(process.env['INIT_CWD'] || process.cwd(), result)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
async function extractBinary(
|
||||
outputDirectory: string,
|
||||
pkgName: string,
|
||||
tarFileName: string
|
||||
) {
|
||||
const cacheDirectory = await getCacheDirectory()
|
||||
|
||||
const extractFromTar = async () => {
|
||||
await tar.x({
|
||||
file: path.join(cacheDirectory, tarFileName),
|
||||
cwd: outputDirectory,
|
||||
strip: 1,
|
||||
})
|
||||
}
|
||||
|
||||
if (!(await fileExists(path.join(cacheDirectory, tarFileName)))) {
|
||||
Log.info(`Downloading swc package ${pkgName}...`)
|
||||
await fs.promises.mkdir(cacheDirectory, { recursive: true })
|
||||
const tempFile = path.join(
|
||||
cacheDirectory,
|
||||
`${tarFileName}.temp-${Date.now()}`
|
||||
)
|
||||
|
||||
const registry = getRegistry()
|
||||
|
||||
await fetch(`${registry}${pkgName}/-/${tarFileName}`).then((res) => {
|
||||
const { ok, body } = res
|
||||
if (!ok) {
|
||||
throw new Error(`request failed with status ${res.status}`)
|
||||
}
|
||||
if (!body) {
|
||||
throw new Error('request failed with empty body')
|
||||
}
|
||||
const cacheWriteStream = fs.createWriteStream(tempFile)
|
||||
return body.pipeTo(
|
||||
new WritableStream({
|
||||
write(chunk) {
|
||||
cacheWriteStream.write(chunk)
|
||||
},
|
||||
close() {
|
||||
cacheWriteStream.close()
|
||||
},
|
||||
})
|
||||
)
|
||||
})
|
||||
await fs.promises.rename(tempFile, path.join(cacheDirectory, tarFileName))
|
||||
}
|
||||
await extractFromTar()
|
||||
|
||||
const cacheFiles = await fs.promises.readdir(cacheDirectory)
|
||||
|
||||
if (cacheFiles.length > MAX_VERSIONS_TO_CACHE) {
|
||||
cacheFiles.sort((a, b) => {
|
||||
if (a.length < b.length) return -1
|
||||
return a.localeCompare(b)
|
||||
})
|
||||
|
||||
// prune oldest versions in cache
|
||||
for (let i = 0; i++; i < cacheFiles.length - MAX_VERSIONS_TO_CACHE) {
|
||||
await fs.promises
|
||||
.unlink(path.join(cacheDirectory, cacheFiles[i]))
|
||||
.catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function downloadNativeNextSwc(
|
||||
version: string,
|
||||
bindingsDirectory: string,
|
||||
triplesABI: Array<string>
|
||||
) {
|
||||
for (const triple of triplesABI) {
|
||||
const pkgName = `@next/swc-${triple}`
|
||||
const tarFileName = `${pkgName.substring(6)}-${version}.tgz`
|
||||
const outputDirectory = path.join(bindingsDirectory, pkgName)
|
||||
|
||||
if (await fileExists(outputDirectory)) {
|
||||
// if the package is already downloaded a different
|
||||
// failure occurred than not being present
|
||||
return
|
||||
}
|
||||
|
||||
await fs.promises.mkdir(outputDirectory, { recursive: true })
|
||||
await extractBinary(outputDirectory, pkgName, tarFileName)
|
||||
}
|
||||
}
|
||||
|
||||
export async function downloadWasmSwc(
|
||||
version: string,
|
||||
wasmDirectory: string,
|
||||
variant: 'nodejs' | 'web' = 'nodejs'
|
||||
) {
|
||||
const pkgName = `@next/swc-wasm-${variant}`
|
||||
const tarFileName = `${pkgName.substring(6)}-${version}.tgz`
|
||||
const outputDirectory = path.join(wasmDirectory, pkgName)
|
||||
|
||||
if (await fileExists(outputDirectory)) {
|
||||
// if the package is already downloaded a different
|
||||
// failure occurred than not being present
|
||||
return
|
||||
}
|
||||
|
||||
await fs.promises.mkdir(outputDirectory, { recursive: true })
|
||||
await extractBinary(outputDirectory, pkgName, tarFileName)
|
||||
}
|
|
@ -1,143 +0,0 @@
|
|||
import os from 'os'
|
||||
import fs from 'fs'
|
||||
import path from 'path'
|
||||
import * as Log from '../build/output/log'
|
||||
import tar from 'next/dist/compiled/tar'
|
||||
const { fetch } = require('next/dist/compiled/undici') as {
|
||||
fetch: typeof global.fetch
|
||||
}
|
||||
const { WritableStream } = require('node:stream/web') as {
|
||||
WritableStream: typeof global.WritableStream
|
||||
}
|
||||
import { fileExists } from './file-exists'
|
||||
import { getRegistry } from './helpers/get-registry'
|
||||
|
||||
const MAX_VERSIONS_TO_CACHE = 5
|
||||
|
||||
export async function downloadWasmSwc(
|
||||
version: string,
|
||||
wasmDirectory: string,
|
||||
variant: 'nodejs' | 'web' = 'nodejs'
|
||||
) {
|
||||
const pkgName = `@next/swc-wasm-${variant}`
|
||||
const tarFileName = `${pkgName.substring(6)}-${version}.tgz`
|
||||
const outputDirectory = path.join(wasmDirectory, pkgName)
|
||||
|
||||
if (await fileExists(outputDirectory)) {
|
||||
// if the package is already downloaded a different
|
||||
// failure occurred than not being present
|
||||
return
|
||||
}
|
||||
|
||||
// get platform specific cache directory adapted from playwright's handling
|
||||
// https://github.com/microsoft/playwright/blob/7d924470d397975a74a19184c136b3573a974e13/packages/playwright-core/src/utils/registry.ts#L141
|
||||
const cacheDirectory = await (async () => {
|
||||
let result
|
||||
const envDefined = process.env['NEXT_SWC_PATH']
|
||||
|
||||
if (envDefined) {
|
||||
result = envDefined
|
||||
} else {
|
||||
let systemCacheDirectory
|
||||
if (process.platform === 'linux') {
|
||||
systemCacheDirectory =
|
||||
process.env.XDG_CACHE_HOME || path.join(os.homedir(), '.cache')
|
||||
} else if (process.platform === 'darwin') {
|
||||
systemCacheDirectory = path.join(os.homedir(), 'Library', 'Caches')
|
||||
} else if (process.platform === 'win32') {
|
||||
systemCacheDirectory =
|
||||
process.env.LOCALAPPDATA ||
|
||||
path.join(os.homedir(), 'AppData', 'Local')
|
||||
} else {
|
||||
/// Attempt to use generic tmp location for un-handled platform
|
||||
if (!systemCacheDirectory) {
|
||||
for (const dir of [
|
||||
path.join(os.homedir(), '.cache'),
|
||||
path.join(os.tmpdir()),
|
||||
]) {
|
||||
if (await fileExists(dir)) {
|
||||
systemCacheDirectory = dir
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!systemCacheDirectory) {
|
||||
console.error(new Error('Unsupported platform: ' + process.platform))
|
||||
process.exit(0)
|
||||
}
|
||||
}
|
||||
result = path.join(systemCacheDirectory, 'next-swc')
|
||||
}
|
||||
|
||||
if (!path.isAbsolute(result)) {
|
||||
// It is important to resolve to the absolute path:
|
||||
// - for unzipping to work correctly;
|
||||
// - so that registry directory matches between installation and execution.
|
||||
// INIT_CWD points to the root of `npm/yarn install` and is probably what
|
||||
// the user meant when typing the relative path.
|
||||
result = path.resolve(process.env['INIT_CWD'] || process.cwd(), result)
|
||||
}
|
||||
return result
|
||||
})()
|
||||
|
||||
await fs.promises.mkdir(outputDirectory, { recursive: true })
|
||||
|
||||
const extractFromTar = async () => {
|
||||
await tar.x({
|
||||
file: path.join(cacheDirectory, tarFileName),
|
||||
cwd: outputDirectory,
|
||||
strip: 1,
|
||||
})
|
||||
}
|
||||
|
||||
if (!(await fileExists(path.join(cacheDirectory, tarFileName)))) {
|
||||
Log.info('Downloading WASM swc package...')
|
||||
await fs.promises.mkdir(cacheDirectory, { recursive: true })
|
||||
const tempFile = path.join(
|
||||
cacheDirectory,
|
||||
`${tarFileName}.temp-${Date.now()}`
|
||||
)
|
||||
|
||||
const registry = getRegistry()
|
||||
|
||||
await fetch(`${registry}${pkgName}/-/${tarFileName}`).then((res) => {
|
||||
const { ok, body } = res
|
||||
if (!ok) {
|
||||
throw new Error(`request failed with status ${res.status}`)
|
||||
}
|
||||
if (!body) {
|
||||
throw new Error('request failed with empty body')
|
||||
}
|
||||
const cacheWriteStream = fs.createWriteStream(tempFile)
|
||||
return body.pipeTo(
|
||||
new WritableStream({
|
||||
write(chunk) {
|
||||
cacheWriteStream.write(chunk)
|
||||
},
|
||||
close() {
|
||||
cacheWriteStream.close()
|
||||
},
|
||||
})
|
||||
)
|
||||
})
|
||||
await fs.promises.rename(tempFile, path.join(cacheDirectory, tarFileName))
|
||||
}
|
||||
await extractFromTar()
|
||||
|
||||
const cacheFiles = await fs.promises.readdir(cacheDirectory)
|
||||
|
||||
if (cacheFiles.length > MAX_VERSIONS_TO_CACHE) {
|
||||
cacheFiles.sort((a, b) => {
|
||||
if (a.length < b.length) return -1
|
||||
return a.localeCompare(b)
|
||||
})
|
||||
|
||||
// prune oldest versions in cache
|
||||
for (let i = 0; i++; i < cacheFiles.length - MAX_VERSIONS_TO_CACHE) {
|
||||
await fs.promises
|
||||
.unlink(path.join(cacheDirectory, cacheFiles[i]))
|
||||
.catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in a new issue