Allow filtering individual test cases inside test files (#55786)

### What?

Updates `run-test.js` to allow running individual test cases inside a test file.

### Why?

So that we can dramatically increase Turbopack's test coverage. Turbopack has implemented most (but not all) necessary features from the webpack bundles. But a single failing test case would prevent us from running any case inside a test file. With case filtering, we're able to run the cases we know will pass and prevent regressions on those.

### How?

Case filtering is only exposed via the `NEXT_EXTERNAL_TESTS_FILTERS` ENV, which points to a JSON file containing a map of test files with the test cases inside those files that are known to pass.

The known-passing test cases can be updated after Turbopack's daily integration test run by running `test/build-turbopack-tests-manifest.js`, which will update the `test/turbopack-tests-manifest.json` manifest.

Closes WEB-1640

Co-authored-by: Tobias Koppers <1365881+sokra@users.noreply.github.com>
This commit is contained in:
Justin Ridgewell 2023-09-22 17:37:48 -04:00 committed by GitHub
parent 8f7d24fab7
commit b0aecccaa2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 14682 additions and 492 deletions

View file

@ -33,9 +33,6 @@ jobs:
uses: ./.github/workflows/build_reusable.yml
with:
skipInstallBuild: 'yes'
# Ensure all of the tests listed in turbopack test manifest actually exists;
# i.e after removing an actual test and if it still stays in the manifest
afterBuild: node -e 'require(\"./test/turbopack-tests-manifest.js\").verifyEnabledTestPath()'
secrets: inherit
build-next:
@ -137,7 +134,7 @@ jobs:
uses: ./.github/workflows/build_reusable.yml
with:
skipForDocsOnly: 'yes'
afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.js" TURBOPACK=1 NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_MODE=dev node run-tests.js --test-pattern '^(test\/development)/.*\.test\.(js|jsx|ts|tsx)$' --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY}
afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_MODE=dev node run-tests.js --test-pattern '^(test\/development)/.*\.test\.(js|jsx|ts|tsx)$' --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY}
secrets: inherit
test-turbopack-integration:
@ -151,7 +148,7 @@ jobs:
with:
nodeVersion: 16
skipForDocsOnly: 'yes'
afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.js" TURBOPACK=1 node run-tests.js --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} --type integration
afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 node run-tests.js --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} --type integration
secrets: inherit
# --type production also runs tests/e2e
@ -166,7 +163,7 @@ jobs:
with:
nodeVersion: 16
skipForDocsOnly: 'yes'
afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.js" TURBOPACK=1 NEXT_TEST_MODE=start node run-tests.js --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} --type production
afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 NEXT_TEST_MODE=start node run-tests.js --timings -g ${{ matrix.group }}/5 -c ${TEST_CONCURRENCY} --type production
secrets: inherit
test-next-swc-wasm:

View file

@ -12,6 +12,14 @@ const { createNextInstall } = require('./test/lib/create-next-install')
const glob = promisify(_glob)
const exec = promisify(execOrig)
function escapeRegexp(str) {
return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
}
/**
* @typedef {{ file: string, cases: 'all' | string[] }} TestFile
*/
const GROUP = process.env.CI ? '##[group]' : ''
const ENDGROUP = process.env.CI ? '##[endgroup]' : ''
@ -19,7 +27,7 @@ const ENDGROUP = process.env.CI ? '##[endgroup]' : ''
// If process.argv contains a test to be executed, this'll append it to the list.
const externalTestsFilterLists = process.env.NEXT_EXTERNAL_TESTS_FILTERS
? require(process.env.NEXT_EXTERNAL_TESTS_FILTERS)
: { enabledTests: [] }
: null
const timings = []
const DEFAULT_NUM_RETRIES = os.platform() === 'win32' ? 2 : 1
const DEFAULT_CONCURRENCY = 2
@ -80,11 +88,11 @@ const cleanUpAndExit = async (code) => {
}, 1)
}
const isMatchingPattern = (pattern, test) => {
const isMatchingPattern = (pattern, file) => {
if (pattern instanceof RegExp) {
return pattern.test(test)
return pattern.test(file)
} else {
return test.startsWith(pattern)
return file.startsWith(pattern)
}
}
@ -149,7 +157,13 @@ async function main() {
console.log('Running tests with concurrency:', concurrency)
let tests = process.argv.filter((arg) => arg.match(/\.test\.(js|ts|tsx)/))
/** @type TestFile[] */
let tests = process.argv
.filter((arg) => arg.match(/\.test\.(js|ts|tsx)/))
.map((file) => ({
file,
cases: 'all',
}))
let prevTimings
if (tests.length === 0) {
@ -165,20 +179,27 @@ async function main() {
cwd: __dirname,
ignore: '**/node_modules/**',
})
).filter((test) => {
if (testPatternRegex) {
return testPatternRegex.test(test)
}
if (filterTestsBy) {
// only include the specified type
if (filterTestsBy === 'none') {
return true
)
.filter((file) => {
if (testPatternRegex) {
return testPatternRegex.test(file)
}
return isMatchingPattern(filterTestsBy, test)
}
// include all except the separately configured types
return !configuredTestTypes.some((type) => isMatchingPattern(type, test))
})
if (filterTestsBy) {
// only include the specified type
if (filterTestsBy === 'none') {
return true
}
return isMatchingPattern(filterTestsBy, file)
}
// include all except the separately configured types
return !configuredTestTypes.some((type) =>
isMatchingPattern(type, file)
)
})
.map((file) => ({
file,
cases: 'all',
}))
}
if (outputTimings && groupArg) {
@ -209,21 +230,34 @@ async function main() {
}
// If there are external manifest contains list of tests, apply it to the test lists.
if (externalTestsFilterLists?.enabledTests.length > 0) {
tests = tests.filter((test) =>
externalTestsFilterLists.enabledTests.some((enabled) =>
enabled.includes(test)
)
)
if (externalTestsFilterLists) {
tests = tests
.filter((test) => {
const info = externalTestsFilterLists[test.file]
return info && info.passed.length > 0 && !info.runtimeError
})
.map((test) => {
const info = externalTestsFilterLists[test.file]
// only run filtered mode when there are failing tests.
// When the whole test suite passes we can run all tests, including newly added ones.
if (info.failed.length > 0) {
test.cases = info.passed
}
return test
})
}
let testNames = [
...new Set(
tests.map((f) => {
return `${f.replace(/\\/g, '/').replace(/\/test$/, '')}`
})
),
]
let testSet = new Set()
tests = tests
.map((test) => {
test.file = test.file.replace(/\\/g, '/').replace(/\/test$/, '')
return test
})
.filter((test) => {
if (testSet.has(test.file)) return false
testSet.add(test.file)
return true
})
if (groupArg) {
const groupParts = groupArg.split('/')
@ -234,7 +268,7 @@ async function main() {
const groups = [[]]
const groupTimes = [0]
for (const testName of testNames) {
for (const test of tests) {
let smallestGroup = groupTimes[0]
let smallestGroupIdx = 0
@ -251,38 +285,39 @@ async function main() {
smallestGroupIdx = i
}
}
groups[smallestGroupIdx].push(testName)
groupTimes[smallestGroupIdx] += prevTimings[testName] || 1
groups[smallestGroupIdx].push(test)
groupTimes[smallestGroupIdx] += prevTimings[test.file] || 1
}
const curGroupIdx = groupPos - 1
testNames = groups[curGroupIdx]
tests = groups[curGroupIdx]
console.log(
'Current group previous accumulated times:',
Math.round(groupTimes[curGroupIdx]) + 's'
)
} else {
const numPerGroup = Math.ceil(testNames.length / groupTotal)
const numPerGroup = Math.ceil(tests.length / groupTotal)
let offset = (groupPos - 1) * numPerGroup
testNames = testNames.slice(offset, offset + numPerGroup)
tests = tests.slice(offset, offset + numPerGroup)
console.log('Splitting without timings')
}
}
if (testNames.length === 0) {
if (tests.length === 0) {
console.log('No tests found for', testType, 'exiting..')
return cleanUpAndExit(1)
}
console.log(`${GROUP}Running tests:
${testNames.join('\n')}
${tests.map((t) => t.file).join('\n')}
${ENDGROUP}`)
console.log(`total: ${testNames.length}`)
console.log(`total: ${tests.length}`)
const hasIsolatedTests = testNames.some((test) => {
const hasIsolatedTests = tests.some((test) => {
return configuredTestTypes.some(
(type) => type !== testFilters.unit && test.startsWith(`test/${type}`)
(type) =>
type !== testFilters.unit && test.file.startsWith(`test/${type}`)
)
})
@ -316,8 +351,8 @@ ${ENDGROUP}`)
console.log(`${ENDGROUP}`)
}
const sema = new Sema(concurrency, { capacity: testNames.length })
const outputSema = new Sema(1, { capacity: testNames.length })
const sema = new Sema(concurrency, { capacity: tests.length })
const outputSema = new Sema(1, { capacity: tests.length })
const children = new Set()
const jestPath = path.join(
__dirname,
@ -328,74 +363,73 @@ ${ENDGROUP}`)
let firstError = true
let killed = false
const runTest = (test = '', isFinalRun, isRetry) =>
const runTest = (/** @type {TestFile} */ test, isFinalRun, isRetry) =>
new Promise((resolve, reject) => {
const start = new Date().getTime()
let outputChunks = []
const shouldRecordTestWithReplay = process.env.RECORD_REPLAY && isRetry
const child = spawn(
jestPath,
[
...(shouldRecordTestWithReplay
? [`--config=jest.replay.config.js`]
: []),
'--runInBand',
'--forceExit',
'--verbose',
'--silent',
...(isTestJob
? ['--json', `--outputFile=${test}${RESULTS_EXT}`]
: []),
test,
],
{
stdio: ['ignore', 'pipe', 'pipe'],
env: {
...process.env,
IS_RETRY: isRetry ? 'true' : undefined,
RECORD_REPLAY: shouldRecordTestWithReplay,
// run tests in headless mode by default
HEADLESS: 'true',
TRACE_PLAYWRIGHT: 'true',
NEXT_TELEMETRY_DISABLED: '1',
// unset CI env so CI behavior is only explicitly
// tested when enabled
CI: '',
CIRCLECI: '',
GITHUB_ACTIONS: '',
CONTINUOUS_INTEGRATION: '',
RUN_ID: '',
BUILD_NUMBER: '',
// Format the output of junit report to include the test name
// For the debugging purpose to compare actual run list to the generated reports
// [NOTE]: This won't affect if junit reporter is not enabled
JEST_JUNIT_OUTPUT_NAME:
test && test.length > 0 ? test.replaceAll('/', '_') : undefined,
// Specify suite name for the test to avoid unexpected merging across different env / grouped tests
// This is not individual suites name (corresponding 'describe'), top level suite name which have redundant names by default
// [NOTE]: This won't affect if junit reporter is not enabled
JEST_SUITE_NAME: [
`${process.env.NEXT_TEST_MODE ?? 'default'}`,
groupArg,
testType,
test,
]
.filter(Boolean)
.join(':'),
...(isFinalRun
? {
// Events can be finicky in CI. This switches to a more
// reliable polling method.
// CHOKIDAR_USEPOLLING: 'true',
// CHOKIDAR_INTERVAL: 500,
// WATCHPACK_POLLING: 500,
}
: {}),
},
}
)
const args = [
...(shouldRecordTestWithReplay
? [`--config=jest.replay.config.js`]
: []),
'--runInBand',
'--forceExit',
'--verbose',
'--silent',
...(isTestJob
? ['--json', `--outputFile=${test.file}${RESULTS_EXT}`]
: []),
test.file,
...(test.cases === 'all'
? []
: [
'--testNamePattern',
`^(${test.cases.map(escapeRegexp).join('|')})$`,
]),
]
const env = {
IS_RETRY: isRetry ? 'true' : undefined,
RECORD_REPLAY: shouldRecordTestWithReplay,
// run tests in headless mode by default
HEADLESS: 'true',
TRACE_PLAYWRIGHT: 'true',
NEXT_TELEMETRY_DISABLED: '1',
// unset CI env so CI behavior is only explicitly
// tested when enabled
CI: '',
CIRCLECI: '',
GITHUB_ACTIONS: '',
CONTINUOUS_INTEGRATION: '',
RUN_ID: '',
BUILD_NUMBER: '',
// Format the output of junit report to include the test name
// For the debugging purpose to compare actual run list to the generated reports
// [NOTE]: This won't affect if junit reporter is not enabled
JEST_JUNIT_OUTPUT_NAME: test.file.replaceAll('/', '_'),
// Specify suite name for the test to avoid unexpected merging across different env / grouped tests
// This is not individual suites name (corresponding 'describe'), top level suite name which have redundant names by default
// [NOTE]: This won't affect if junit reporter is not enabled
JEST_SUITE_NAME: [
`${process.env.NEXT_TEST_MODE ?? 'default'}`,
groupArg,
testType,
test.file,
]
.filter(Boolean)
.join(':'),
...(isFinalRun
? {
// Events can be finicky in CI. This switches to a more
// reliable polling method.
// CHOKIDAR_USEPOLLING: 'true',
// CHOKIDAR_INTERVAL: 500,
// WATCHPACK_POLLING: 500,
}
: {}),
}
const handleOutput = (type) => (chunk) => {
if (hideOutput) {
outputChunks.push({ type, chunk })
@ -403,7 +437,23 @@ ${ENDGROUP}`)
process.stdout.write(chunk)
}
}
child.stdout.on('data', handleOutput('stdout'))
const stdout = handleOutput('stdout')
stdout(
[
...Object.entries(env).map((e) => `${e[0]}=${e[1]}`),
jestPath,
...args.map((a) => `'${a}'`),
].join(' ')
)
const child = spawn(jestPath, args, {
stdio: ['ignore', 'pipe', 'pipe'],
env: {
...process.env,
...env,
},
})
child.stdout.on('data', stdout)
child.stderr.on('data', handleOutput('stderr'))
children.add(child)
@ -417,11 +467,11 @@ ${ENDGROUP}`)
firstError && !killed && !shouldContinueTestsOnError
if (isExpanded) {
firstError = false
process.stdout.write(`${test} output:\n`)
process.stdout.write(`${test.file} output:\n`)
} else if (killed) {
process.stdout.write(`${GROUP}${test} output (killed)\n`)
process.stdout.write(`${GROUP}${test.file} output (killed)\n`)
} else {
process.stdout.write(`${GROUP}${test} output\n`)
process.stdout.write(`${GROUP}${test.file} output\n`)
}
// limit out to last 64kb so that we don't
// run out of log room in CI
@ -429,9 +479,9 @@ ${ENDGROUP}`)
process.stdout.write(chunk)
}
if (isExpanded) {
process.stdout.write(`end of ${test} output\n`)
process.stdout.write(`end of ${test.file} output\n`)
} else {
process.stdout.write(`end of ${test} output\n${ENDGROUP}\n`)
process.stdout.write(`end of ${test.file} output\n${ENDGROUP}\n`)
}
outputSema.release()
}
@ -450,7 +500,7 @@ ${ENDGROUP}`)
__dirname,
'test/traces',
path
.relative(path.join(__dirname, 'test'), test)
.relative(path.join(__dirname, 'test'), test.file)
.replace(/\//g, '-')
)
)
@ -463,13 +513,13 @@ ${ENDGROUP}`)
const originalRetries = numRetries
await Promise.all(
testNames.map(async (test) => {
const dirName = path.dirname(test)
tests.map(async (test) => {
const dirName = path.dirname(test.file)
let dirSema = directorySemas.get(dirName)
// we only restrict 1 test per directory for
// legacy integration tests
if (test.startsWith('test/integration') && dirSema === undefined) {
if (test.file.startsWith('test/integration') && dirSema === undefined) {
directorySemas.set(dirName, (dirSema = new Sema(1)))
}
if (dirSema) await dirSema.acquire()
@ -478,34 +528,38 @@ ${ENDGROUP}`)
let passed = false
const shouldSkipRetries = skipRetryTestManifest.find((t) =>
t.includes(test)
t.includes(test.file)
)
const numRetries = shouldSkipRetries ? 0 : originalRetries
if (shouldSkipRetries) {
console.log(`Skipping retry for ${test} due to skipRetryTestManifest`)
console.log(
`Skipping retry for ${test.file} due to skipRetryTestManifest`
)
}
for (let i = 0; i < numRetries + 1; i++) {
try {
console.log(`Starting ${test} retry ${i}/${numRetries}`)
console.log(`Starting ${test.file} retry ${i}/${numRetries}`)
const time = await runTest(
test,
shouldSkipRetries || i === numRetries,
shouldSkipRetries || i > 0
)
timings.push({
file: test,
file: test.file,
time,
})
passed = true
console.log(
`Finished ${test} on retry ${i}/${numRetries} in ${time / 1000}s`
`Finished ${test.file} on retry ${i}/${numRetries} in ${
time / 1000
}s`
)
break
} catch (err) {
if (i < numRetries) {
try {
let testDir = path.dirname(path.join(__dirname, test))
let testDir = path.dirname(path.join(__dirname, test.file))
// if test is nested in a test folder traverse up a dir to ensure
// we clean up relevant test files
@ -517,13 +571,15 @@ ${ENDGROUP}`)
await exec(`git checkout "${testDir}"`)
} catch (err) {}
} else {
console.error(`${test} failed due to ${err}`)
console.error(`${test.file} failed due to ${err}`)
}
}
}
if (!passed) {
console.error(`${test} failed to pass within ${numRetries} retries`)
console.error(
`${test.file} failed to pass within ${numRetries} retries`
)
if (!shouldContinueTestsOnError) {
killed = true
@ -531,7 +587,7 @@ ${ENDGROUP}`)
cleanUpAndExit(1)
} else {
console.log(
`CONTINUE_ON_ERROR enabled, continuing tests after ${test} failed`
`CONTINUE_ON_ERROR enabled, continuing tests after ${test.file} failed`
)
}
}
@ -539,7 +595,10 @@ ${ENDGROUP}`)
// Emit test output if test failed or if we're continuing tests on error
if ((!passed || shouldContinueTestsOnError) && isTestJob) {
try {
const testsOutput = await fs.readFile(`${test}${RESULTS_EXT}`, 'utf8')
const testsOutput = await fs.readFile(
`${test.file}${RESULTS_EXT}`,
'utf8'
)
const obj = JSON.parse(testsOutput)
obj.processEnv = {
NEXT_TEST_MODE: process.env.NEXT_TEST_MODE,

View file

@ -0,0 +1,128 @@
const fetch = require('node-fetch')
const fs = require('fs')
const override = process.argv.includes('--override')
// TODO: Switch to nextjs-integration-test-data branch once https://github.com/vercel/turbo/pull/5999 is merged.
const RESULT_URL =
'https://raw.githubusercontent.com/vercel/turbo/nextjs-integration-test-data/test-results/main/nextjs-test-results.json'
const PASSING_JSON_PATH = `${__dirname}/turbopack-tests-manifest.json`
const WORKING_PATH = '/home/runner/work/turbo/turbo/'
const INITIALIZING_TEST_CASES = [
'compile successfully',
'should build successfully',
]
const SKIPPED_TEST_SUITES = new Set([
'test/integration/router-rerender/test/index.test.js',
'test/e2e/basepath.test.ts',
'test/development/acceptance-app/ReactRefreshRequire.test.ts',
'test/integration/dynamic-routing/test/middleware.test.js',
'test/integration/css/test/css-modules.test.js',
'test/development/acceptance/ReactRefreshRequire.test.ts',
'test/integration/custom-routes/test/index.test.js',
'test/integration/absolute-assetprefix/test/index.test.js',
'test/e2e/middleware-rewrites/test/index.test.ts',
])
async function updatePassingTests() {
const passing = { __proto__: null }
const res = await fetch(RESULT_URL)
const results = await res.json()
for (const result of results.result) {
const runtimeError = result.data.numRuntimeErrorTestSuites > 0
for (const testResult of result.data.testResults) {
const filepath = stripWorkingPath(testResult.name)
for (const file of duplicateFileNames(filepath)) {
if (SKIPPED_TEST_SUITES.has(file)) continue
const fileResults = (passing[file] ??= {
passed: [],
failed: [],
pending: [],
runtimeError,
})
let initializationFailed = false
for (const testCase of testResult.assertionResults) {
let { fullName, status } = testCase
if (
status === 'failed' &&
INITIALIZING_TEST_CASES.some((name) => fullName.includes(name))
) {
initializationFailed = true
} else if (initializationFailed) {
status = 'failed'
}
const statusArray = fileResults[status]
if (!statusArray) {
throw new Error(`unexpected status "${status}"`)
}
statusArray.push(fullName)
}
}
}
}
for (const info of Object.values(passing)) {
info.failed = [...new Set(info.failed)]
info.pending = [...new Set(info.pending)]
info.passed = [
...new Set(info.passed.filter((name) => !info.failed.includes(name))),
]
}
if (!override) {
const oldPassingData = JSON.parse(
fs.readFileSync(PASSING_JSON_PATH, 'utf8')
)
for (const file of Object.keys(oldPassingData)) {
const newData = passing[file]
const oldData = oldPassingData[file]
if (!newData) continue
// We only want to keep test cases from the old data that are still exiting
oldData.passed = oldData.passed.filter(
(name) => newData.failed.includes(name) || newData.passed.includes(name)
)
// Grab test cases that passed before, but fail now
const shouldPass = new Set(
oldData.passed.filter((name) => newData.failed.includes(name))
)
if (shouldPass.size > 0) {
const list = JSON.stringify([...shouldPass], 0, 2)
console.log(
`${file} has ${shouldPass.size} test(s) that should pass but failed: ${list}`
)
}
// Merge the old passing tests with the new ones
newData.passed = [...new Set([...oldData.passed, ...newData.passed])]
// but remove them also from the failed list
newData.failed = newData.failed.filter((name) => !shouldPass.has(name))
}
}
fs.writeFileSync(PASSING_JSON_PATH, JSON.stringify(passing, null, 2))
}
function stripWorkingPath(path) {
if (!path.startsWith(WORKING_PATH)) {
throw new Error(
`found unexpected working path in "${path}", expected it to begin with ${WORKING_PATH}`
)
}
return path.slice(WORKING_PATH.length)
}
function duplicateFileNames(path) {
if (path.includes('/src/')) {
const dist = path.replace('/src/', '/dist/').replace(/.tsx?$/, '.js')
if (fs.existsSync(`${__dirname}/../${dist}`)) {
return [path, dist]
}
}
return [path]
}
updatePassingTests()

View file

@ -31,6 +31,9 @@ describe('Build Output', () => {
)} };`
)
}
;({ stdout } = await nextBuild(appDir, [], {
stdout: true,
}))
})
if (hasExperimentalConfig) {
@ -40,10 +43,6 @@ describe('Build Output', () => {
}
it('should not include internal pages', async () => {
;({ stdout } = await nextBuild(appDir, [], {
stdout: true,
}))
expect(stdout).toMatch(/\/ (.* )?\d{1,} B/)
expect(stdout).toMatch(/\+ First Load JS shared by all [ 0-9.]* kB/)
expect(stdout).toMatch(/ chunks\/main-[0-9a-z]{16}\.js [ 0-9.]* kB/)

View file

@ -1,356 +0,0 @@
// Tests that are currently enabled with experimental Turbopack in CI.
// Only tests that are actively testing against Turbopack should
// be enabled here
const enabledTests = [
'test/development/acceptance-app/server-components.test.ts',
'test/development/acceptance/hydration-error.test.ts',
'test/development/api-cors-with-rewrite/index.test.ts',
'test/development/app-dir/basic/basic.test.ts',
'test/development/app-dir/multiple-compiles-single-route/multiple-compiles-single-route.test.ts',
'test/development/app-dir/strict-mode-enabled-by-default/strict-mode-enabled-by-default.test.ts',
'test/development/basic/define-class-fields.test.ts',
'test/development/basic/emotion-swc.test.ts',
'test/development/basic/legacy-decorators.test.ts',
'test/development/basic/misc.test.ts',
'test/development/basic/node-builtins.test.ts',
'test/development/basic/next-rs-api.test.ts',
'test/development/basic/tailwind-jit.test.ts',
'test/development/basic/theme-ui.test.ts',
'test/development/client-dev-overlay/index.test.ts',
'test/development/correct-tsconfig-defaults/index.test.ts',
'test/development/dotenv-default-expansion/index.test.ts',
'test/development/experimental-https-server/https-server.generated-key.test.ts',
'test/development/experimental-https-server/https-server.provided-key.test.ts',
'test/development/gssp-notfound/index.test.ts',
'test/development/jsconfig-path-reloading/index.test.ts',
'test/development/middleware-warnings/index.test.ts',
'test/development/next-font/deprecated-package.test.ts',
'test/development/repeated-dev-edits/repeated-dev-edits.test.ts',
'test/development/tsconfig-path-reloading/index.test.ts',
'test/development/typescript-auto-install/index.test.ts',
'test/development/watch-config-file/index.test.ts',
'test/e2e/app-dir/app-fetch-deduping/app-fetch-deduping.test.ts',
'test/e2e/app-dir/app-static/app-fetch-logging.test.ts',
'test/e2e/app-dir/app-validation/validation.test.ts',
'test/e2e/app-dir/asset-prefix/asset-prefix.test.ts',
'test/e2e/app-dir/async-component-preload/async-component-preload.test.ts',
'test/e2e/app-dir/autoscroll-with-css-modules/index.test.ts',
'test/e2e/app-dir/build-size/index.test.ts',
'test/e2e/app-dir/crypto-globally-available/crypto-globally-available.test.ts',
'test/e2e/app-dir/deopted-into-client-rendering-warning/deopted-into-client-rendering-warning.test.ts',
'test/e2e/app-dir/front-redirect-issue/front-redirect-issue.test.ts',
'test/e2e/app-dir/headers-static-bailout/headers-static-bailout.test.ts',
'test/e2e/app-dir/hello-world/hello-world.test.ts',
'test/e2e/app-dir/i18n-hybrid/i18n-hybrid.test.js',
'test/e2e/app-dir/import/import.test.ts',
'test/e2e/app-dir/interoperability-with-pages/navigation.test.ts',
'test/e2e/app-dir/layout-params/layout-params.test.ts',
'test/e2e/app-dir/metadata-missing-metadata-base/index.test.ts',
'test/e2e/app-dir/metadata-suspense/index.test.ts',
'test/e2e/app-dir/next-config/index.test.ts',
'test/e2e/app-dir/route-page-manifest-bug/route-page-manifest-bug.test.ts',
'test/e2e/app-dir/router-stuck-dynamic-static-segment/router-stuck-dynamic-static-segment.test.ts',
'test/e2e/app-dir/searchparams-static-bailout/searchparams-static-bailout.test.ts',
'test/e2e/app-dir/similar-pages-paths/similar-pages-paths.test.ts',
'test/e2e/app-dir/test-template/{{ toFileName name }}/{{ toFileName name }}.test.ts',
'test/e2e/app-dir/third-parties/basic.test.ts',
'test/e2e/app-dir/use-params/use-params.test.ts',
'test/e2e/app-dir/use-selected-layout-segment-s/use-selected-layout-segment-s.test.ts',
'test/e2e/browserslist-extends/index.test.ts',
'test/e2e/children-page/index.test.ts',
'test/e2e/config-promise-export/async-function.test.ts',
'test/e2e/config-promise-export/promise.test.ts',
'test/e2e/dynamic-route-interpolation/index.test.ts',
'test/e2e/handle-non-hoisted-swc-helpers/index.test.ts',
'test/e2e/hello-world/hello-world.test.ts',
'test/e2e/i18n-api-support/index.test.ts',
'test/e2e/i18n-disallow-multiple-locales/i18n-disallow-multiple-locales.test.ts',
'test/e2e/link-with-api-rewrite/index.test.ts',
'test/e2e/middleware-fetches-with-body/index.test.ts',
'test/e2e/next-head/index.test.ts',
'test/e2e/next-image-forward-ref/index.test.ts',
'test/e2e/no-eslint-warn-with-no-eslint-config/index.test.ts',
'test/e2e/nonce-head-manager/index.test.ts',
'test/e2e/optimized-loading/test/index.test.ts',
'test/e2e/pages-performance-mark/index.test.ts',
'test/e2e/postcss-config-cjs/index.test.ts',
'test/e2e/prerender-crawler.test.ts',
'test/e2e/proxy-request-with-middleware/test/index.test.ts',
'test/e2e/repeated-forward-slashes-error/repeated-forward-slashes-error.test.ts',
'test/e2e/ssr-react-context/index.test.ts',
'test/e2e/styled-jsx/index.test.ts',
'test/e2e/test-template/{{ toFileName name }}/{{ toFileName name }}.test.ts',
'test/e2e/test-utils-tests/basic/basic.test.ts',
'test/e2e/third-parties/index.test.ts',
'test/e2e/trailingslash-with-rewrite/index.test.ts',
'test/e2e/transpile-packages/index.test.ts',
'test/e2e/type-module-interop/index.test.ts',
'test/e2e/typescript-version-no-warning/typescript-version-no-warning.test.ts',
'test/e2e/typescript-version-warning/typescript-version-warning.test.ts',
'test/e2e/undici-fetch/index.test.ts',
'test/integration/api-support/test/index.test.js',
'test/integration/app-dir-export/test/config.test.ts',
'test/integration/404-page/test/index.test.js',
'test/integration/404-page-app/test/index.test.js',
'test/integration/404-page-custom-error/test/index.test.js',
'test/integration/404-page-ssg/test/index.test.js',
'test/integration/create-next-app/templates-app.test.ts',
'test/integration/create-next-app/templates-pages.test.ts',
'test/integration/custom-routes-i18n-index-redirect/test/index.test.js',
'test/integration/dist-dir/test/index.test.js',
'test/integration/index-index/test/index.test.js',
'test/integration/next-image-new/middleware/test/index.test.ts',
'test/integration/next-image-new/react-virtualized/test/index.test.ts',
'test/integration/next-image-new/typescript/test/index.test.ts',
'test/integration/next-image-new/unoptimized/test/index.test.ts',
'test/integration/revalidate-as-path/test/index.test.js',
'test/integration/rewrites-destination-query-array/test/index.test.js',
'test/integration/root-optional-revalidate/test/index.test.js',
'test/integration/route-index/test/index.test.js',
'test/integration/route-indexes/test/index.test.js',
'test/integration/scss/test/dev-css-handling.test.js',
'test/integration/src-dir-support-double-dir/test/index.test.js',
'test/integration/src-dir-support/test/index.test.js',
'test/integration/ssg-data-404/test/index.test.js',
'test/integration/ssg-dynamic-routes-404-page/test/index.test.js',
'test/integration/static-404/test/index.test.js',
'test/integration/static-page-name/test/index.test.js',
'test/integration/trailing-slash-dist/test/index.test.js',
'test/integration/trailing-slashes-href-resolving/test/index.test.js',
'test/production/app-dir-hide-suppressed-error-during-next-export/index.test.ts',
'test/production/app-dir-prefetch-non-iso-url/index.test.ts',
'test/production/jest/new-link-behavior.test.ts',
'test/production/jest/next-image-preload/next-image-preload.test.ts',
'test/production/jest/rsc/lib/utils.test.js',
'test/production/jest/transpile-packages.test.ts',
'test/production/postcss-plugin-config-as-string/index.test.ts',
]
/// Naive check to ensure that the enabled tests actually exist
const verifyEnabledTestPath = () => {
const fs = require('fs')
const nonExistTests = enabledTests.filter(
(testPath) => !fs.existsSync(testPath)
)
if (Array.isArray(nonExistTests) && nonExistTests.length > 0) {
console.error(
`The following tests are enabled but do not exist:`,
nonExistTests
)
throw new Error('Invalid test path(s) found')
}
}
module.exports = { enabledTests, verifyEnabledTestPath }
/* Old turbopack enabled tests:
'test/development/acceptance-app/ReactRefresh.test.ts',
'test/development/acceptance-app/ReactRefreshLogBoxMisc.test.ts',
'test/development/acceptance-app/ReactRefreshRequire.test.ts',
// 'test/development/acceptance-app/app-hmr-changes.test.ts', (FLAKY)
'test/development/acceptance-app/dynamic-error.test.ts',
'test/development/acceptance-app/version-staleness.test.ts',
'test/development/acceptance/ReactRefreshLogBox-scss.test.ts',
'test/development/acceptance/ReactRefreshLogBoxMisc.test.ts',
'test/development/api-cors-with-rewrite/index.test.ts',
'test/development/app-dir/multiple-compiles-single-route/multiple-compiles-single-route.test.ts',
// x-ref: below test is flakey and needs to be investigated further
// 'test/development/app-hmr/hmr.test.ts',
'test/development/basic/define-class-fields.test.ts',
'test/development/basic/emotion-swc.test.ts',
'test/development/basic/legacy-decorators.test.ts',
'test/development/basic/project-directory-rename.test.ts',
'test/development/basic/theme-ui.test.ts',
'test/development/dotenv-default-expansion/index.test.ts',
'test/development/jsconfig-path-reloading/index.test.ts',
'test/development/middleware-warnings/index.test.ts',
'test/development/project-directory-with-styled-jsx-suffix/index.test.ts',
'test/development/repeated-dev-edits/repeated-dev-edits.test.ts',
'test/development/tsconfig-path-reloading/index.test.ts',
'test/e2e/app-dir/_allow-underscored-root-directory/_allow-underscored-root-directory.test.ts',
'test/e2e/app-dir/actions/app-action-invalid.test.ts',
'test/e2e/app-dir/actions/app-action-size-limit-invalid.test.ts',
'test/e2e/app-dir/app-alias/app-alias.test.ts',
'test/e2e/app-dir/app-client-cache/client-cache.test.ts',
'test/e2e/app-dir/app-css-pageextensions/index.test.ts',
'test/e2e/app-dir/app-edge-root-layout/index.test.ts',
'test/e2e/app-dir/app-edge/app-edge.test.ts',
'test/e2e/app-dir/app-prefetch-false/app-prefetch-false.test.ts',
'test/e2e/app-dir/app-prefetch/prefetching.test.ts',
'test/e2e/app-dir/app-validation/validation.test.ts',
'test/e2e/app-dir/asset-prefix/asset-prefix.test.ts',
'test/e2e/app-dir/async-component-preload/async-component-preload.test.ts',
'test/e2e/app-dir/autoscroll-with-css-modules/index.test.ts',
'test/e2e/app-dir/back-button-download-bug/back-button-download-bug.test.ts',
'test/e2e/app-dir/build-size/index.test.ts',
'test/e2e/app-dir/global-error/global-error.test.ts',
'test/e2e/app-dir/hello-world/hello-world.test.ts',
'test/e2e/app-dir/import/import.test.ts',
'test/e2e/app-dir/layout-params/layout-params.test.ts',
'test/e2e/app-dir/metadata-suspense/index.test.ts',
'test/e2e/app-dir/rewrites-redirects/rewrites-redirects.test.ts',
'test/e2e/app-dir/route-page-manifest-bug/route-page-manifest-bug.test.ts',
'test/e2e/app-dir/router-autoscroll/router-autoscroll.test.ts',
'test/e2e/app-dir/router-stuck-dynamic-static-segment/router-stuck-dynamic-static-segment.test.ts',
'test/e2e/app-dir/rsc-basic/rsc-basic.test.ts',
'test/e2e/app-dir/search-params-react-key/layout-params.test.ts',
'test/e2e/app-dir/searchparams-static-bailout/searchparams-static-bailout.test.ts',
'test/e2e/app-dir/test-template/{{ toFileName name }}/{{ toFileName name }}.test.ts',
'test/e2e/app-dir/use-selected-layout-segment-s/use-selected-layout-segment-s.test.ts',
'test/e2e/app-dir/crypto-globally-available/crypto-globally-available.test.ts',
'test/e2e/browserslist-extends/index.test.ts',
'test/e2e/config-promise-export/async-function.test.ts',
'test/e2e/config-promise-export/promise.test.ts',
'test/e2e/conflicting-app-page-error/index.test.ts',
'test/e2e/disable-js-preload/test/index.test.js',
'test/e2e/hello-world/hello-world.test.ts',
'test/e2e/i18n-api-support/index.test.ts',
'test/e2e/i18n-disallow-multiple-locales/i18n-disallow-multiple-locales.test.ts',
'test/e2e/i18n-ignore-rewrite-source-locale/rewrites-with-basepath.test.ts',
'test/e2e/i18n-ignore-rewrite-source-locale/rewrites.test.ts',
'test/e2e/link-with-api-rewrite/index.test.ts',
'test/e2e/middleware-fetches-with-body/index.test.ts',
'test/e2e/middleware-shallow-link/index.test.ts',
'test/e2e/new-link-behavior/child-a-tag-error.test.ts',
'test/e2e/new-link-behavior/index.test.ts',
'test/e2e/new-link-behavior/material-ui.test.ts',
'test/e2e/new-link-behavior/stitches.test.ts',
'test/e2e/new-link-behavior/typescript.test.ts',
'test/e2e/next-head/index.test.ts',
'test/e2e/next-image-forward-ref/index.test.ts',
'test/e2e/no-eslint-warn-with-no-eslint-config/index.test.ts',
'test/e2e/nonce-head-manager/index.test.ts',
'test/e2e/postcss-config-cjs/index.test.ts',
'test/e2e/proxy-request-with-middleware/test/index.test.ts',
'test/e2e/repeated-forward-slashes-error/repeated-forward-slashes-error.test.ts',
'test/e2e/styled-jsx/index.test.ts',
'test/e2e/test-template/{{ toFileName name }}/{{ toFileName name }}.test.ts',
'test/e2e/test-utils-tests/basic/basic.test.ts',
'test/e2e/trailingslash-with-rewrite/index.test.ts',
'test/e2e/transpile-packages/index.test.ts',
'test/e2e/type-module-interop/index.test.ts',
'test/e2e/typescript-version-no-warning/typescript-version-no-warning.test.ts',
'test/e2e/typescript-version-warning/typescript-version-warning.test.ts',
'test/e2e/undici-fetch/index.test.ts',
'test/e2e/yarn-pnp/test/with-eslint.test.ts',
'test/e2e/yarn-pnp/test/with-next-sass.test.ts',
'test/integration/404-page-custom-error/test/index.test.js',
'test/integration/amp-export-validation/test/index.test.js',
'test/integration/amphtml-custom-validator/test/index.test.js',
'test/integration/amphtml-fragment-style/test/index.test.js',
'test/integration/api-body-parser/test/index.test.js',
'test/integration/api-catch-all/test/index.test.js',
'test/integration/app-aspath/test/index.test.js',
'test/integration/app-config-asset-prefix/test/index.test.js',
'test/integration/app-functional/test/index.test.js',
'test/integration/auto-export-error-bail/test/index.test.js',
'test/integration/auto-export-query-error/test/index.test.js',
'test/integration/bigint/test/index.test.js',
'test/integration/catches-missing-getStaticProps/test/index.test.js',
'test/integration/clean-distdir/test/index.test.js',
'test/integration/client-navigation-a11y/test/index.test.js',
// TODO: re-enable once the logging is aligned
// 'test/integration/config-experimental-warning/test/index.test.js',
'test/integration/config-schema-check/test/index.test.js',
'test/integration/config-syntax-error/test/index.test.js',
'test/integration/config-validation/test/index.test.ts',
'test/integration/conflicting-ssg-paths/test/index.test.js',
'test/integration/create-next-app/index.test.ts',
'test/integration/create-next-app/package-manager.test.ts',
'test/integration/create-next-app/templates-app.test.ts',
'test/integration/create-next-app/templates-pages.test.ts',
'test/integration/css/test/dev-css-handling.test.js',
'test/integration/custom-error-page-exception/test/index.test.js',
'test/integration/custom-server-types/test/index.test.js',
'test/integration/custom-server/test/index.test.js',
'test/integration/dedupes-scripts/test/index.test.js',
'test/integration/development-hmr-refresh/test/index.test.js',
'test/integration/disable-js/test/index.test.js',
'test/integration/document-head-warnings/test/index.test.js',
'test/integration/duplicate-pages/test/index.test.js',
'test/integration/dynamic-require/test/index.test.js',
'test/integration/dynamic-route-rename/test/index.test.js',
'test/integration/empty-object-getInitialProps/test/index.test.js',
'test/integration/errors-on-output-to-public/test/index.test.js',
'test/integration/errors-on-output-to-static/test/index.test.js',
'test/integration/eslint/test/lint-cache.test.js',
'test/integration/eslint/test/next-lint.test.js',
'test/integration/export-404/test/index.test.js',
'test/integration/export-default-map/test/index.test.js',
'test/integration/export-dynamic-pages/test/index.test.js',
'test/integration/export-fallback-true-error/test/index.test.js',
'test/integration/export-getInitialProps-warn/test/index.test.js',
'test/integration/export-image-default/test/index.test.js',
'test/integration/export-image-loader-legacy/test/index.test.js',
'test/integration/export-index-not-found-gsp/test/index.test.ts',
'test/integration/export-intent/test/index.test.js',
'test/integration/export-no-build/test/index.test.js',
'test/integration/export-progress-status-message/test/index.test.js',
'test/integration/export-subfolders/test/index.test.js',
'test/integration/fallback-modules/test/index.test.js',
'test/integration/filesystempublicroutes/test/index.test.js',
'test/integration/firebase-grpc/test/index.test.js',
'test/integration/future/test/index.test.js',
'test/integration/gsp-extension/test/index.test.js',
'test/integration/handles-export-errors/test/index.test.js',
'test/integration/hashbang/test/index.test.js',
'test/integration/index-index/test/index.test.js',
'test/integration/initial-ref/test/index.test.js',
'test/integration/invalid-config-values/test/index.test.js',
'test/integration/invalid-page-automatic-static-optimization/test/index.test.js',
'test/integration/invalid-revalidate-values/test/index.test.js',
'test/integration/invalid-server-options/test/index.test.js',
'test/integration/json-serialize-original-error/test/index.test.js',
'test/integration/legacy-ssg-methods-error/test/index.test.js',
'test/integration/link-ref/test/index.test.js',
'test/integration/link-with-multiple-child/test/index.test.js',
'test/integration/link-without-router/test/index.test.js',
'test/integration/middleware-build-errors/test/index.test.js',
'test/integration/middleware-overrides-node.js-api/test/index.test.ts',
'test/integration/missing-document-component-error/test/index.test.js',
'test/integration/next-image-legacy/custom-resolver/test/index.test.ts',
'test/integration/next-image-legacy/no-intersection-observer-fallback/test/index.test.ts',
'test/integration/next-image-legacy/noscript/test/index.test.ts',
'test/integration/next-image-legacy/react-virtualized/test/index.test.ts',
'test/integration/next-image-legacy/unoptimized/test/index.test.ts',
'test/integration/next-image-new/react-virtualized/test/index.test.ts',
// 'test/integration/next-image-new/unoptimized/test/index.test.ts', (FLAKY)
'test/integration/no-op-export/test/index.test.js',
'test/integration/non-next-dist-exclude/test/index.test.js',
'test/integration/non-standard-node-env-warning/test/index.test.js',
'test/integration/ondemand/test/index.test.js',
'test/integration/optional-chaining-nullish-coalescing/test/index.test.js',
'test/integration/plugin-mdx-rs/test/index.test.js',
'test/integration/prerender-invalid-catchall-params/test/index.test.js',
'test/integration/prerender-invalid-paths/test/index.test.js',
'test/integration/prerender-legacy/test/index.test.js',
'test/integration/production-build-dir/test/index.test.js',
'test/integration/production-start-no-build/test/index.test.js',
'test/integration/router-hash-navigation/test/index.test.js',
'test/integration/router-prefetch/test/index.test.js',
'test/integration/router-rerender/test/index.test.js',
'test/integration/scss/test/dev-css-handling.test.js',
'test/integration/src-dir-support-double-dir/test/index.test.js',
'test/integration/ssg-dynamic-routes-404-page/test/index.test.js',
'test/integration/static-404/test/index.test.js',
'test/integration/static-page-name/test/index.test.js',
'test/integration/trailing-slashes-href-resolving/test/index.test.js',
'test/integration/tsconfig-verifier/test/index.test.js',
'test/integration/typescript-baseurl/test/index.test.js',
'test/integration/typescript-external-dir/project/test/index.test.js',
'test/integration/typescript-filtered-files/test/index.test.js',
// 'test/integration/typescript-hmr/test/index.test.js', (FLAKY)
'test/integration/webpack-config-mainjs/test/index.test.js',
'test/integration/with-electron/test/index.test.js',
'test/production/ci-missing-typescript-deps/index.test.ts',
'test/production/enoent-during-require/index.test.ts',
'test/production/eslint-plugin-deps/index.test.ts',
'test/production/fallback-export-error/index.test.ts',
'test/production/jest/new-link-behavior.test.ts',
'test/production/jest/transpile-packages.test.ts',
'test/production/postcss-plugin-config-as-string/index.test.ts',
'test/production/supports-module-resolution-nodenext/supports-moduleresolution-nodenext.test.ts',
*/

File diff suppressed because it is too large Load diff