test(turbopack): run next.js integration tests with denylist (#50172)
<!-- Thanks for opening a PR! Your contribution is much appreciated. To make sure your PR is handled as smoothly as possible we request that you follow the checklist sections below. Choose the right checklist for the change(s) that you're making: ## For Contributors ### Improving Documentation or adding/fixing Examples - The "examples guidelines" are followed from our contributing doc https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md - Make sure the linting passes by running `pnpm build && pnpm lint`. See https://github.com/vercel/next.js/blob/canary/contributing/repository/linting.md ### Fixing a bug - Related issues linked using `fixes #number` - Tests added. See: https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs - Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md ### Adding a feature - Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR. (A discussion must be opened, see https://github.com/vercel/next.js/discussions/new?category=ideas) - Related issues/discussions are linked using `fixes #number` - e2e tests added (https://github.com/vercel/next.js/blob/canary/contributing/core/testing.md#writing-tests-for-nextjs - Documentation added - Telemetry added. In case of a feature if it's used or not. - Errors have a helpful link attached, see https://github.com/vercel/next.js/blob/canary/contributing.md ## For Maintainers - Minimal description (aim for explaining to someone not on the team to understand the PR) - When linking to a Slack thread, you might want to share details of the conclusion - Link both the Linear (Fixes NEXT-xxx) and the GitHub issues - Add review comments if necessary to explain to the reviewer the logic behind a change ### Why? ### How? Closes NEXT- Fixes # --> ### What? WEB-1086. This PR flips the way running next.js integration test with Turbopack, instead of using allowlist to run selectively enabled test now uses denylists. The immediate effect is now any new test being added (as a new file) will be executed with Turbopack by default, and should be manually excluded if it fails. Still, the number of tests we run remains same.
This commit is contained in:
parent
1e86c8b5be
commit
2fc0160a26
4 changed files with 139 additions and 13 deletions
120
.github/workflows/build_test_deploy.yml
vendored
120
.github/workflows/build_test_deploy.yml
vendored
|
@ -812,9 +812,8 @@ jobs:
|
|||
- run: xvfb-run node run-tests.js test/integration/with-electron/test/index.test.js
|
||||
if: ${{needs.build.outputs.docsChange == 'nope'}}
|
||||
|
||||
# A job to run sets of tests with turbopack enabled. These tests are considered as `stable`,
|
||||
# that running with turbopack should always pass.
|
||||
testTurbopack:
|
||||
# A job to run sets of devlopment tests with turbopack enabled.
|
||||
testDevTurbopack:
|
||||
name: Test Development (Turbopack)
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build, build-native-test]
|
||||
|
@ -832,6 +831,10 @@ jobs:
|
|||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
node: [18]
|
||||
# Currently turbopack tests does not utilize groups yet
|
||||
group: [1]
|
||||
steps:
|
||||
- run: echo "${{needs.build.outputs.docsChange}}"
|
||||
|
||||
|
@ -853,10 +856,119 @@ jobs:
|
|||
name: next-swc-test-binary
|
||||
path: packages/next-swc/native
|
||||
|
||||
- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type development --timings -c 1 >> /proc/1/fd/1"
|
||||
- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type development --timings >> /proc/1/fd/1"
|
||||
name: Run test/development
|
||||
if: ${{needs.build.outputs.docsChange == 'nope'}}
|
||||
|
||||
testDevE2ETurbopack:
|
||||
name: Test Development (E2E) (Turbopack)
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build, build-native-test]
|
||||
timeout-minutes: 35
|
||||
env:
|
||||
NEXT_TELEMETRY_DISABLED: 1
|
||||
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
|
||||
# Enabling backtrace will makes snapshot tests fail
|
||||
RUST_BACKTRACE: 0
|
||||
# Path to the custom next-swc bindings.
|
||||
NEXT_BINDINGS_BIN: /work/packages/next-swc/native/next-swc.linux-x64-gnu.node
|
||||
# Glob pattern to run specific tests with --turbo.
|
||||
NEXT_DEV_TEST_GLOB: '*'
|
||||
NEXT_EXTERNAL_TESTS_FILTERS: /work/packages/next-swc/crates/next-dev-tests/tests-manifest.js
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
node: [18]
|
||||
# Currently turbopack tests does not utilize groups yet
|
||||
group: [1]
|
||||
steps:
|
||||
- run: echo "${{needs.build.outputs.docsChange}}"
|
||||
|
||||
# https://github.com/actions/virtual-environments/issues/1187
|
||||
- name: tune linux network
|
||||
run: sudo ethtool -K eth0 tx off rx off
|
||||
|
||||
- uses: actions/cache@v3
|
||||
timeout-minutes: 5
|
||||
if: ${{needs.build.outputs.docsChange == 'nope'}}
|
||||
id: restore-build
|
||||
with:
|
||||
path: ./*
|
||||
key: ${{ github.sha }}-${{ github.run_number }}
|
||||
|
||||
- uses: actions/download-artifact@v3
|
||||
if: ${{needs.build.outputs.docsChange == 'nope'}}
|
||||
with:
|
||||
name: next-swc-test-binary
|
||||
path: packages/next-swc/native
|
||||
|
||||
- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && curl -s https://install-node.vercel.app/v${{ matrix.node }} | FORCE=1 bash && node -v && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 NEXT_TEST_MODE=dev TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --type e2e --timings >> /proc/1/fd/1"
|
||||
name: Run test/e2e (dev)
|
||||
if: ${{needs.build.outputs.docsChange == 'nope'}}
|
||||
|
||||
- name: Upload test trace
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: test-trace
|
||||
if-no-files-found: ignore
|
||||
retention-days: 2
|
||||
path: |
|
||||
test/traces
|
||||
|
||||
testIntegrationTurbopack:
|
||||
name: Test Integration (Turbopack)
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build, build-native-test]
|
||||
timeout-minutes: 35
|
||||
env:
|
||||
NEXT_TELEMETRY_DISABLED: 1
|
||||
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
|
||||
# Enabling backtrace will makes snapshot tests fail
|
||||
RUST_BACKTRACE: 0
|
||||
# Path to the custom next-swc bindings.
|
||||
NEXT_BINDINGS_BIN: /work/packages/next-swc/native/next-swc.linux-x64-gnu.node
|
||||
# Glob pattern to run specific tests with --turbo.
|
||||
NEXT_DEV_TEST_GLOB: '*'
|
||||
NEXT_EXTERNAL_TESTS_FILTERS: /work/packages/next-swc/crates/next-dev-tests/tests-manifest.js
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
group: [1]
|
||||
steps:
|
||||
- run: echo "${{needs.build.outputs.docsChange}}"
|
||||
|
||||
# https://github.com/actions/virtual-environments/issues/1187
|
||||
- name: tune linux network
|
||||
run: sudo ethtool -K eth0 tx off rx off
|
||||
|
||||
- uses: actions/cache@v3
|
||||
timeout-minutes: 5
|
||||
if: ${{needs.build.outputs.docsChange == 'nope'}}
|
||||
id: restore-build
|
||||
with:
|
||||
path: ./*
|
||||
key: ${{ github.sha }}-${{ github.run_number }}
|
||||
|
||||
- uses: actions/download-artifact@v3
|
||||
if: ${{needs.build.outputs.docsChange == 'nope'}}
|
||||
with:
|
||||
name: next-swc-test-binary
|
||||
path: packages/next-swc/native
|
||||
|
||||
- run: docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.28.1-jammy /bin/bash -c "cd /work && NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && npm i -g pnpm@${PNPM_VERSION} > /dev/null && NEXT_EXTERNAL_TESTS_FILTERS=${NEXT_EXTERNAL_TESTS_FILTERS} __INTERNAL_NEXT_DEV_TEST_TURBO_DEV=TRUE __INTERNAL_CUSTOM_TURBOPACK_BINDINGS=${NEXT_BINDINGS_BIN} __INTERNAL_NEXT_DEV_TEST_TURBO_GLOB_MATCH=${NEXT_DEV_TEST_GLOB} NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_JOB=1 TEST_TIMINGS_TOKEN=${{ secrets.TEST_TIMINGS_TOKEN }} xvfb-run node run-tests.js --timings >> /proc/1/fd/1"
|
||||
if: ${{needs.build.outputs.docsChange == 'nope'}}
|
||||
|
||||
- name: Upload test trace
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: test-trace
|
||||
if-no-files-found: ignore
|
||||
retention-days: 2
|
||||
path: |
|
||||
test/traces
|
||||
|
||||
testsPass:
|
||||
name: thank you, next
|
||||
runs-on: ubuntu-latest
|
||||
|
|
|
@ -8,10 +8,11 @@ const path = require('path')
|
|||
|
||||
const generateManifest = (enabledTests, disabledTests) => `
|
||||
// Tests that are currently enabled with Turbopack in CI.
|
||||
// Add new test when Turbopack updates to fix / implement a feature.
|
||||
// This list is not actively used, more of high level pictures of what tests are enabled.
|
||||
const enabledTests = ${enabledTests}
|
||||
|
||||
// Tests that are currently disabled with Turbopack in CI.
|
||||
// Any tests not listed in here are assumed to be enabled.
|
||||
const disabledTests = ${disabledTests}
|
||||
|
||||
module.exports = {
|
||||
|
@ -30,7 +31,7 @@ const main = async () => {
|
|||
enabledTests = manifest.enabledTests
|
||||
disabledTests = manifest.disabledTests
|
||||
} else {
|
||||
throw new Error('a')
|
||||
throw new Error('Manifest should exists')
|
||||
}
|
||||
|
||||
// Collect all test files
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
// Tests that are currently enabled with Turbopack in CI.
|
||||
// Add new test when Turbopack updates to fix / implement a feature.
|
||||
// This list is not actively used, more of high level pictures of what tests are enabled.
|
||||
const enabledTests = [
|
||||
'test/development/acceptance-app/dynamic-error.test.ts',
|
||||
'test/development/basic/legacy-decorators.test.ts',
|
||||
|
@ -9,6 +9,7 @@ const enabledTests = [
|
|||
]
|
||||
|
||||
// Tests that are currently disabled with Turbopack in CI.
|
||||
// Any tests not listed in here are assumed to be enabled.
|
||||
const disabledTests = [
|
||||
'test/development/acceptance-app/app-hmr-changes.test.ts',
|
||||
'test/development/acceptance-app/component-stack.test.ts',
|
||||
|
@ -76,6 +77,7 @@ const disabledTests = [
|
|||
'test/e2e/404-page-router/index.test.ts',
|
||||
'test/e2e/app-dir-legacy-edge-runtime-config/index.test.ts',
|
||||
'test/e2e/app-dir/_allow-underscored-root-directory/_allow-underscored-root-directory.test.ts',
|
||||
'test/e2e/app-dir/actions/app-action-export.test.ts',
|
||||
'test/e2e/app-dir/actions/app-action.test.ts',
|
||||
'test/e2e/app-dir/app-a11y/index.test.ts',
|
||||
'test/e2e/app-dir/app-alias/app-alias.test.ts',
|
||||
|
@ -108,6 +110,7 @@ const disabledTests = [
|
|||
'test/e2e/app-dir/dynamic-href/dynamic-href.test.ts',
|
||||
'test/e2e/app-dir/dynamic/dynamic.test.ts',
|
||||
'test/e2e/app-dir/edge-runtime-node-compatibility/edge-runtime-node-compatibility.test.ts',
|
||||
'test/e2e/app-dir/error-boundary-and-not-found-linking/error-boundary-and-not-found-linking.test.ts',
|
||||
'test/e2e/app-dir/front-redirect-issue/front-redirect-issue.test.ts',
|
||||
'test/e2e/app-dir/global-error/global-error.test.ts',
|
||||
'test/e2e/app-dir/hooks/hooks.test.ts',
|
||||
|
@ -672,6 +675,7 @@ const disabledTests = [
|
|||
'test/unit/esm-interpolate/esm-interpolate.test.tsx',
|
||||
'test/unit/find-config.test.ts',
|
||||
'test/unit/find-page-file.test.ts',
|
||||
'test/unit/get-module-build-info.test.ts',
|
||||
'test/unit/get-node-options-without-inspect.test.ts',
|
||||
'test/unit/get-page-static-infos.test.ts',
|
||||
'test/unit/get-project-dir.test.ts',
|
||||
|
|
21
run-tests.js
21
run-tests.js
|
@ -12,11 +12,11 @@ const { createNextInstall } = require('./test/lib/create-next-install')
|
|||
const glob = promisify(_glob)
|
||||
const exec = promisify(execOrig)
|
||||
|
||||
// Try to read an external array-based json to filter tests to be executed.
|
||||
// Try to read an external array-based json to filter tests to be allowed / or disallowed.
|
||||
// If process.argv contains a test to be executed, this'll append it to the list.
|
||||
const externalTestsFilterLists = process.env.NEXT_EXTERNAL_TESTS_FILTERS
|
||||
? require(process.env.NEXT_EXTERNAL_TESTS_FILTERS).enabledTests
|
||||
: []
|
||||
? require(process.env.NEXT_EXTERNAL_TESTS_FILTERS)
|
||||
: { enabledTests: [], disabledTests: [] }
|
||||
const timings = []
|
||||
const DEFAULT_NUM_RETRIES = os.platform() === 'win32' ? 2 : 1
|
||||
const DEFAULT_CONCURRENCY = 2
|
||||
|
@ -143,9 +143,7 @@ async function main() {
|
|||
|
||||
console.log('Running tests with concurrency:', concurrency)
|
||||
|
||||
let tests = process.argv
|
||||
.filter((arg) => arg.match(/\.test\.(js|ts|tsx)/))
|
||||
.concat(externalTestsFilterLists)
|
||||
let tests = process.argv.filter((arg) => arg.match(/\.test\.(js|ts|tsx)/))
|
||||
let prevTimings
|
||||
|
||||
if (tests.length === 0) {
|
||||
|
@ -190,6 +188,17 @@ async function main() {
|
|||
}
|
||||
}
|
||||
|
||||
// If there are external manifest contains list of tests, apply it to the test lists.
|
||||
// Specifically, we filters out `disabledTests` from named export of the manifest.
|
||||
if (externalTestsFilterLists?.disabledTests.length > 0) {
|
||||
tests = tests.filter(
|
||||
(test) =>
|
||||
!externalTestsFilterLists.disabledTests.some((disabled) =>
|
||||
disabled.includes(test)
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
let testNames = [
|
||||
...new Set(
|
||||
tests.map((f) => {
|
||||
|
|
Loading…
Reference in a new issue