diff --git a/.github/workflows/build_reusable.yml b/.github/workflows/build_reusable.yml index 6cc1bc92c6..34d45a0c6a 100644 --- a/.github/workflows/build_reusable.yml +++ b/.github/workflows/build_reusable.yml @@ -49,6 +49,16 @@ on: required: true description: 'name of the step, to be used for the upload artifact unique key ' type: string + timeout_minutes: + description: 'Timeout in minutes' + required: false + type: number + default: 30 + runs_on_labels: + description: 'List of runner labels' + required: false + type: string + default: '["self-hosted", "linux", "x64", "metal"]' env: NAPI_CLI_VERSION: 2.14.7 @@ -69,15 +79,13 @@ env: DD_ENV: 'ci' TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }} NEXT_TEST_JOB: 1 + VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }} + VERCEL_TEST_TEAM: vtest314-next-e2e-tests jobs: build: - timeout-minutes: 30 - runs-on: - - 'self-hosted' - - 'linux' - - 'x64' - - 'metal' + timeout-minutes: ${{ inputs.timeout_minutes }} + runs-on: ${{ fromJson(inputs.runs_on_labels) }} outputs: input_step_key: ${{ steps.var.outputs.input_step_key }} @@ -90,7 +98,12 @@ jobs: script: | core.setOutput('input_step_key', '${{ inputs.stepName }}'.toLowerCase().replaceAll(/[/.]/g, '-').trim('-')); - - run: fnm use --install-if-missing ${{ inputs.nodeVersion || env.NODE_LTS_VERSION }} + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.nodeVersion || env.NODE_LTS_VERSION }} + check-latest: true + - run: node -v - run: corepack enable - run: pwd @@ -175,7 +188,7 @@ jobs: - run: turbo run get-test-timings -- --build ${{ github.sha }} - run: /bin/bash -c "${{ inputs.afterBuild }}" - timeout-minutes: 30 + timeout-minutes: ${{ inputs.timeout_minutes }} - name: Upload artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/test_e2e_deploy_release.yml b/.github/workflows/test_e2e_deploy_release.yml new file mode 100644 index 0000000000..5ef425c9a3 --- /dev/null +++ b/.github/workflows/test_e2e_deploy_release.yml @@ -0,0 +1,95 @@ +name: Test E2E (Vercel Deploy), scheduled + +on: + # run on every release/prerelease + release: + types: [published] + # allow triggering manually as well + workflow_dispatch: + +env: + VERCEL_TEST_TEAM: vtest314-next-e2e-tests + VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.BROKEN_DEPLOY_SLACK_WEBHOOK_URL }} + DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }} + DD_ENV: 'ci' + +jobs: + setup: + runs-on: ubuntu-latest + if: github.repository_owner == 'vercel' + steps: + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_LTS_VERSION }} + check-latest: true + + - name: Setup pnpm + run: corepack enable + + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 25 + + - name: Setup test project + run: | + pnpm install + pnpm run build + node scripts/run-e2e-test-project-reset.mjs + + test-deploy: + name: test deploy + needs: setup + uses: ./.github/workflows/build_reusable.yml + secrets: inherit + strategy: + fail-fast: true + matrix: + group: [1/5, 2/5, 3/5, 4/5, 5/5, 6/6] + with: + afterBuild: NEXT_TEST_MODE=deploy NEXT_EXTERNAL_TESTS_FILTERS="test/deploy-tests-manifest.json" node run-tests.js --timings -g ${{ matrix.group }} -c 2 --type e2e + skipNativeBuild: 'yes' + stepName: 'test-deploy-${{ matrix.group }}' + timeout_minutes: 180 + runs_on_labels: '["ubuntu-latest"]' + + report-test-results-to-datadog: + needs: test-deploy + if: ${{ always() }} + + runs-on: ubuntu-latest + name: report test results to datadog + steps: + - name: Download test report artifacts + id: download-test-reports + uses: actions/download-artifact@v4 + with: + pattern: test-reports-* + path: test + merge-multiple: true + + - name: Upload test report to datadog + run: | + if [ -d ./test/test-junit-report ]; then + DD_ENV=ci npx @datadog/datadog-ci@2.23.1 junit upload --tags test.type:deploy --service nextjs ./test/test-junit-report + fi + + report-failure: + name: report failure to slack + needs: test-deploy + if: needs.test-deploy.conclusion == 'failure' + runs-on: ubuntu-latest + steps: + - name: send webhook + uses: slackapi/slack-github-action@v1.25.0 + with: + payload: | + { + "commit_title": ${{ toJSON(github.event.workflow_run.display_title) }}, + "commit_url": "github.com/${{ github.repository }}/commit/${{ github.event.workflow_run.head_sha }}", + "workflow_run_url": "github.com/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/attempts/${{ github.event.workflow_run.run_attempt }}" + } + env: + SLACK_WEBHOOK_URL: ${{ env.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/test_e2e_deploy_scheduled.yml b/.github/workflows/test_e2e_deploy_scheduled.yml deleted file mode 100644 index 02729a8c4a..0000000000 --- a/.github/workflows/test_e2e_deploy_scheduled.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: Test E2E (Vercel Deploy), scheduled - -on: - schedule: - # run every day at midnight - - cron: '0 0 * * *' - # allow triggering manually as well - workflow_dispatch: - -jobs: - test: - if: github.repository_owner == 'vercel' - runs-on: ubuntu-latest - - env: - CARGO_PROFILE_RELEASE_LTO: 'true' - DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }} - DD_ENV: 'ci' - NAPI_CLI_VERSION: 2.16.2 - NEXT_JUNIT_TEST_REPORT: 'true' - NEXT_TELEMETRY_DISABLED: 1 - NEXT_TEST_CONTINUE_ON_ERROR: 1 - NEXT_TEST_JOB: 1 - NEXT_TEST_MODE: 'deploy' - NODE_LTS_VERSION: 20 - TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }} - TURBO_REMOTE_ONLY: 'true' - TURBO_TEAM: 'vercel' - TURBO_VERSION: 1.13.3-canary.2 - VERCEL_TEST_TEAM: vtest314-next-e2e-tests - VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }} - - strategy: - fail-fast: false - matrix: - group: [1, 2] - - steps: - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: ${{ env.NODE_LTS_VERSION }} - check-latest: true - - - name: Setup pnpm - run: corepack enable - - - name: Checkout - uses: actions/checkout@v4 - with: - fetch-depth: 25 - - - name: Setup tests - run: | - pnpm install - pnpm run build - npm i -g vercel@latest - node scripts/run-e2e-test-project-reset.mjs - - - name: Run tests - run: | - docker run --rm -v $(pwd):/work mcr.microsoft.com/playwright:v1.41.2-jammy /bin/bash -c "cd /work && \ - NODE_VERSION=${{ env.NODE_LTS_VERSION }} ./scripts/setup-node.sh && \ - corepack enable > /dev/null && \ - NEXT_JUNIT_TEST_REPORT=${{ env.NEXT_JUNIT_TEST_REPORT }} \ - DATADOG_API_KEY=${{ env.DATADOG_API_KEY }} \ - DD_ENV=${{ env.DD_ENV }} \ - VERCEL_TEST_TOKEN=${{ env.VERCEL_TEST_TOKEN }} \ - VERCEL_TEST_TEAM=${{ env.VERCEL_TEST_TEAM }} \ - NEXT_TEST_JOB=${{ env.NEXT_TEST_JOB }} \ - NEXT_TEST_MODE=${{ env.NEXT_TEST_MODE }} \ - TEST_TIMINGS_TOKEN=${{ env.TEST_TIMINGS_TOKEN }} \ - xvfb-run node run-tests.js --type e2e --timings -g ${{ matrix.group }}/2 -c 1 >> /proc/1/fd/1" - - - name: Save test report as artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: test-reports - if-no-files-found: ignore - retention-days: 2 - path: test/test-junit-report - - - name: Upload test report to Datadog - continue-on-error: true - run: | - pnpx @datadog/datadog-ci@2.23.1 junit upload --tags test.type:nextjs_deploy_e2e --service nextjs ./test/test-junit-report diff --git a/run-tests.js b/run-tests.js index 98dd35e5de..412f429ee1 100644 --- a/run-tests.js +++ b/run-tests.js @@ -227,7 +227,12 @@ async function main() { } } - console.log('Running tests with concurrency:', options.concurrency) + console.log( + 'Running tests with concurrency:', + options.concurrency, + 'in test mode', + process.env.NEXT_TEST_MODE + ) /** @type TestFile[] */ let tests = argv._.filter((arg) => @@ -470,7 +475,8 @@ ${ENDGROUP}`) RECORD_REPLAY: shouldRecordTestWithReplay, // run tests in headless mode by default HEADLESS: 'true', - TRACE_PLAYWRIGHT: 'true', + TRACE_PLAYWRIGHT: + process.env.NEXT_TEST_MODE === 'deploy' ? undefined : 'true', NEXT_TELEMETRY_DISABLED: '1', // unset CI env so CI behavior is only explicitly // tested when enabled @@ -691,32 +697,6 @@ ${ENDGROUP}`) } } - // Emit test output if test failed or if we're continuing tests on error - if ((!passed || shouldContinueTestsOnError) && isTestJob) { - try { - const testsOutput = await fsp.readFile( - `${test.file}${RESULTS_EXT}`, - 'utf8' - ) - const obj = JSON.parse(testsOutput) - obj.processEnv = { - NEXT_TEST_MODE: process.env.NEXT_TEST_MODE, - HEADLESS: process.env.HEADLESS, - } - await outputSema.acquire() - if (GROUP) console.log(`${GROUP}Result as JSON for tooling`) - console.log( - `--test output start--`, - JSON.stringify(obj), - `--test output end--` - ) - if (ENDGROUP) console.log(ENDGROUP) - outputSema.release() - } catch (err) { - console.log(`Failed to load test output`, err) - } - } - sema.release() if (dirSema) dirSema.release() }) diff --git a/test/deploy-tests-manifest.json b/test/deploy-tests-manifest.json new file mode 100644 index 0000000000..62e9475d67 --- /dev/null +++ b/test/deploy-tests-manifest.json @@ -0,0 +1,80 @@ +{ + "version": 2, + "suites": {}, + "rules": { + "include": [ + "test/e2e/**/*.test.{t,j}s{,x}", + "test/production/**/*.test.{t,j}s{,x}" + ], + "exclude": [ + "test/e2e/app-dir/app-client-cache/client-cache.original.test.ts", + "test/e2e/app-dir/app-routes/app-custom-routes.test.ts", + "test/e2e/app-dir/next-after-app/index.test.ts", + "test/e2e/app-dir/scss/nm-module-nested/nm-module-nested.test.ts", + "test/e2e/cancel-request/stream-cancel.test.ts", + "test/e2e/edge-pages-support/edge-document.test.ts", + "test/e2e/new-link-behavior/material-ui.test.ts", + "test/e2e/react-dnd-compile/react-dnd-compile.test.ts", + "test/e2e/next-test/next-test.test.ts", + "test/e2e/skip-trailing-slash-redirect/index.test.ts", + "test/e2e/tsconfig-module-preserve/index.test.ts", + "test/e2e/app-dir/app-compilation/index.test.ts", + "test/e2e/app-dir/parallel-route-not-found-params/parallel-route-not-found-params.test.ts", + "test/e2e/app-dir/ppr-navigations/loading-tsx-no-partial-rendering/loading-tsx-no-partial-rendering.test.ts", + "test/e2e/app-dir/ppr/ppr.test.ts", + "test/e2e/app-dir/rsc-webpack-loader/rsc-webpack-loader.test.ts", + "test/e2e/app-dir/scss/compilation-and-prefixing/compilation-and-prefixing.test.ts", + "test/e2e/app-dir/typeof-window/typeof-window.test.ts", + "test/e2e/app-dir/webpack-loader-conditions/webpack-loader-conditions.test.ts", + "test/e2e/app-dir/x-forwarded-headers/x-forwarded-headers.test.ts", + "test/e2e/edge-compiler-module-exports-preference/index.test.ts", + "test/e2e/swc-warnings/index.test.ts", + "test/e2e/third-parties/index.test.ts", + "test/e2e/useselectedlayoutsegment-s-in-pages-router/useselectedlayoutsegment-s-in-pages-router.test.ts", + "test/e2e/app-dir/app-prefetch-false-loading/app-prefetch-false-loading.test.ts", + "test/e2e/app-dir/app-client-cache/client-cache.experimental.test.ts", + "test/e2e/app-dir/app-routes-client-component/app-routes-client-component.test.ts", + "test/e2e/app-dir/app-routes/app-custom-route-base-path.test.ts", + "test/e2e/app-dir/mdx/mdx.test.ts", + "test/e2e/app-dir/modularizeimports/modularizeimports.test.ts", + "test/e2e/app-dir/navigation/navigation.test.ts", + "test/e2e/app-dir/parallel-routes-and-interception/parallel-routes-and-interception.test.ts", + "test/e2e/app-dir/ppr-navigations/avoid-popstate-flash/avoid-popstate-flash.test.ts", + "test/e2e/app-dir/server-components-externals/index.test.ts", + "test/e2e/app-dir/third-parties/basic.test.ts", + "test/e2e/edge-can-read-request-body/index.test.ts", + "test/e2e/esm-externals/esm-externals.test.ts", + "test/e2e/i18n-data-route/i18n-data-route.test.ts", + "test/e2e/next-phase/index.test.ts", + "test/e2e/app-dir/actions-navigation/index.test.ts", + "test/e2e/app-dir/app-static/app-static-custom-handler.test.ts", + "test/e2e/app-dir/conflicting-page-segments/conflicting-page-segments.test.ts", + "test/e2e/app-dir/interception-route-prefetch-cache/interception-route-prefetch-cache.test.ts", + "test/e2e/app-dir/missing-suspense-with-csr-bailout/missing-suspense-with-csr-bailout.test.ts", + "test/e2e/app-dir/options-request/options-request.test.ts", + "test/e2e/app-dir/next-image/next-image-proxy.test.ts", + "test/e2e/app-dir/ppr-navigations/stale-prefetch-entry/stale-prefetch-entry.test.ts", + "test/e2e/app-dir/parallel-routes-revalidation/parallel-routes-revalidation.test.ts", + "test/e2e/app-dir/revalidate-dynamic/revalidate-dynamic.test.ts", + "test/e2e/app-dir/scss/npm-import-nested/npm-import-nested.test.ts", + "test/e2e/app-dir/syntax-highlighter-crash/syntax-highlighter-crash.test.ts", + "test/e2e/favicon-short-circuit/favicon-short-circuit.test.ts", + "test/e2e/new-link-behavior/stitches.test.ts", + "test/e2e/next-image-forward-ref/index.test.ts", + "test/e2e/react-compiler/react-compiler.test.ts", + "test/e2e/revalidate-reason/revalidate-reason.test.ts", + "test/e2e/app-dir/app-static/app-static.test.ts", + "test/e2e/app-dir/actions/app-action.test.ts", + "test/e2e/app-dir/i18n-hybrid/i18n-hybrid.test.js", + "test/e2e/app-dir/metadata/metadata.test.ts", + "test/e2e/app-dir/rsc-basic/rsc-basic.test.ts", + "test/e2e/app-dir/scss/nm-module/nm-module.test.ts", + "test/e2e/app-dir/static-shell-debugging/static-shell-debugging.test.ts", + "test/e2e/basepath.test.ts", + "test/e2e/postcss-config-cjs/index.test.ts", + "test/e2e/socket-io/index.test.js", + "test/e2e/middleware-matcher/index.test.ts", + "test/e2e/next-script/index.test.ts" + ] + } +} diff --git a/test/get-test-filter.js b/test/get-test-filter.js index 76840e2e6d..bf6cc21852 100644 --- a/test/get-test-filter.js +++ b/test/get-test-filter.js @@ -7,6 +7,11 @@ function getTestFilter() { : null if (!manifest) return null + console.log( + 'Filtering tests using manifest:', + process.env.NEXT_EXTERNAL_TESTS_FILTERS + ) + // For the legacy manifest without a version, we assume it's a complete list // of all the tests. if (!manifest.version || typeof manifest.version !== 'number') { diff --git a/test/lib/next-modes/next-deploy.ts b/test/lib/next-modes/next-deploy.ts index 78ecf83673..007a2ea94c 100644 --- a/test/lib/next-modes/next-deploy.ts +++ b/test/lib/next-modes/next-deploy.ts @@ -76,6 +76,7 @@ export class NextDeployInstance extends NextInstance { { cwd: this.testDir, env: vercelEnv, + reject: false, } ) @@ -117,12 +118,13 @@ export class NextDeployInstance extends NextInstance { { cwd: this.testDir, env: vercelEnv, + reject: false, } ) if (deployRes.exitCode !== 0) { throw new Error( - `Failed to deploy project ${linkRes.stdout} ${linkRes.stderr} (${linkRes.exitCode})` + `Failed to deploy project ${deployRes.stdout} ${deployRes.stderr} (${deployRes.exitCode})` ) } // the CLI gives just the deployment URL back when not a TTY @@ -151,6 +153,7 @@ export class NextDeployInstance extends NextInstance { ['logs', this._url, '--output', 'raw', ...vercelFlags], { env: vercelEnv, + reject: false, } ) if (logs.exitCode !== 0) {