From d91e78adb7d167dc167b8adc9c1335e5106c24e9 Mon Sep 17 00:00:00 2001 From: Tim Neutkens Date: Tue, 19 Mar 2024 10:26:14 +0100 Subject: [PATCH] Rename turbopack-tests-manifest to turbopack-dev-tests-manifest (#63409) ## What? - Renames the Turbopack tests manifest to reflect that it only holds development tests. - Creates initial plumbing for Turbopack build tests manifest (currently empty) - Added running tests in the Turbopack builds test manifest on PRs - Implements uploading the Turbopack builds manifest to areweturboyet What this doesn't implement: - Updating the Turbopack builds manifest Open questions: - Since the manifest is empty there are no test results, I had to add handling for that in `run-tests.js`: https://github.com/vercel/next.js/pull/63409/files#diff-269567847b110d8ecaaade3ab592df207bba02c7b446db23d72f18ff0e61d335R359 but not sure if that exit case was added for a specific special reason. Closes NEXT-2837 --- .github/CODEOWNERS | 4 +- .../actions/upload-turboyet-data/src/main.js | 103 ++++++++++++++---- .github/workflows/build_and_test.yml | 42 ++++++- .../turbopack-update-tests-manifest.yml | 2 +- .../turbopack-upload-tests-manifest.yml | 2 +- run-tests.js | 5 +- ... => build-turbopack-dev-tests-manifest.js} | 2 +- test/turbopack-build-tests-manifest.json | 8 ++ ...json => turbopack-dev-tests-manifest.json} | 0 9 files changed, 137 insertions(+), 31 deletions(-) rename test/{build-turbopack-tests-manifest.js => build-turbopack-dev-tests-manifest.js} (99%) create mode 100644 test/turbopack-build-tests-manifest.json rename test/{turbopack-tests-manifest.json => turbopack-dev-tests-manifest.json} (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 243795a3a1..23dbcc36d8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -41,5 +41,5 @@ Cargo.toml @timneutkens @i Cargo.lock @timneutkens @ijjk @shuding @huozhi @vercel/turbopack /.cargo/config.toml @timneutkens @ijjk @shuding @huozhi @vercel/turbopack /.config/nextest.toml @timneutkens @ijjk @shuding @huozhi @vercel/turbopack -/test/build-turbopack-tests-manifest.js @timneutkens @ijjk @shuding @huozhi @vercel/turbopack -/test/turbopack-tests-manifest.json @timneutkens @ijjk @shuding @huozhi @vercel/turbopack +/test/build-turbopack-dev-tests-manifest.js @timneutkens @ijjk @shuding @huozhi @vercel/turbopack +/test/turbopack-dev-tests-manifest.json @timneutkens @ijjk @shuding @huozhi @vercel/turbopack diff --git a/.github/actions/upload-turboyet-data/src/main.js b/.github/actions/upload-turboyet-data/src/main.js index f6171e4c4e..3c3284ad27 100644 --- a/.github/actions/upload-turboyet-data/src/main.js +++ b/.github/actions/upload-turboyet-data/src/main.js @@ -3,23 +3,23 @@ const path = require('path') const { createClient } = require('@vercel/kv') -async function main() { - try { - const file = path.join(process.cwd(), 'test/turbopack-tests-manifest.json') +async function collectResults(manifestFile) { + const file = path.join(process.cwd(), manifestFile) - let passingTests = '' - let failingTests = '' - let passCount = 0 - let failCount = 0 + let passingTests = '' + let failingTests = '' + let passCount = 0 + let failCount = 0 - const contents = await fs.readFile(file, 'utf-8') - const results = JSON.parse(contents) + const contents = await fs.readFile(file, 'utf-8') + const results = JSON.parse(contents) - const currentDate = new Date() - const isoString = currentDate.toISOString() - const timestamp = isoString.slice(0, 19).replace('T', ' ') + const currentDate = new Date() + const isoString = currentDate.toISOString() + const timestamp = isoString.slice(0, 19).replace('T', ' ') - for (const [testFileName, result] of Object.entries(results)) { + if (results.version === 2) { + for (const [testFileName, result] of Object.entries(results.suites)) { let suitePassCount = 0 let suiteFailCount = 0 @@ -56,25 +56,86 @@ async function main() { } } + const testRun = `${process.env.GITHUB_SHA}\t${timestamp}\t${passCount}/${ + passCount + failCount + }` + return { testRun, passingTests, failingTests } + } else { + for (const [testFileName, result] of Object.entries(results)) { + let suitePassCount = 0 + let suiteFailCount = 0 + + suitePassCount += result.passed.length + suiteFailCount += result.failed.length + + if (suitePassCount > 0) { + passingTests += `${testFileName}\n` + } + + if (suiteFailCount > 0) { + failingTests += `${testFileName}\n` + } + + for (const passed of result.passed) { + const passedName = passed.replaceAll('`', '\\`') + passingTests += `* ${passedName}\n` + } + + for (const passed of result.failed) { + const failedName = passed.replaceAll('`', '\\`') + failingTests += `* ${failedName}\n` + } + + passCount += suitePassCount + failCount += suiteFailCount + + if (suitePassCount > 0) { + passingTests += `\n` + } + + if (suiteFailCount > 0) { + failingTests += `\n` + } + } + const testRun = `${process.env.GITHUB_SHA}\t${timestamp}\t${passCount}/${ + passCount + failCount + }` + + return { testRun, passingTests, failingTests } + } +} + +async function main() { + try { + const developmentResult = await collectResults( + 'test/turbopack-dev-tests-manifest.json' + ) + + const productionResult = await collectResults( + 'test/turbopack-build-tests-manifest.json' + ) + const kv = createClient({ url: process.env.TURBOYET_KV_REST_API_URL, token: process.env.TURBOYET_KV_REST_API_TOKEN, }) - const testRun = `${process.env.GITHUB_SHA}\t${timestamp}\t${passCount}/${ - passCount + failCount - }` + console.log('TEST RESULT DEVELOPMENT') + console.log(developmentResult.testRun) - console.log('TEST RESULT') - console.log(testRun) + console.log('TEST RESULT PRODUCTION') + console.log(productionResult.testRun) - await kv.rpush('test-runs', testRun) + await kv.rpush('test-runs', developmentResult.testRun) + await kv.rpush('test-runs-production', productionResult.testRun) console.log('SUCCESSFULLY SAVED RUNS') - await kv.set('passing-tests', passingTests) + await kv.set('passing-tests', developmentResult.passingTests) + await kv.set('passing-tests-production', productionResult.passingTests) console.log('SUCCESSFULLY SAVED PASSING') - await kv.set('failing-tests', failingTests) + await kv.set('failing-tests', developmentResult.failingTests) + await kv.set('failing-tests-production', productionResult.failingTests) console.log('SUCCESSFULLY SAVED FAILING') } catch (error) { console.log(error) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 3668499c1f..f5a08acd5b 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -163,12 +163,12 @@ jobs: group: [1/5, 2/5, 3/5, 4/5, 5/5] uses: ./.github/workflows/build_reusable.yml with: - afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_MODE=dev node run-tests.js --test-pattern '^(test\/(development|e2e))/.*\.test\.(js|jsx|ts|tsx)$' --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} + afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-dev-tests-manifest.json" TURBOPACK=1 NEXT_E2E_TEST_TIMEOUT=240000 NEXT_TEST_MODE=dev node run-tests.js --test-pattern '^(test\/(development|e2e))/.*\.test\.(js|jsx|ts|tsx)$' --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} stepName: 'test-turbopack-dev-${{ matrix.group }}' secrets: inherit test-turbopack-integration: - name: test turbopack integration + name: test turbopack development integration needs: ['changes', 'build-next'] if: ${{ needs.changes.outputs.docs-only == 'false' }} @@ -179,10 +179,42 @@ jobs: uses: ./.github/workflows/build_reusable.yml with: nodeVersion: 18.17.0 - afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-tests-manifest.json" TURBOPACK=1 node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type integration + afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-dev-tests-manifest.json" TURBOPACK=1 node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type integration stepName: 'test-turbopack-integration-${{ matrix.group }}' secrets: inherit + test-turbopack-production: + name: test turbopack production + needs: ['changes', 'build-next'] + if: ${{ needs.changes.outputs.docs-only == 'false' }} + + strategy: + fail-fast: false + matrix: + group: [1/5, 2/5, 3/5, 4/5, 5/5] + uses: ./.github/workflows/build_reusable.yml + with: + nodeVersion: 18.17.0 + afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-build-tests-manifest.json" TURBOPACK=1 TURBOPACK_BUILD=1 node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type start + stepName: 'test-turbopack-production-${{ matrix.group }}' + secrets: inherit + + test-turbopack-production-integration: + name: test turbopack production integration + needs: ['changes', 'build-next'] + if: ${{ needs.changes.outputs.docs-only == 'false' }} + + strategy: + fail-fast: false + matrix: + group: [1/5, 2/5, 3/5, 4/5, 5/5] + uses: ./.github/workflows/build_reusable.yml + with: + nodeVersion: 18.17.0 + afterBuild: RUST_BACKTRACE=0 NEXT_EXTERNAL_TESTS_FILTERS="$(pwd)/test/turbopack-build-tests-manifest.json" TURBOPACK=1 TURBOPACK_BUILD=1 node run-tests.js --timings -g ${{ matrix.group }} -c ${TEST_CONCURRENCY} --type integration + stepName: 'test-turbopack-production-integration-${{ matrix.group }}' + secrets: inherit + test-next-swc-wasm: name: test next-swc wasm needs: ['changes', 'build-next'] @@ -339,7 +371,7 @@ jobs: stepName: 'test-ppr-prod-${{ matrix.group }}' secrets: inherit - report-test-results: + report-test-results-to-datadog: needs: [ 'changes', @@ -352,6 +384,8 @@ jobs: 'test-ppr-integration', 'test-turbopack-dev', 'test-turbopack-integration', + 'test-turbopack-production', + 'test-turbopack-production-integration', ] if: ${{ always() && needs.changes.outputs.docs-only == 'false' && !github.event.pull_request.head.repo.fork }} diff --git a/.github/workflows/turbopack-update-tests-manifest.yml b/.github/workflows/turbopack-update-tests-manifest.yml index d186bfd579..7b9b11277a 100644 --- a/.github/workflows/turbopack-update-tests-manifest.yml +++ b/.github/workflows/turbopack-update-tests-manifest.yml @@ -38,6 +38,6 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN_PULL_REQUESTS }} BRANCH_NAME: turbopack-manifest - SCRIPT: test/build-turbopack-tests-manifest.js + SCRIPT: test/build-turbopack-dev-tests-manifest.js PR_TITLE: Update Turbopack test manifest PR_BODY: This auto-generated PR updates the integration test manifest used when testing Turbopack. diff --git a/.github/workflows/turbopack-upload-tests-manifest.yml b/.github/workflows/turbopack-upload-tests-manifest.yml index 82b57b445c..fbc4f4be4c 100644 --- a/.github/workflows/turbopack-upload-tests-manifest.yml +++ b/.github/workflows/turbopack-upload-tests-manifest.yml @@ -29,7 +29,7 @@ jobs: # Always run build manifest script to get the latest value - run: | - node ./test/build-turbopack-tests-manifest.js + node ./test/build-turbopack-dev-tests-manifest.js env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN_PULL_REQUESTS }} diff --git a/run-tests.js b/run-tests.js index 4d33433610..4bb9acdd36 100644 --- a/run-tests.js +++ b/run-tests.js @@ -356,9 +356,12 @@ async function main() { } } + if (!tests) { + tests = [] + } + if (tests.length === 0) { console.log('No tests found for', options.type, 'exiting..') - return cleanUpAndExit(1) } console.log(`${GROUP}Running tests: diff --git a/test/build-turbopack-tests-manifest.js b/test/build-turbopack-dev-tests-manifest.js similarity index 99% rename from test/build-turbopack-tests-manifest.js rename to test/build-turbopack-dev-tests-manifest.js index f02afc6df8..bd49d3d703 100644 --- a/test/build-turbopack-tests-manifest.js +++ b/test/build-turbopack-dev-tests-manifest.js @@ -13,7 +13,7 @@ async function format(text) { const override = process.argv.includes('--override') -const PASSING_JSON_PATH = `${__dirname}/turbopack-tests-manifest.json` +const PASSING_JSON_PATH = `${__dirname}/turbopack-dev-tests-manifest.json` const WORKING_PATH = '/root/actions-runner/_work/next.js/next.js/' const INITIALIZING_TEST_CASES = [ diff --git a/test/turbopack-build-tests-manifest.json b/test/turbopack-build-tests-manifest.json new file mode 100644 index 0000000000..91e930d126 --- /dev/null +++ b/test/turbopack-build-tests-manifest.json @@ -0,0 +1,8 @@ +{ + "version": 2, + "suites": {}, + "rules": { + "include": [], + "exclude": [] + } +} diff --git a/test/turbopack-tests-manifest.json b/test/turbopack-dev-tests-manifest.json similarity index 100% rename from test/turbopack-tests-manifest.json rename to test/turbopack-dev-tests-manifest.json