ci(workflow): enable test trace upload (#51107)
### What? WEB-1150. This PR is an attempt to upload next.js's test results into datadog test trace to track its status. Originally it tried to use automatic trace injection (dd-trace/ci/init). However, due to some of custom environments / setup we use it was not compatible out of the box. Instead, this PR injects a new test reportert to generate junit report, then upload it at once at the end of the testing pipeline. The reporter is configured to run when necessary variables set, local run should not be affected. One thing to note is this report will not count retry results, as it'll create duplicated test entry with multiple results since we runts jest per individual test. This'll allow to detect flaky test easier, but also it means we'll get bit of skewed test results compare to the real world as first failure will be accounted as test fail immediately.
This commit is contained in:
parent
272ffffd59
commit
bcb63f3250
7 changed files with 81 additions and 28 deletions
22
.github/workflows/build_and_test.yml
vendored
22
.github/workflows/build_and_test.yml
vendored
|
@ -23,6 +23,7 @@ env:
|
||||||
# canary next-swc binaries in the monorepo
|
# canary next-swc binaries in the monorepo
|
||||||
NEXT_SKIP_NATIVE_POSTINSTALL: 1
|
NEXT_SKIP_NATIVE_POSTINSTALL: 1
|
||||||
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
|
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
|
||||||
|
DATADOG_TRACE_NEXTJS_TEST: 'true'
|
||||||
DD_ENV: 'ci'
|
DD_ENV: 'ci'
|
||||||
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
|
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
|
||||||
NEXT_TEST_JOB: 1
|
NEXT_TEST_JOB: 1
|
||||||
|
@ -181,6 +182,27 @@ jobs:
|
||||||
afterBuild: pnpm playwright install && BROWSER_NAME=firefox node run-tests.js test/integration/production/test/index.test.js && BROWSER_NAME=safari NEXT_TEST_MODE=start node run-tests.js -c 1 test/integration/production/test/index.test.js test/e2e/basepath.test.ts && BROWSER_NAME=safari DEVICE_NAME='iPhone XR' node run-tests.js -c 1 test/production/prerender-prefetch/index.test.ts
|
afterBuild: pnpm playwright install && BROWSER_NAME=firefox node run-tests.js test/integration/production/test/index.test.js && BROWSER_NAME=safari NEXT_TEST_MODE=start node run-tests.js -c 1 test/integration/production/test/index.test.js test/e2e/basepath.test.ts && BROWSER_NAME=safari DEVICE_NAME='iPhone XR' node run-tests.js -c 1 test/production/prerender-prefetch/index.test.ts
|
||||||
secrets: inherit
|
secrets: inherit
|
||||||
|
|
||||||
|
report-test-results:
|
||||||
|
needs: ['test-dev', 'test-prod', 'test-integration', 'test-turbopack-dev']
|
||||||
|
runs-on: [self-hosted, linux, x64]
|
||||||
|
name: upload test trace to datadog
|
||||||
|
# We'll not block the CI if the upload fails
|
||||||
|
continue-on-error: true
|
||||||
|
steps:
|
||||||
|
- uses: actions/download-artifact@v3
|
||||||
|
with:
|
||||||
|
name: Test trace reports
|
||||||
|
path: test
|
||||||
|
- name: Upload traces
|
||||||
|
run: |
|
||||||
|
ls -al ./test
|
||||||
|
npm install -g junit-report-merger@6.0.2 @datadog/datadog-ci@2.14.0
|
||||||
|
jrm ./nextjs-test-result-junit.xml "test/test-junit-report/**/*.xml"
|
||||||
|
jrm ./turbopack-test-result-junit.xml "test/turbopack-test-junit-report/**/*.xml"
|
||||||
|
# Put a separate tag for the tests with turbopack to distinguish between same test names
|
||||||
|
DD_ENV=ci datadog-ci junit upload --tags test.type:nextjs --service nextjs ./nextjs-test-result-junit.xml
|
||||||
|
DD_ENV=ci datadog-ci junit upload --tags test.type:turbopack --service nextjs ./turbopack-test-result-junit.xml
|
||||||
|
|
||||||
tests-pass:
|
tests-pass:
|
||||||
needs:
|
needs:
|
||||||
[
|
[
|
||||||
|
|
11
.github/workflows/build_reusable.yml
vendored
11
.github/workflows/build_reusable.yml
vendored
|
@ -53,6 +53,7 @@ env:
|
||||||
# canary next-swc binaries in the monorepo
|
# canary next-swc binaries in the monorepo
|
||||||
NEXT_SKIP_NATIVE_POSTINSTALL: 1
|
NEXT_SKIP_NATIVE_POSTINSTALL: 1
|
||||||
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
|
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
|
||||||
|
DATADOG_TRACE_NEXTJS_TEST: 'true'
|
||||||
DD_ENV: 'ci'
|
DD_ENV: 'ci'
|
||||||
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
|
TEST_TIMINGS_TOKEN: ${{ secrets.TEST_TIMINGS_TOKEN }}
|
||||||
NEXT_TEST_JOB: 1
|
NEXT_TEST_JOB: 1
|
||||||
|
@ -140,3 +141,13 @@ jobs:
|
||||||
with:
|
with:
|
||||||
name: turbo run summary
|
name: turbo run summary
|
||||||
path: .turbo/runs
|
path: .turbo/runs
|
||||||
|
|
||||||
|
- name: Upload test reports artifact
|
||||||
|
uses: actions/upload-artifact@v3
|
||||||
|
if: ${{ inputs.afterBuild }}
|
||||||
|
with:
|
||||||
|
name: Test trace reports
|
||||||
|
path: |
|
||||||
|
test/test-junit-report
|
||||||
|
test/turbopack-test-junit-report
|
||||||
|
if-no-files-found: ignore
|
||||||
|
|
|
@ -25,5 +25,33 @@ const customJestConfig = {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Check if the environment variable is set to enable test trace,
|
||||||
|
// Insert a reporter to generate a junit report to upload.
|
||||||
|
// This won't count for the retry to avoid duplicated test being reported twice
|
||||||
|
// - which means our test trace will report test results for the flaky test as failed without retry.
|
||||||
|
const shouldEnableTestTrace =
|
||||||
|
process.env.DATADOG_API_KEY &&
|
||||||
|
process.env.DATADOG_TRACE_NEXTJS_TEST &&
|
||||||
|
!process.env.IS_RETRY
|
||||||
|
|
||||||
|
if (shouldEnableTestTrace) {
|
||||||
|
if (!customJestConfig.reporters) {
|
||||||
|
customJestConfig.reporters = ['default']
|
||||||
|
}
|
||||||
|
|
||||||
|
const outputDirectory = process.env.__INTERNAL_NEXT_DEV_TEST_TURBO_DEV
|
||||||
|
? '<rootDir>/turbopack-test-junit-report'
|
||||||
|
: '<rootDir>/test-junit-report'
|
||||||
|
customJestConfig.reporters.push([
|
||||||
|
'jest-junit',
|
||||||
|
{
|
||||||
|
outputDirectory,
|
||||||
|
reportTestSuiteErrors: 'true',
|
||||||
|
uniqueOutputName: 'true',
|
||||||
|
outputName: 'nextjs-test-junit',
|
||||||
|
},
|
||||||
|
])
|
||||||
|
}
|
||||||
|
|
||||||
// createJestConfig is exported in this way to ensure that next/jest can load the Next.js config which is async
|
// createJestConfig is exported in this way to ensure that next/jest can load the Next.js config which is async
|
||||||
module.exports = createJestConfig(customJestConfig)
|
module.exports = createJestConfig(customJestConfig)
|
||||||
|
|
|
@ -152,6 +152,7 @@
|
||||||
"isomorphic-unfetch": "3.0.0",
|
"isomorphic-unfetch": "3.0.0",
|
||||||
"jest": "27.0.6",
|
"jest": "27.0.6",
|
||||||
"jest-extended": "1.2.1",
|
"jest-extended": "1.2.1",
|
||||||
|
"jest-junit": "16.0.0",
|
||||||
"json5": "2.2.3",
|
"json5": "2.2.3",
|
||||||
"ky": "0.19.1",
|
"ky": "0.19.1",
|
||||||
"ky-universal": "0.6.0",
|
"ky-universal": "0.6.0",
|
||||||
|
|
|
@ -111,6 +111,7 @@ importers:
|
||||||
isomorphic-unfetch: 3.0.0
|
isomorphic-unfetch: 3.0.0
|
||||||
jest: 27.0.6
|
jest: 27.0.6
|
||||||
jest-extended: 1.2.1
|
jest-extended: 1.2.1
|
||||||
|
jest-junit: 16.0.0
|
||||||
json5: 2.2.3
|
json5: 2.2.3
|
||||||
ky: 0.19.1
|
ky: 0.19.1
|
||||||
ky-universal: 0.6.0
|
ky-universal: 0.6.0
|
||||||
|
@ -277,6 +278,7 @@ importers:
|
||||||
isomorphic-unfetch: 3.0.0
|
isomorphic-unfetch: 3.0.0
|
||||||
jest: 27.0.6
|
jest: 27.0.6
|
||||||
jest-extended: 1.2.1
|
jest-extended: 1.2.1
|
||||||
|
jest-junit: 16.0.0
|
||||||
json5: 2.2.3
|
json5: 2.2.3
|
||||||
ky: 0.19.1
|
ky: 0.19.1
|
||||||
ky-universal: 0.6.0_ky@0.19.1
|
ky-universal: 0.6.0_ky@0.19.1
|
||||||
|
@ -15600,6 +15602,16 @@ packages:
|
||||||
- supports-color
|
- supports-color
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
|
/jest-junit/16.0.0:
|
||||||
|
resolution: {integrity: sha512-A94mmw6NfJab4Fg/BlvVOUXzXgF0XIH6EmTgJ5NDPp4xoKq0Kr7sErb+4Xs9nZvu58pJojz5RFGpqnZYJTrRfQ==}
|
||||||
|
engines: {node: '>=10.12.0'}
|
||||||
|
dependencies:
|
||||||
|
mkdirp: 1.0.4
|
||||||
|
strip-ansi: 6.0.1
|
||||||
|
uuid: 8.3.2
|
||||||
|
xml: 1.0.1
|
||||||
|
dev: true
|
||||||
|
|
||||||
/jest-leak-detector/27.0.6:
|
/jest-leak-detector/27.0.6:
|
||||||
resolution: {integrity: sha512-2/d6n2wlH5zEcdctX4zdbgX8oM61tb67PQt4Xh8JFAIy6LRKUnX528HulkaG6nD5qDl5vRV1NXejCe1XRCH5gQ==}
|
resolution: {integrity: sha512-2/d6n2wlH5zEcdctX4zdbgX8oM61tb67PQt4Xh8JFAIy6LRKUnX528HulkaG6nD5qDl5vRV1NXejCe1XRCH5gQ==}
|
||||||
engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0}
|
engines: {node: ^10.13.0 || ^12.13.0 || ^14.15.0 || >=15.0.0}
|
||||||
|
@ -25397,6 +25409,10 @@ packages:
|
||||||
resolution: {integrity: sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==}
|
resolution: {integrity: sha512-A5CUptxDsvxKJEU3yO6DuWBSJz/qizqzJKOMIfUJHETbBw/sFaDxgd6fxm1ewUaM0jZ444Fc5vC5ROYurg/4Pw==}
|
||||||
dev: true
|
dev: true
|
||||||
|
|
||||||
|
/xml/1.0.1:
|
||||||
|
resolution: {integrity: sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw==}
|
||||||
|
dev: true
|
||||||
|
|
||||||
/xmlchars/2.2.0:
|
/xmlchars/2.2.0:
|
||||||
resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==}
|
resolution: {integrity: sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==}
|
||||||
dev: true
|
dev: true
|
||||||
|
|
29
run-tests.js
29
run-tests.js
|
@ -322,33 +322,6 @@ async function main() {
|
||||||
|
|
||||||
const shouldRecordTestWithReplay = process.env.RECORD_REPLAY && isRetry
|
const shouldRecordTestWithReplay = process.env.RECORD_REPLAY && isRetry
|
||||||
|
|
||||||
let shouldEnableTestTrace = false
|
|
||||||
let traceInitPath = null
|
|
||||||
|
|
||||||
try {
|
|
||||||
traceInitPath = require.resolve('dd-trace/ci/init')
|
|
||||||
// Enable trace only if tracer (dd-trace) can be resolved
|
|
||||||
shouldEnableTestTrace =
|
|
||||||
process.env.DATADOG_API_KEY && process.env.DATADOG_TRACE_NEXTJS_TEST
|
|
||||||
} catch (e) {
|
|
||||||
shouldEnableTestTrace = false
|
|
||||||
}
|
|
||||||
|
|
||||||
const traceEnv =
|
|
||||||
shouldEnableTestTrace && traceInitPath
|
|
||||||
? {
|
|
||||||
DD_API_KEY: process.env.DATADOG_API_KEY,
|
|
||||||
DD_CIVISIBILITY_AGENTLESS_ENABLED: 1,
|
|
||||||
DD_ENV: 'ci',
|
|
||||||
DD_SERVICE: 'nextjs',
|
|
||||||
NODE_OPTIONS: `-r ${traceInitPath}`,
|
|
||||||
}
|
|
||||||
: {}
|
|
||||||
|
|
||||||
if (shouldEnableTestTrace) {
|
|
||||||
console.log(`Running test with Datadog tracing enabled`)
|
|
||||||
}
|
|
||||||
|
|
||||||
const child = spawn(
|
const child = spawn(
|
||||||
jestPath,
|
jestPath,
|
||||||
[
|
[
|
||||||
|
@ -368,7 +341,7 @@ async function main() {
|
||||||
stdio: ['ignore', 'pipe', 'pipe'],
|
stdio: ['ignore', 'pipe', 'pipe'],
|
||||||
env: {
|
env: {
|
||||||
...process.env,
|
...process.env,
|
||||||
...traceEnv,
|
IS_RETRY: isRetry ? 'true' : undefined,
|
||||||
RECORD_REPLAY: shouldRecordTestWithReplay,
|
RECORD_REPLAY: shouldRecordTestWithReplay,
|
||||||
// run tests in headless mode by default
|
// run tests in headless mode by default
|
||||||
HEADLESS: 'true',
|
HEADLESS: 'true',
|
||||||
|
|
2
test/.gitignore
vendored
2
test/.gitignore
vendored
|
@ -2,3 +2,5 @@
|
||||||
.vscode
|
.vscode
|
||||||
|
|
||||||
e2e/**/tsconfig.json
|
e2e/**/tsconfig.json
|
||||||
|
test-junit-report/
|
||||||
|
turbopack-test-junit-report/
|
Loading…
Reference in a new issue