Update Turbopack test manifest from GitHub Actions artifact (#58394)
### What? We can use the GitHub actions artifact (which is already produced right now) instead of a separate git branch to get the latest test results. This also means we don't have a dependency back to the turbo repo for the daily tests. Closes PACK-1951
This commit is contained in:
parent
b8a18f6e13
commit
2a8f7ae1b1
9 changed files with 26163 additions and 241 deletions
10
.github/actions/next-integration-stat/action.yml
vendored
10
.github/actions/next-integration-stat/action.yml
vendored
|
@ -3,20 +3,18 @@ author: Turbopack team
|
||||||
description: 'Display next.js integration test failure status'
|
description: 'Display next.js integration test failure status'
|
||||||
|
|
||||||
inputs:
|
inputs:
|
||||||
# Github token to use to create test report comment. If not specified, the default token will be used with username 'github-actions'
|
|
||||||
token:
|
token:
|
||||||
default: ${{ github.token }}
|
default: ${{ github.token }}
|
||||||
|
description: 'GitHub token used to create the test report comment. If not specified, the default GitHub actions token will be used'
|
||||||
|
|
||||||
# The base of the test results to compare against. If not specified, will try to compare with latest main branch's test results.
|
|
||||||
diff_base:
|
diff_base:
|
||||||
default: 'main'
|
default: 'main'
|
||||||
|
description: "The base of the test results to compare against. If not specified, will try to compare with latest main branch's test results."
|
||||||
|
|
||||||
# Include full test failure message in the report.
|
|
||||||
# This is currently disabled as we have too many failed test cases, causes
|
|
||||||
# too many report comment generated.
|
|
||||||
expand_full_result_message:
|
expand_full_result_message:
|
||||||
default: 'false'
|
default: 'false'
|
||||||
|
description: 'Whether to include the full test failure message in the report. This is currently disabled as we have too many failed test cases, which would lead to massive comments.'
|
||||||
|
|
||||||
runs:
|
runs:
|
||||||
using: node16
|
using: node20
|
||||||
main: index.js
|
main: index.js
|
||||||
|
|
26138
.github/actions/next-integration-stat/index.js
vendored
26138
.github/actions/next-integration-stat/index.js
vendored
File diff suppressed because one or more lines are too long
|
@ -8,7 +8,6 @@
|
||||||
"lint:prettier": "prettier -c . --cache --ignore-path=../../../.prettierignore"
|
"lint:prettier": "prettier -c . --cache --ignore-path=../../../.prettierignore"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@turbo/eslint-config": "workspace:*",
|
|
||||||
"@types/node": "^18.11.18",
|
"@types/node": "^18.11.18",
|
||||||
"@vercel/ncc": "0.34.0",
|
"@vercel/ncc": "0.34.0",
|
||||||
"typescript": "^4.4.4"
|
"typescript": "^4.4.4"
|
||||||
|
|
|
@ -6,41 +6,7 @@ const fs = require('fs')
|
||||||
const path = require('path')
|
const path = require('path')
|
||||||
const semver = require('semver')
|
const semver = require('semver')
|
||||||
|
|
||||||
/**
|
/// <reference path="./manifest" />
|
||||||
* Models parsed test results output from next.js integration test.
|
|
||||||
* This is a subset of the full test result output from jest, partially compatible.
|
|
||||||
*/
|
|
||||||
interface TestResult {
|
|
||||||
numFailedTestSuites: number
|
|
||||||
numFailedTests: number
|
|
||||||
numPassedTestSuites: number
|
|
||||||
numPassedTests: number
|
|
||||||
numPendingTestSuites: number
|
|
||||||
numPendingTests: number
|
|
||||||
numRuntimeErrorTestSuites: number
|
|
||||||
numTodoTests: number
|
|
||||||
numTotalTestSuites: number
|
|
||||||
numTotalTests: number
|
|
||||||
startTime: number
|
|
||||||
success: boolean
|
|
||||||
testResults?: Array<{
|
|
||||||
assertionResults?: Array<{
|
|
||||||
ancestorTitles?: Array<string> | null
|
|
||||||
failureMessages?: Array<string> | null
|
|
||||||
fullName: string
|
|
||||||
location?: null
|
|
||||||
status: string
|
|
||||||
title: string
|
|
||||||
}> | null
|
|
||||||
endTime: number
|
|
||||||
message: string
|
|
||||||
name: string
|
|
||||||
startTime: number
|
|
||||||
status: string
|
|
||||||
summary: string
|
|
||||||
}> | null
|
|
||||||
wasInterrupted: boolean
|
|
||||||
}
|
|
||||||
|
|
||||||
type Octokit = ReturnType<typeof getOctokit>
|
type Octokit = ReturnType<typeof getOctokit>
|
||||||
|
|
||||||
|
@ -53,18 +19,6 @@ type ExistingComment =
|
||||||
ReturnType<Octokit['rest']['issues']['listComments']>
|
ReturnType<Octokit['rest']['issues']['listComments']>
|
||||||
>['data'][number]
|
>['data'][number]
|
||||||
| undefined
|
| undefined
|
||||||
interface JobResult {
|
|
||||||
job: string
|
|
||||||
data: TestResult
|
|
||||||
}
|
|
||||||
interface TestResultManifest {
|
|
||||||
nextjsVersion: string
|
|
||||||
ref: string
|
|
||||||
buildTime?: string
|
|
||||||
buildSize?: string
|
|
||||||
result: Array<JobResult>
|
|
||||||
flakyMonitorJobResults: Array<JobResult>
|
|
||||||
}
|
|
||||||
|
|
||||||
// A comment marker to identify the comment created by this action.
|
// A comment marker to identify the comment created by this action.
|
||||||
const BOT_COMMENT_MARKER = `<!-- __marker__ next.js integration stats __marker__ -->`
|
const BOT_COMMENT_MARKER = `<!-- __marker__ next.js integration stats __marker__ -->`
|
||||||
|
|
49
.github/actions/next-integration-stat/src/manifest.d.ts
vendored
Normal file
49
.github/actions/next-integration-stat/src/manifest.d.ts
vendored
Normal file
|
@ -0,0 +1,49 @@
|
||||||
|
interface JobResult {
|
||||||
|
job: string
|
||||||
|
data: TestResult
|
||||||
|
}
|
||||||
|
|
||||||
|
interface TestResultManifest {
|
||||||
|
nextjsVersion: string
|
||||||
|
ref: string
|
||||||
|
buildTime?: string
|
||||||
|
buildSize?: string
|
||||||
|
result: Array<JobResult>
|
||||||
|
flakyMonitorJobResults: Array<JobResult>
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Models parsed test results output from next.js integration test.
|
||||||
|
* This is a subset of the full test result output from jest, partially compatible.
|
||||||
|
*/
|
||||||
|
interface TestResult {
|
||||||
|
numFailedTestSuites: number
|
||||||
|
numFailedTests: number
|
||||||
|
numPassedTestSuites: number
|
||||||
|
numPassedTests: number
|
||||||
|
numPendingTestSuites: number
|
||||||
|
numPendingTests: number
|
||||||
|
numRuntimeErrorTestSuites: number
|
||||||
|
numTodoTests: number
|
||||||
|
numTotalTestSuites: number
|
||||||
|
numTotalTests: number
|
||||||
|
startTime: number
|
||||||
|
success: boolean
|
||||||
|
testResults?: Array<{
|
||||||
|
assertionResults?: Array<{
|
||||||
|
ancestorTitles?: Array<string> | null
|
||||||
|
failureMessages?: Array<string> | null
|
||||||
|
fullName: string
|
||||||
|
location?: null
|
||||||
|
status: string
|
||||||
|
title: string
|
||||||
|
}> | null
|
||||||
|
endTime: number
|
||||||
|
message: string
|
||||||
|
name: string
|
||||||
|
startTime: number
|
||||||
|
status: string
|
||||||
|
summary: string
|
||||||
|
}> | null
|
||||||
|
wasInterrupted: boolean
|
||||||
|
}
|
|
@ -162,6 +162,7 @@
|
||||||
"jest-extended": "4.0.2",
|
"jest-extended": "4.0.2",
|
||||||
"jest-junit": "16.0.0",
|
"jest-junit": "16.0.0",
|
||||||
"json5": "2.2.3",
|
"json5": "2.2.3",
|
||||||
|
"kleur": "^4.1.0",
|
||||||
"ky": "0.19.1",
|
"ky": "0.19.1",
|
||||||
"ky-universal": "0.6.0",
|
"ky-universal": "0.6.0",
|
||||||
"lerna": "4.0.0",
|
"lerna": "4.0.0",
|
||||||
|
|
|
@ -332,6 +332,9 @@ importers:
|
||||||
json5:
|
json5:
|
||||||
specifier: 2.2.3
|
specifier: 2.2.3
|
||||||
version: 2.2.3
|
version: 2.2.3
|
||||||
|
kleur:
|
||||||
|
specifier: ^4.1.0
|
||||||
|
version: 4.1.3
|
||||||
ky:
|
ky:
|
||||||
specifier: 0.19.1
|
specifier: 0.19.1
|
||||||
version: 0.19.1
|
version: 0.19.1
|
||||||
|
|
28
test/build-turbopack-tests-manifest.d.ts
vendored
Normal file
28
test/build-turbopack-tests-manifest.d.ts
vendored
Normal file
|
@ -0,0 +1,28 @@
|
||||||
|
/// <reference path="../.github/actions/next-integration-stat/src/manifest.d.ts" />
|
||||||
|
|
||||||
|
type ListArtifactsResponse = {
|
||||||
|
total_count: number
|
||||||
|
artifacts: Artifact[]
|
||||||
|
}
|
||||||
|
|
||||||
|
type Artifact = {
|
||||||
|
id: number
|
||||||
|
node_id: string
|
||||||
|
name: string
|
||||||
|
size_in_bytes: number
|
||||||
|
url: string
|
||||||
|
archive_download_url: string
|
||||||
|
expired: false
|
||||||
|
created_at: string
|
||||||
|
expires_at: string
|
||||||
|
updated_at: string
|
||||||
|
workflow_run: WorkflowRun
|
||||||
|
}
|
||||||
|
|
||||||
|
type WorkflowRun = {
|
||||||
|
id: number
|
||||||
|
repository_id: number
|
||||||
|
head_repository_id: number
|
||||||
|
head_branch: string
|
||||||
|
head_sha: string
|
||||||
|
}
|
|
@ -1,6 +1,10 @@
|
||||||
const fetch = require('node-fetch')
|
|
||||||
const fs = require('fs')
|
const fs = require('fs')
|
||||||
|
const os = require('os')
|
||||||
|
const path = require('path')
|
||||||
|
|
||||||
const prettier = require('prettier')
|
const prettier = require('prettier')
|
||||||
|
const execa = require('execa')
|
||||||
|
const { bold } = require('kleur')
|
||||||
|
|
||||||
async function format(text) {
|
async function format(text) {
|
||||||
const options = await prettier.resolveConfig(__filename)
|
const options = await prettier.resolveConfig(__filename)
|
||||||
|
@ -9,10 +13,8 @@ async function format(text) {
|
||||||
|
|
||||||
const override = process.argv.includes('--override')
|
const override = process.argv.includes('--override')
|
||||||
|
|
||||||
const RESULT_URL =
|
|
||||||
'https://raw.githubusercontent.com/vercel/turbo/nextjs-integration-test-data/test-results/main/nextjs-test-results.json'
|
|
||||||
const PASSING_JSON_PATH = `${__dirname}/turbopack-tests-manifest.json`
|
const PASSING_JSON_PATH = `${__dirname}/turbopack-tests-manifest.json`
|
||||||
const WORKING_PATH = '/home/runner/work/turbo/turbo/'
|
const WORKING_PATH = '/root/actions-runner/_work/next.js/next.js/'
|
||||||
|
|
||||||
const INITIALIZING_TEST_CASES = [
|
const INITIALIZING_TEST_CASES = [
|
||||||
'compile successfully',
|
'compile successfully',
|
||||||
|
@ -106,11 +108,93 @@ const SKIPPED_TEST_SUITES = {
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
|
|
||||||
async function updatePassingTests() {
|
/**
|
||||||
const passing = { __proto__: null }
|
* @param title {string}
|
||||||
const res = await fetch(RESULT_URL)
|
* @param file {string}
|
||||||
const results = await res.json()
|
* @param args {readonly string[]}
|
||||||
|
* @returns {execa.ExecaChildProcess}
|
||||||
|
*/
|
||||||
|
function exec(title, file, args) {
|
||||||
|
logCommand(title, `${file} ${args.join(' ')}`)
|
||||||
|
|
||||||
|
return execa(file, args, {
|
||||||
|
stderr: 'inherit',
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param {string} title
|
||||||
|
* @param {string} [command]
|
||||||
|
*/
|
||||||
|
function logCommand(title, command) {
|
||||||
|
let message = `\n${bold().underline(title)}\n`
|
||||||
|
|
||||||
|
if (command) {
|
||||||
|
message += `> ${bold(command)}\n`
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log(message)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @returns {Promise<Artifact>}
|
||||||
|
*/
|
||||||
|
async function fetchLatestTestArtifact() {
|
||||||
|
const { stdout } = await exec(
|
||||||
|
'Getting latest test artifacts from GitHub actions',
|
||||||
|
'gh',
|
||||||
|
['api', '/repos/vercel/next.js/actions/artifacts?name=test-results']
|
||||||
|
)
|
||||||
|
|
||||||
|
/** @type {ListArtifactsResponse} */
|
||||||
|
const res = JSON.parse(stdout)
|
||||||
|
|
||||||
|
for (const artifact of res.artifacts) {
|
||||||
|
if (artifact.expired || artifact.workflow_run.head_branch !== 'canary') {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
return artifact
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error('no valid test-results artifact was found for branch canary')
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @returns {Promise<TestResultManifest>}
|
||||||
|
*/
|
||||||
|
async function fetchTestResults() {
|
||||||
|
const artifact = await fetchLatestTestArtifact()
|
||||||
|
|
||||||
|
const subprocess = exec('Downloading artifact archive', 'gh', [
|
||||||
|
'api',
|
||||||
|
`/repos/vercel/next.js/actions/artifacts/${artifact.id}/zip`,
|
||||||
|
])
|
||||||
|
|
||||||
|
const filePath = path.join(
|
||||||
|
os.tmpdir(),
|
||||||
|
`next-test-results.${Math.floor(Math.random() * 1000).toString(16)}.zip`
|
||||||
|
)
|
||||||
|
|
||||||
|
subprocess.stdout.pipe(fs.createWriteStream(filePath))
|
||||||
|
|
||||||
|
await subprocess
|
||||||
|
|
||||||
|
const { stdout } = await exec('Extracting test results manifest', 'unzip', [
|
||||||
|
'-pj',
|
||||||
|
filePath,
|
||||||
|
'nextjs-test-results.json',
|
||||||
|
])
|
||||||
|
|
||||||
|
return JSON.parse(stdout)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function updatePassingTests() {
|
||||||
|
const results = await fetchTestResults()
|
||||||
|
|
||||||
|
logCommand('Processing results...')
|
||||||
|
|
||||||
|
const passing = { __proto__: null }
|
||||||
for (const result of results.result) {
|
for (const result of results.result) {
|
||||||
const runtimeError = result.data.numRuntimeErrorTestSuites > 0
|
const runtimeError = result.data.numRuntimeErrorTestSuites > 0
|
||||||
for (const testResult of result.data.testResults) {
|
for (const testResult of result.data.testResults) {
|
||||||
|
@ -153,13 +237,11 @@ async function updatePassingTests() {
|
||||||
|
|
||||||
if (skippedPassingNames.length > 0) {
|
if (skippedPassingNames.length > 0) {
|
||||||
console.log(
|
console.log(
|
||||||
`${filepath} has ${
|
`${bold().red(filepath)} has ${
|
||||||
skippedPassingNames.length
|
skippedPassingNames.length
|
||||||
} passing tests that are marked as skipped: ${JSON.stringify(
|
} passing tests that are marked as skipped:\n${skippedPassingNames
|
||||||
skippedPassingNames,
|
.map((name) => ` - ${name}`)
|
||||||
0,
|
.join('\n')}\n`
|
||||||
2
|
|
||||||
)}`
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -190,9 +272,12 @@ async function updatePassingTests() {
|
||||||
oldData.passed.filter((name) => newData.failed.includes(name))
|
oldData.passed.filter((name) => newData.failed.includes(name))
|
||||||
)
|
)
|
||||||
if (shouldPass.size > 0) {
|
if (shouldPass.size > 0) {
|
||||||
const list = JSON.stringify([...shouldPass], 0, 2)
|
|
||||||
console.log(
|
console.log(
|
||||||
`${file} has ${shouldPass.size} test(s) that should pass but failed: ${list}`
|
`${bold().red(file)} has ${
|
||||||
|
shouldPass.size
|
||||||
|
} test(s) that should pass but failed:\n${Array.from(shouldPass)
|
||||||
|
.map((name) => ` - ${name}`)
|
||||||
|
.join('\n')}\n`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
// Merge the old passing tests with the new ones
|
// Merge the old passing tests with the new ones
|
||||||
|
@ -203,7 +288,9 @@ async function updatePassingTests() {
|
||||||
.sort()
|
.sort()
|
||||||
|
|
||||||
if (!oldData.runtimeError && newData.runtimeError) {
|
if (!oldData.runtimeError && newData.runtimeError) {
|
||||||
console.log(`${file} has a runtime error that is shouldn't have`)
|
console.log(
|
||||||
|
`${bold().red(file)} has a runtime error that is shouldn't have\n`
|
||||||
|
)
|
||||||
newData.runtimeError = false
|
newData.runtimeError = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -246,4 +333,7 @@ function stripWorkingPath(path) {
|
||||||
return path.slice(WORKING_PATH.length)
|
return path.slice(WORKING_PATH.length)
|
||||||
}
|
}
|
||||||
|
|
||||||
updatePassingTests()
|
updatePassingTests().catch((e) => {
|
||||||
|
console.error(e)
|
||||||
|
process.exit(1)
|
||||||
|
})
|
||||||
|
|
Loading…
Reference in a new issue