Update tests running and add splitting by timings for Azure (#10214)
* Add splitting by timings for Azure * Add --timings flag for azure * Update timings API * Update timings var * Fix test directory not being reset before re-trying * Update to re-use CircleCI timing data * Bump concurrency for Azure * Remove extra logging * Update timeout for create-next-app tests Co-authored-by: Joe Haddad <timer150@gmail.com>
This commit is contained in:
parent
bf31b4ee8d
commit
c1cbad0155
3 changed files with 107 additions and 9 deletions
|
@ -62,5 +62,5 @@ jobs:
|
|||
displayName: 'Install dependencies'
|
||||
|
||||
- script: |
|
||||
node run-tests.js -g $(group)
|
||||
node run-tests.js -g $(group) --timings
|
||||
displayName: 'Run tests'
|
||||
|
|
112
run-tests.js
112
run-tests.js
|
@ -1,6 +1,7 @@
|
|||
const path = require('path')
|
||||
const _glob = require('glob')
|
||||
const fs = require('fs-extra')
|
||||
const fetch = require('node-fetch')
|
||||
const { promisify } = require('util')
|
||||
const { Sema } = require('async-sema')
|
||||
const { spawn, exec: execOrig } = require('child_process')
|
||||
|
@ -23,12 +24,62 @@ const timings = []
|
|||
|
||||
console.log('Running tests with concurrency:', concurrency)
|
||||
let tests = process.argv.filter(arg => arg.endsWith('.test.js'))
|
||||
let prevTimings
|
||||
|
||||
if (tests.length === 0) {
|
||||
tests = await glob('**/*.test.js', {
|
||||
nodir: true,
|
||||
cwd: path.join(__dirname, 'test'),
|
||||
})
|
||||
|
||||
if (outputTimings) {
|
||||
console.log('Fetching previous timings data')
|
||||
const metaRes = await fetch(
|
||||
`https://circleci.com/api/v1.1/project/github/zeit/next.js/`
|
||||
)
|
||||
|
||||
if (metaRes.ok) {
|
||||
const buildsMeta = await metaRes.json()
|
||||
let buildNumber
|
||||
|
||||
for (const build of buildsMeta) {
|
||||
if (
|
||||
build.status === 'success' &&
|
||||
build.build_parameters &&
|
||||
build.build_parameters.CIRCLE_JOB === 'test'
|
||||
) {
|
||||
buildNumber = build.build_num
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
const timesRes = await fetch(
|
||||
`https://circleci.com/api/v1.1/project/github/zeit/next.js/${buildNumber}/tests`
|
||||
)
|
||||
|
||||
if (timesRes.ok) {
|
||||
const { tests } = await timesRes.json()
|
||||
prevTimings = {}
|
||||
|
||||
for (const test of tests) {
|
||||
prevTimings[test.file] = test.run_time
|
||||
}
|
||||
|
||||
if (Object.keys(prevTimings).length > 0) {
|
||||
console.log('Fetched previous timings data')
|
||||
} else {
|
||||
prevTimings = null
|
||||
}
|
||||
} else {
|
||||
console.log(
|
||||
'Failed to fetch previous timings status:',
|
||||
timesRes.status
|
||||
)
|
||||
}
|
||||
} else {
|
||||
console.log('Failed to fetch timings meta status:', metaRes.status)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let testNames = [
|
||||
|
@ -49,9 +100,43 @@ const timings = []
|
|||
let offset = groupPos === 1 ? 0 : (groupPos - 1) * numPerGroup - 1
|
||||
// if there's an odd number of suites give the first group the extra
|
||||
if (testNames.length % 2 !== 0 && groupPos !== 1) offset++
|
||||
testNames = testNames.splice(offset, numPerGroup)
|
||||
}
|
||||
|
||||
if (prevTimings) {
|
||||
const groups = [[]]
|
||||
const groupTimes = [0]
|
||||
|
||||
for (const testName of testNames) {
|
||||
let smallestGroup = groupTimes[0]
|
||||
let smallestGroupIdx = 0
|
||||
|
||||
// get the samllest group time to add current one to
|
||||
for (let i = 1; i < groupTotal; i++) {
|
||||
if (!groups[i]) {
|
||||
groups[i] = []
|
||||
groupTimes[i] = 0
|
||||
}
|
||||
|
||||
const time = groupTimes[i]
|
||||
if (time < smallestGroup) {
|
||||
smallestGroup = time
|
||||
smallestGroupIdx = i
|
||||
}
|
||||
}
|
||||
groups[smallestGroupIdx].push(testName)
|
||||
groupTimes[smallestGroupIdx] += prevTimings[testName] || 1
|
||||
}
|
||||
|
||||
const curGroupIdx = groupPos - 1
|
||||
testNames = groups[curGroupIdx]
|
||||
|
||||
console.log(
|
||||
'Current group previous accumulated times:',
|
||||
Math.round(groupTimes[curGroupIdx]) + 's'
|
||||
)
|
||||
} else {
|
||||
testNames = testNames.splice(offset, numPerGroup)
|
||||
}
|
||||
}
|
||||
console.log('Running tests:', '\n', ...testNames.map(name => `${name}\n`))
|
||||
|
||||
const sema = new Sema(concurrency, { capacity: testNames.length })
|
||||
|
@ -61,7 +146,7 @@ const timings = []
|
|||
)
|
||||
const children = new Set()
|
||||
|
||||
const runTest = (test = '') =>
|
||||
const runTest = (test = '', usePolling) =>
|
||||
new Promise((resolve, reject) => {
|
||||
const start = new Date().getTime()
|
||||
const child = spawn(
|
||||
|
@ -69,6 +154,17 @@ const timings = []
|
|||
[jestPath, '--runInBand', '--forceExit', '--verbose', test],
|
||||
{
|
||||
stdio: 'inherit',
|
||||
env: {
|
||||
...process.env,
|
||||
...(usePolling
|
||||
? {
|
||||
// Events can be finicky in CI. This switches to a more reliable
|
||||
// polling method.
|
||||
CHOKIDAR_USEPOLLING: 'true',
|
||||
CHOKIDAR_INTERVAL: 500,
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
}
|
||||
)
|
||||
children.add(child)
|
||||
|
@ -86,7 +182,7 @@ const timings = []
|
|||
|
||||
for (let i = 0; i < NUM_RETRIES + 1; i++) {
|
||||
try {
|
||||
const time = await runTest(test)
|
||||
const time = await runTest(test, i > 0)
|
||||
timings.push({
|
||||
file: test,
|
||||
time,
|
||||
|
@ -96,9 +192,10 @@ const timings = []
|
|||
} catch (err) {
|
||||
if (i < NUM_RETRIES) {
|
||||
try {
|
||||
console.log('Cleaning test files for', test)
|
||||
await exec(`git clean -fdx "${path.join(__dirname, test)}"`)
|
||||
await exec(`git checkout "${path.join(__dirname, test)}"`)
|
||||
const testDir = path.dirname(path.join(__dirname, test))
|
||||
console.log('Cleaning test files at', testDir)
|
||||
await exec(`git clean -fdx "${testDir}"`)
|
||||
await exec(`git checkout "${testDir}"`)
|
||||
} catch (err) {}
|
||||
}
|
||||
}
|
||||
|
@ -123,6 +220,7 @@ const timings = []
|
|||
|
||||
for (const timing of timings) {
|
||||
const timeInSeconds = timing.time / 1000
|
||||
|
||||
junitData += `
|
||||
<testsuite name="${timing.file}" file="${
|
||||
timing.file
|
||||
|
|
|
@ -20,7 +20,7 @@ const run = (...args) => execa('node', [cli, ...args], { cwd })
|
|||
|
||||
describe('create next app', () => {
|
||||
beforeAll(async () => {
|
||||
jest.setTimeout(1000 * 30)
|
||||
jest.setTimeout(1000 * 60)
|
||||
await mkdirp(cwd)
|
||||
})
|
||||
|
||||
|
|
Loading…
Reference in a new issue