From c1cbad015575e11df0d14739dd6619b458fbabbe Mon Sep 17 00:00:00 2001 From: JJ Kasper Date: Thu, 23 Jan 2020 11:37:01 -0600 Subject: [PATCH] Update tests running and add splitting by timings for Azure (#10214) * Add splitting by timings for Azure * Add --timings flag for azure * Update timings API * Update timings var * Fix test directory not being reset before re-trying * Update to re-use CircleCI timing data * Bump concurrency for Azure * Remove extra logging * Update timeout for create-next-app tests Co-authored-by: Joe Haddad --- azure-pipelines.yml | 2 +- run-tests.js | 112 ++++++++++++++++-- .../integration/create-next-app/index.test.js | 2 +- 3 files changed, 107 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6f9f985a42..40e8b3f12e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -62,5 +62,5 @@ jobs: displayName: 'Install dependencies' - script: | - node run-tests.js -g $(group) + node run-tests.js -g $(group) --timings displayName: 'Run tests' diff --git a/run-tests.js b/run-tests.js index fce23f7b30..e527dda476 100644 --- a/run-tests.js +++ b/run-tests.js @@ -1,6 +1,7 @@ const path = require('path') const _glob = require('glob') const fs = require('fs-extra') +const fetch = require('node-fetch') const { promisify } = require('util') const { Sema } = require('async-sema') const { spawn, exec: execOrig } = require('child_process') @@ -23,12 +24,62 @@ const timings = [] console.log('Running tests with concurrency:', concurrency) let tests = process.argv.filter(arg => arg.endsWith('.test.js')) + let prevTimings if (tests.length === 0) { tests = await glob('**/*.test.js', { nodir: true, cwd: path.join(__dirname, 'test'), }) + + if (outputTimings) { + console.log('Fetching previous timings data') + const metaRes = await fetch( + `https://circleci.com/api/v1.1/project/github/zeit/next.js/` + ) + + if (metaRes.ok) { + const buildsMeta = await metaRes.json() + let buildNumber + + for (const build of buildsMeta) { + if ( + build.status === 'success' && + build.build_parameters && + build.build_parameters.CIRCLE_JOB === 'test' + ) { + buildNumber = build.build_num + break + } + } + + const timesRes = await fetch( + `https://circleci.com/api/v1.1/project/github/zeit/next.js/${buildNumber}/tests` + ) + + if (timesRes.ok) { + const { tests } = await timesRes.json() + prevTimings = {} + + for (const test of tests) { + prevTimings[test.file] = test.run_time + } + + if (Object.keys(prevTimings).length > 0) { + console.log('Fetched previous timings data') + } else { + prevTimings = null + } + } else { + console.log( + 'Failed to fetch previous timings status:', + timesRes.status + ) + } + } else { + console.log('Failed to fetch timings meta status:', metaRes.status) + } + } } let testNames = [ @@ -49,9 +100,43 @@ const timings = [] let offset = groupPos === 1 ? 0 : (groupPos - 1) * numPerGroup - 1 // if there's an odd number of suites give the first group the extra if (testNames.length % 2 !== 0 && groupPos !== 1) offset++ - testNames = testNames.splice(offset, numPerGroup) - } + if (prevTimings) { + const groups = [[]] + const groupTimes = [0] + + for (const testName of testNames) { + let smallestGroup = groupTimes[0] + let smallestGroupIdx = 0 + + // get the samllest group time to add current one to + for (let i = 1; i < groupTotal; i++) { + if (!groups[i]) { + groups[i] = [] + groupTimes[i] = 0 + } + + const time = groupTimes[i] + if (time < smallestGroup) { + smallestGroup = time + smallestGroupIdx = i + } + } + groups[smallestGroupIdx].push(testName) + groupTimes[smallestGroupIdx] += prevTimings[testName] || 1 + } + + const curGroupIdx = groupPos - 1 + testNames = groups[curGroupIdx] + + console.log( + 'Current group previous accumulated times:', + Math.round(groupTimes[curGroupIdx]) + 's' + ) + } else { + testNames = testNames.splice(offset, numPerGroup) + } + } console.log('Running tests:', '\n', ...testNames.map(name => `${name}\n`)) const sema = new Sema(concurrency, { capacity: testNames.length }) @@ -61,7 +146,7 @@ const timings = [] ) const children = new Set() - const runTest = (test = '') => + const runTest = (test = '', usePolling) => new Promise((resolve, reject) => { const start = new Date().getTime() const child = spawn( @@ -69,6 +154,17 @@ const timings = [] [jestPath, '--runInBand', '--forceExit', '--verbose', test], { stdio: 'inherit', + env: { + ...process.env, + ...(usePolling + ? { + // Events can be finicky in CI. This switches to a more reliable + // polling method. + CHOKIDAR_USEPOLLING: 'true', + CHOKIDAR_INTERVAL: 500, + } + : {}), + }, } ) children.add(child) @@ -86,7 +182,7 @@ const timings = [] for (let i = 0; i < NUM_RETRIES + 1; i++) { try { - const time = await runTest(test) + const time = await runTest(test, i > 0) timings.push({ file: test, time, @@ -96,9 +192,10 @@ const timings = [] } catch (err) { if (i < NUM_RETRIES) { try { - console.log('Cleaning test files for', test) - await exec(`git clean -fdx "${path.join(__dirname, test)}"`) - await exec(`git checkout "${path.join(__dirname, test)}"`) + const testDir = path.dirname(path.join(__dirname, test)) + console.log('Cleaning test files at', testDir) + await exec(`git clean -fdx "${testDir}"`) + await exec(`git checkout "${testDir}"`) } catch (err) {} } } @@ -123,6 +220,7 @@ const timings = [] for (const timing of timings) { const timeInSeconds = timing.time / 1000 + junitData += `