Update tests running and add splitting by timings for Azure (#10214)

* Add splitting by timings for Azure

* Add --timings flag for azure

* Update timings API

* Update timings var

* Fix test directory not being reset before re-trying

* Update to re-use CircleCI timing data

* Bump concurrency for Azure

* Remove extra logging

* Update timeout for create-next-app tests

Co-authored-by: Joe Haddad <timer150@gmail.com>

azure-pipelines.yml

@@ -62,5 +62,5 @@ jobs:
         displayName: 'Install dependencies'
 
       - script: |
-          node run-tests.js -g $(group)
+          node run-tests.js -g $(group) --timings
         displayName: 'Run tests'

run-tests.js

@@ -1,6 +1,7 @@
 const path = require('path')
 const _glob = require('glob')
 const fs = require('fs-extra')
+const fetch = require('node-fetch')
 const { promisify } = require('util')
 const { Sema } = require('async-sema')
 const { spawn, exec: execOrig } = require('child_process')
@@ -23,12 +24,62 @@ const timings = []
 
   console.log('Running tests with concurrency:', concurrency)
   let tests = process.argv.filter(arg => arg.endsWith('.test.js'))
+  let prevTimings
 
   if (tests.length === 0) {
     tests = await glob('**/*.test.js', {
       nodir: true,
       cwd: path.join(__dirname, 'test'),
     })
+
+    if (outputTimings) {
+      console.log('Fetching previous timings data')
+      const metaRes = await fetch(
+        `https://circleci.com/api/v1.1/project/github/zeit/next.js/`
+      )
+
+      if (metaRes.ok) {
+        const buildsMeta = await metaRes.json()
+        let buildNumber
+
+        for (const build of buildsMeta) {
+          if (
+            build.status === 'success' &&
+            build.build_parameters &&
+            build.build_parameters.CIRCLE_JOB === 'test'
+          ) {
+            buildNumber = build.build_num
+            break
+          }
+        }
+
+        const timesRes = await fetch(
+          `https://circleci.com/api/v1.1/project/github/zeit/next.js/${buildNumber}/tests`
+        )
+
+        if (timesRes.ok) {
+          const { tests } = await timesRes.json()
+          prevTimings = {}
+
+          for (const test of tests) {
+            prevTimings[test.file] = test.run_time
+          }
+
+          if (Object.keys(prevTimings).length > 0) {
+            console.log('Fetched previous timings data')
+          } else {
+            prevTimings = null
+          }
+        } else {
+          console.log(
+            'Failed to fetch previous timings status:',
+            timesRes.status
+          )
+        }
+      } else {
+        console.log('Failed to fetch timings meta status:', metaRes.status)
+      }
+    }
   }
 
   let testNames = [
@@ -49,9 +100,43 @@ const timings = []
     let offset = groupPos === 1 ? 0 : (groupPos - 1) * numPerGroup - 1
     // if there's an odd number of suites give the first group the extra
     if (testNames.length % 2 !== 0 && groupPos !== 1) offset++
-    testNames = testNames.splice(offset, numPerGroup)
+
+    if (prevTimings) {
+      const groups = [[]]
+      const groupTimes = [0]
+
+      for (const testName of testNames) {
+        let smallestGroup = groupTimes[0]
+        let smallestGroupIdx = 0
+
+        // get the samllest group time to add current one to
+        for (let i = 1; i < groupTotal; i++) {
+          if (!groups[i]) {
+            groups[i] = []
+            groupTimes[i] = 0
           }
 
+          const time = groupTimes[i]
+          if (time < smallestGroup) {
+            smallestGroup = time
+            smallestGroupIdx = i
+          }
+        }
+        groups[smallestGroupIdx].push(testName)
+        groupTimes[smallestGroupIdx] += prevTimings[testName] || 1
+      }
+
+      const curGroupIdx = groupPos - 1
+      testNames = groups[curGroupIdx]
+
+      console.log(
+        'Current group previous accumulated times:',
+        Math.round(groupTimes[curGroupIdx]) + 's'
+      )
+    } else {
+      testNames = testNames.splice(offset, numPerGroup)
+    }
+  }
   console.log('Running tests:', '\n', ...testNames.map(name => `${name}\n`))
 
   const sema = new Sema(concurrency, { capacity: testNames.length })
@@ -61,7 +146,7 @@ const timings = []
   )
   const children = new Set()
 
-  const runTest = (test = '') =>
+  const runTest = (test = '', usePolling) =>
     new Promise((resolve, reject) => {
       const start = new Date().getTime()
       const child = spawn(
@@ -69,6 +154,17 @@ const timings = []
         [jestPath, '--runInBand', '--forceExit', '--verbose', test],
         {
           stdio: 'inherit',
+          env: {
+            ...process.env,
+            ...(usePolling
+              ? {
+                  // Events can be finicky in CI. This switches to a more reliable
+                  // polling method.
+                  CHOKIDAR_USEPOLLING: 'true',
+                  CHOKIDAR_INTERVAL: 500,
+                }
+              : {}),
+          },
         }
       )
       children.add(child)
@@ -86,7 +182,7 @@ const timings = []
 
       for (let i = 0; i < NUM_RETRIES + 1; i++) {
         try {
-          const time = await runTest(test)
+          const time = await runTest(test, i > 0)
           timings.push({
             file: test,
             time,
@@ -96,9 +192,10 @@ const timings = []
         } catch (err) {
           if (i < NUM_RETRIES) {
             try {
-              console.log('Cleaning test files for', test)
-              await exec(`git clean -fdx "${path.join(__dirname, test)}"`)
-              await exec(`git checkout "${path.join(__dirname, test)}"`)
+              const testDir = path.dirname(path.join(__dirname, test))
+              console.log('Cleaning test files at', testDir)
+              await exec(`git clean -fdx "${testDir}"`)
+              await exec(`git checkout "${testDir}"`)
             } catch (err) {}
           }
         }
@@ -123,6 +220,7 @@ const timings = []
 
     for (const timing of timings) {
       const timeInSeconds = timing.time / 1000
+
       junitData += `
         <testsuite name="${timing.file}" file="${
         timing.file

test/integration/create-next-app/index.test.js

@@ -20,7 +20,7 @@ const run = (...args) => execa('node', [cli, ...args], { cwd })
 
 describe('create next app', () => {
   beforeAll(async () => {
-    jest.setTimeout(1000 * 30)
+    jest.setTimeout(1000 * 60)
     await mkdirp(cwd)
   })