From 42f6dc3d9ca64feec3feccd51c7a1fc590ee6994 Mon Sep 17 00:00:00 2001
From: JJ Kasper <jj@jjsweb.site>
Date: Wed, 19 Jun 2024 07:01:24 -0700
Subject: [PATCH] Add concurrency for flakey tests detection (#67019)

This ensures we parallelize our flakey test detection since when a lot
are changed it can hit the timeout and not be able to run them all.
---
 .github/workflows/build_and_test.yml | 18 ++++++++---
 scripts/test-new-tests.mjs           | 46 ++++++++++++++++++++++++++--
 2 files changed, 58 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 6c401000d4..b8a88e329f 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -284,10 +284,15 @@ jobs:
     needs: ['changes', 'build-native', 'build-next']
     if: ${{ needs.changes.outputs.docs-only == 'false' }}
 
+    strategy:
+      fail-fast: false
+      matrix:
+        group: [1/4, 2/4, 3/4, 4/4]
+
     uses: ./.github/workflows/build_reusable.yml
     with:
-      afterBuild: node scripts/test-new-tests.mjs --dev-mode
-      stepName: 'test-new-tests-dev'
+      afterBuild: node scripts/test-new-tests.mjs --dev-mode --group ${{ matrix.group }}
+      stepName: 'test-new-tests-dev-${{matrix.group}}'
 
     secrets: inherit
 
@@ -296,10 +301,15 @@ jobs:
     needs: ['changes', 'build-native', 'build-next']
     if: ${{ needs.changes.outputs.docs-only == 'false' }}
 
+    strategy:
+      fail-fast: false
+      matrix:
+        group: [1/4, 2/4, 3/4, 4/4]
+
     uses: ./.github/workflows/build_reusable.yml
     with:
-      afterBuild: node scripts/test-new-tests.mjs --prod-mode
-      stepName: 'test-new-tests-start'
+      afterBuild: node scripts/test-new-tests.mjs --prod-mode --group ${{ matrix.group }}
+      stepName: 'test-new-tests-start-${{matrix.group}}'
 
     secrets: inherit
 
diff --git a/scripts/test-new-tests.mjs b/scripts/test-new-tests.mjs
index 680fcafff2..a0ef51e019 100644
--- a/scripts/test-new-tests.mjs
+++ b/scripts/test-new-tests.mjs
@@ -2,9 +2,23 @@
 import fs from 'fs/promises'
 import execa from 'execa'
 import path from 'path'
+import yargs from 'yargs'
 
 async function main() {
-  let testMode = process.argv.includes('--dev-mode') ? 'dev' : 'start'
+  let argv = await yargs(process.argv.slice(2))
+    .boolean('--dev-mode')
+    .string('group').argv
+
+  let testMode = argv['--dev-mode'] ? 'dev' : 'start'
+  const rawGroup = argv['group']
+  let currentGroup = 1
+  let groupTotal = 1
+
+  if (rawGroup) {
+    ;[currentGroup, groupTotal] = rawGroup
+      .split('/')
+      .map((item) => Number(item))
+  }
 
   let eventData = {}
 
@@ -102,7 +116,35 @@ async function main() {
     )
   )
 
-  const currentTests = testMode === 'dev' ? devTests : prodTests
+  let currentTests = testMode === 'dev' ? devTests : prodTests
+
+  /**
+    @type {Array<string[]>}
+  */
+  const fileGroups = []
+
+  for (const test of currentTests) {
+    let smallestGroup = fileGroups[0]
+    let smallestGroupIdx = 0
+
+    // get the smallest group time to add current one to
+    for (let i = 0; i < groupTotal; i++) {
+      if (!fileGroups[i]) {
+        fileGroups[i] = []
+      }
+
+      if (
+        smallestGroup &&
+        fileGroups[i] &&
+        fileGroups[i].length < smallestGroup.length
+      ) {
+        smallestGroup = fileGroups[i]
+        smallestGroupIdx = i
+      }
+    }
+    fileGroups[smallestGroupIdx].push(test)
+  }
+  currentTests = fileGroups[currentGroup - 1] || []
 
   if (currentTests.length === 0) {
     console.log(`No added/changed tests detected`)