Add Replay integration for dev e2e tests (#40955)

<!--
Thanks for opening a PR! Your contribution is much appreciated.
To make sure your PR is handled as smoothly as possible we request that
you follow the checklist sections below.
Choose the right checklist for the change that you're making:
-->

## Bug

- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Errors have a helpful link attached, see `contributing.md`

## Feature

- [ ] Implements an existing feature request or RFC. Make sure the
feature request has been accepted for implementation before opening a
PR.
- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see `contributing.md`

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm lint`
- [ ] The "examples guidelines" are followed from [our contributing
doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)

@ijjk moving this here.

Co-authored-by: JJ Kasper <jj@jjsweb.site>

.github/workflows/build_test_deploy.yml

@@ -405,9 +405,27 @@ jobs:
         timeout-minutes: 5
         if: ${{needs.build.outputs.docsChange == 'nope'}}
 
-      - run: NEXT_TEST_MODE=dev node run-tests.js --type e2e --timings -g ${{ matrix.group }}/3
+      - run: npx @replayio/playwright install chromium
+        if: ${{needs.build.outputs.docsChange == 'nope'}}
+
+      - run: node run-tests.js --type e2e --timings -g ${{ matrix.group }}/3
         name: Run test/e2e (dev)
         if: ${{needs.build.outputs.docsChange == 'nope'}}
+        env:
+          RECORD_REPLAY_METADATA_TEST_RUN_TITLE: testDevE2E / Group ${{ matrix.group }} / Node ${{ matrix.node }}
+          RECORD_ALL_CONTENT: 1
+          RECORD_REPLAY: 1
+          NEXT_TEST_MODE: dev
+          RECORD_REPLAY_TEST_METRICS: 1
+          RECORD_REPLAY_WEBHOOK_URL: ${{ secrets.RECORD_REPLAY_WEBHOOK_URL }}
+          # DEBUG: pw:browser*
+
+      - uses: replayio/action-upload@v0.4.5
+        if: always()
+        with:
+          api-key: rwk_iKsQnEoQwKd31WAJxgN9ARPFuAlyXlVrDH4uhYpRnti
+          public: true
+          filter: ${{ 'function($v) { $v.metadata.test.result = "failed" }' }}
 
       - name: Upload test trace
         if: always()

jest.replay.config.js

@@ -0,0 +1,18 @@
+const nextJest = require('next/jest')
+
+const createJestConfig = nextJest()
+
+// Any custom config you want to pass to Jest
+const customJestConfig = {
+  testMatch: ['**/*.test.js', '**/*.test.ts', '**/*.test.tsx'],
+  setupFilesAfterEnv: ['<rootDir>/jest-setup-after-env.ts'],
+  verbose: true,
+  rootDir: 'test',
+  modulePaths: ['<rootDir>/lib'],
+  transformIgnorePatterns: ['/next[/\\\\]dist/', '/\\.next/'],
+  testTimeout: 60000,
+  testRunner: '@replayio/jest/runner',
+}
+
+// createJestConfig is exported in this way to ensure that next/jest can load the Next.js config which is async
+module.exports = createJestConfig(customJestConfig)

package.json

@@ -69,6 +69,8 @@
     "@next/polyfill-module": "workspace:*",
     "@next/polyfill-nomodule": "workspace:*",
     "@next/swc": "workspace:*",
+    "@replayio/jest": "27.5.1-alpha.4",
+    "@replayio/playwright": "0.2.26",
     "@svgr/webpack": "5.5.0",
     "@swc/cli": "0.1.55",
     "@swc/core": "1.2.203",

run-tests.js

@@ -238,13 +238,19 @@ async function main() {
     `jest${process.platform === 'win32' ? '.CMD' : ''}`
   )
 
-  const runTest = (test = '', isFinalRun) =>
+  const runTest = (test = '', isFinalRun, isRetry) =>
     new Promise((resolve, reject) => {
       const start = new Date().getTime()
       let outputChunks = []
+
+      const shouldRecordTestWithReplay = process.env.RECORD_REPLAY && isRetry
+
       const child = spawn(
         jestPath,
         [
+          ...(shouldRecordTestWithReplay
+            ? [`--config=jest.replay.config.js`]
+            : []),
           '--runInBand',
           '--forceExit',
           '--verbose',
@@ -257,6 +263,7 @@ async function main() {
           stdio: ['ignore', 'pipe', 'pipe'],
           env: {
             ...process.env,
+            RECORD_REPLAY: shouldRecordTestWithReplay,
             // run tests in headless mode by default
             HEADLESS: 'true',
             TRACE_PLAYWRIGHT: 'true',
@@ -336,7 +343,7 @@ async function main() {
       for (let i = 0; i < numRetries + 1; i++) {
         try {
           console.log(`Starting ${test} retry ${i}/${numRetries}`)
-          const time = await runTest(test, i === numRetries)
+          const time = await runTest(test, i === numRetries, i > 0)
           timings.push({
             file: test,
             time,

test/lib/browsers/playwright.ts

@@ -62,16 +62,23 @@ export class Playwright extends BrowserInterface {
       }
     }
 
-    if (browserName === 'safari') {
-      browser = await webkit.launch({ headless })
-    } else if (browserName === 'firefox') {
-      browser = await firefox.launch({ headless })
-    } else {
-      browser = await chromium.launch({ headless, devtools: !headless })
-    }
+    browser = await this.launchBrowser(browserName, { headless })
     context = await browser.newContext({ locale, ...device })
   }
 
+  async launchBrowser(browserName: string, launchOptions: Record<string, any>) {
+    if (browserName === 'safari') {
+      return await webkit.launch(launchOptions)
+    } else if (browserName === 'firefox') {
+      return await firefox.launch(launchOptions)
+    } else {
+      return await chromium.launch({
+        devtools: !launchOptions.headless,
+        ...launchOptions,
+      })
+    }
+  }
+
   async get(url: string): Promise<void> {
     return page.goto(url) as any
   }

test/lib/browsers/replay.ts

@@ -0,0 +1,16 @@
+import { getExecutablePath } from '@replayio/playwright'
+import { Playwright } from './playwright'
+export { quit } from './playwright'
+
+export class Replay extends Playwright {
+  async launchBrowser(browserName: string, launchOptions: Record<string, any>) {
+    const browser: any = browserName === 'chrome' ? 'chromium' : browserName
+    return super.launchBrowser(browserName, {
+      ...launchOptions,
+      executablePath: getExecutablePath(browser) || undefined,
+      env: {
+        ...process.env,
+      },
+    })
+  }
+}

test/lib/next-webdriver.ts

@@ -84,6 +84,10 @@ export default async function webdriver(
     const { Selenium, quit } = await import('./browsers/selenium')
     CurrentInterface = Selenium
     browserQuit = quit
+  } else if (process.env.RECORD_REPLAY === 'true') {
+    const { Replay, quit } = await require('./browsers/replay')
+    CurrentInterface = Replay
+    browserQuit = quit
   } else {
     const { Playwright, quit } = await import('./browsers/playwright')
     CurrentInterface = Playwright