Fix isolated tests on windows and update azure config (#44457)

Fixes handling in isolated tests for windows and adds initial setup to run the main `app-dir` test suite. Also adds retrying when fetching test timings fails due to rate limiting. Closes: https://github.com/vercel/next.js/pull/44331
2022-12-31 00:12:42 -08:00 · 2022-12-31 00:12:42 -08:00 · 50857dad46
commit 50857dad46
parent 448c9c82ed
9 changed files with 166 additions and 151 deletions
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@ -109,7 +109,7 @@ stages:
            condition: eq(variables['isDocsOnly'], 'No')
            displayName: 'Run tests'

-      - job: test_integration_app_dir
+      - job: test_e2e_dev
        pool:
          vmImage: 'windows-2019'
        steps:
@ -139,6 +139,44 @@ stages:
            condition: eq(variables['isDocsOnly'], 'No')

          - script: |
-              node run-tests.js -c 1 test/integration/app-dir-basic/test/index.test.js
+              node run-tests.js -c 1 --debug test/e2e/app-dir/app/index.test.ts
            condition: eq(variables['isDocsOnly'], 'No')
-            displayName: 'Run tests'
+            displayName: 'Run tests (E2E Development)'
+            env:
+              NEXT_TEST_MODE: 'dev'
+
+      - job: test_e2e_prod
+        pool:
+          vmImage: 'windows-2019'
+        steps:
+          - task: NodeTool@0
+            inputs:
+              versionSpec: $(node_16_version)
+            displayName: 'Install Node.js'
+
+          - bash: |
+              node scripts/run-for-change.js --not --type docs --exec echo "##vso[task.setvariable variable=isDocsOnly]No"
+            displayName: 'Check Docs Only Change'
+
+          - script: npm i -g pnpm@$(PNPM_VERSION)
+            condition: eq(variables['isDocsOnly'], 'No')
+
+          - script: pnpm config set store-dir $(PNPM_CACHE_FOLDER)
+            condition: eq(variables['isDocsOnly'], 'No')
+
+          - script: pnpm store path
+            condition: eq(variables['isDocsOnly'], 'No')
+
+          - script: pnpm install && pnpm run build
+            condition: eq(variables['isDocsOnly'], 'No')
+            displayName: 'Install and build'
+
+          - script: npx playwright install chromium
+            condition: eq(variables['isDocsOnly'], 'No')
+
+          - script: |
+              node run-tests.js -c 1 --debug test/e2e/app-dir/app/index.test.ts
+            condition: eq(variables['isDocsOnly'], 'No')
+            displayName: 'Run tests (E2E Production)'
+            env:
+              NEXT_TEST_MODE: 'start'
--- a/run-tests.js
+++ b/run-tests.js
@ -49,11 +49,22 @@ const cleanUpAndExit = async (code) => {
 }

 async function getTestTimings() {
-  const timingsRes = await fetch(TIMINGS_API, {
-    headers: {
-      ...TIMINGS_API_HEADERS,
-    },
-  })
+  let timingsRes
+
+  const doFetch = () =>
+    fetch(TIMINGS_API, {
+      headers: {
+        ...TIMINGS_API_HEADERS,
+      },
+    })
+  timingsRes = await doFetch()
+
+  if (timingsRes.status === 403) {
+    const delay = 15
+    console.log(`Got 403 response waiting ${delay} seconds before retry`)
+    await new Promise((resolve) => setTimeout(resolve, delay * 1000))
+    timingsRes = await doFetch()
+  }

  if (!timingsRes.ok) {
    throw new Error(`request status: ${timingsRes.status}`)
@ -219,6 +230,7 @@ async function main() {
  })

  if (
+    process.platform !== 'win32' &&
    process.env.NEXT_TEST_MODE !== 'deploy' &&
    ((testType && testType !== 'unit') || hasIsolatedTests)
  ) {
--- a/test/integration/app-dir-basic/app/blog/page.js
+++ b/test/integration/app-dir-basic/app/blog/page.js
@ -1,3 +0,0 @@
-export default function page() {
-  return <div id="blog">this is blog</div>
-}
--- a/test/integration/app-dir-basic/app/layout.js
+++ b/test/integration/app-dir-basic/app/layout.js
@ -1,8 +0,0 @@
-export default function RootLayout({ children }) {
-  return (
-    <html>
-      <head></head>
-      <body>{children}</body>
-    </html>
-  )
-}
--- a/test/integration/app-dir-basic/app/page.js
+++ b/test/integration/app-dir-basic/app/page.js
@ -1,3 +0,0 @@
-export default function page() {
-  return <div id="home">this is home</div>
-}
--- a/test/integration/app-dir-basic/next.config.js
+++ b/test/integration/app-dir-basic/next.config.js
@ -1,5 +0,0 @@
-module.exports = {
-  experimental: {
-    appDir: true,
-  },
-}
--- a/test/integration/app-dir-basic/test/index.test.js
+++ b/test/integration/app-dir-basic/test/index.test.js
@ -1,34 +0,0 @@
-/* eslint-env jest */
-
-import { join } from 'path'
-import cheerio from 'cheerio'
-import { runDevSuite, runProdSuite, renderViaHTTP } from 'next-test-utils'
-
-import webdriver from 'next-webdriver'
-const appDir = join(__dirname, '..')
-
-function runTests(context, env) {
-  describe('App Dir Basic', () => {
-    it('should render html properly', async () => {
-      const $index = cheerio.load(await renderViaHTTP(context.appPort, '/'))
-      const $blog = cheerio.load(await renderViaHTTP(context.appPort, '/blog'))
-
-      expect($index('#home').text()).toBe('this is home')
-      expect($blog('#blog').text()).toBe('this is blog')
-    })
-
-    it('should hydrate pages properly', async () => {
-      const browser = await webdriver(context.appPort, '/')
-      const indexHtml = await browser.waitForElementByCss('#home').text()
-      const url = await browser.url()
-      await browser.loadPage(url + 'blog')
-      const blogHtml = await browser.waitForElementByCss('#blog').text()
-
-      expect(indexHtml).toBe('this is home')
-      expect(blogHtml).toBe('this is blog')
-    })
-  })
-}
-
-runDevSuite('App Dir Basic', appDir, { runTests })
-runProdSuite('App Dir Basic', appDir, { runTests })
--- a/test/lib/next-modes/next-dev.ts
+++ b/test/lib/next-modes/next-dev.ts
@ -1,4 +1,4 @@
-import { spawn } from 'child_process'
+import { spawn } from 'cross-spawn'
 import { Span } from 'next/trace'
 import { NextInstance } from './base'

@ -36,65 +36,70 @@ export class NextDevInstance extends NextInstance {
    }

    await new Promise<void>((resolve, reject) => {
-      this.childProcess = spawn(startArgs[0], startArgs.slice(1), {
-        cwd: useDirArg ? process.cwd() : this.testDir,
-        stdio: ['ignore', 'pipe', 'pipe'],
-        shell: false,
-        env: {
-          ...process.env,
-          ...this.env,
-          NODE_ENV: '' as any,
-          PORT: this.forcedPort || '0',
-          __NEXT_TEST_MODE: '1',
-          __NEXT_TEST_WITH_DEVTOOL: '1',
-        },
-      })
+      try {
+        this.childProcess = spawn(startArgs[0], startArgs.slice(1), {
+          cwd: useDirArg ? process.cwd() : this.testDir,
+          stdio: ['ignore', 'pipe', 'pipe'],
+          shell: false,
+          env: {
+            ...process.env,
+            ...this.env,
+            NODE_ENV: '' as any,
+            PORT: this.forcedPort || '0',
+            __NEXT_TEST_MODE: '1',
+            __NEXT_TEST_WITH_DEVTOOL: '1',
+          },
+        })

-      this._cliOutput = ''
+        this._cliOutput = ''

-      this.childProcess.stdout.on('data', (chunk) => {
-        const msg = chunk.toString()
-        process.stdout.write(chunk)
-        this._cliOutput += msg
-        this.emit('stdout', [msg])
-      })
-      this.childProcess.stderr.on('data', (chunk) => {
-        const msg = chunk.toString()
-        process.stderr.write(chunk)
-        this._cliOutput += msg
-        this.emit('stderr', [msg])
-      })
+        this.childProcess.stdout.on('data', (chunk) => {
+          const msg = chunk.toString()
+          process.stdout.write(chunk)
+          this._cliOutput += msg
+          this.emit('stdout', [msg])
+        })
+        this.childProcess.stderr.on('data', (chunk) => {
+          const msg = chunk.toString()
+          process.stderr.write(chunk)
+          this._cliOutput += msg
+          this.emit('stderr', [msg])
+        })

-      this.childProcess.on('close', (code, signal) => {
-        if (this.isStopping) return
-        if (code || signal) {
-          throw new Error(
-            `next dev exited unexpectedly with code/signal ${code || signal}`
-          )
-        }
-      })
-      const readyCb = (msg) => {
-        if (msg.includes('started server on') && msg.includes('url:')) {
-          // turbo devserver emits stdout in rust directly, can contain unexpected chars with color codes
-          // strip out again for the safety
-          this._url = msg
-            .split('url: ')
-            .pop()
-            .trim()
-            .split(require('os').EOL)[0]
-          try {
-            this._parsedUrl = new URL(this._url)
-          } catch (err) {
-            reject({
-              err,
-              msg,
-            })
+        this.childProcess.on('close', (code, signal) => {
+          if (this.isStopping) return
+          if (code || signal) {
+            throw new Error(
+              `next dev exited unexpectedly with code/signal ${code || signal}`
+            )
+          }
+        })
+        const readyCb = (msg) => {
+          if (msg.includes('started server on') && msg.includes('url:')) {
+            // turbo devserver emits stdout in rust directly, can contain unexpected chars with color codes
+            // strip out again for the safety
+            this._url = msg
+              .split('url: ')
+              .pop()
+              .trim()
+              .split(require('os').EOL)[0]
+            try {
+              this._parsedUrl = new URL(this._url)
+            } catch (err) {
+              reject({
+                err,
+                msg,
+              })
+            }
+            this.off('stdout', readyCb)
+            resolve()
          }
-          this.off('stdout', readyCb)
-          resolve()
        }
+        this.on('stdout', readyCb)
+      } catch (err) {
+        require('console').error(`Failed to run ${startArgs.join(' ')}`, err)
+        setTimeout(() => process.exit(1), 0)
      }
-      this.on('stdout', readyCb)
    })
  }
 }
--- a/test/lib/next-modes/next-start.ts
+++ b/test/lib/next-modes/next-start.ts
@ -1,7 +1,7 @@
 import path from 'path'
 import fs from 'fs-extra'
 import { NextInstance } from './base'
-import { spawn, SpawnOptions } from 'child_process'
+import { spawn, SpawnOptions } from 'cross-spawn'
 import { Span } from 'next/trace'

 export class NextStartInstance extends NextInstance {
@ -65,20 +65,26 @@ export class NextStartInstance extends NextInstance {

    await new Promise<void>((resolve, reject) => {
      console.log('running', buildArgs.join(' '))
-      this.childProcess = spawn(
-        buildArgs[0],
-        buildArgs.slice(1),
-        this.spawnOpts
-      )
-      this.handleStdio(this.childProcess)
-      this.childProcess.on('exit', (code, signal) => {
-        this.childProcess = null
-        if (code || signal)
-          reject(
-            new Error(`next build failed with code/signal ${code || signal}`)
-          )
-        else resolve()
-      })
+
+      try {
+        this.childProcess = spawn(
+          buildArgs[0],
+          buildArgs.slice(1),
+          this.spawnOpts
+        )
+        this.handleStdio(this.childProcess)
+        this.childProcess.on('exit', (code, signal) => {
+          this.childProcess = null
+          if (code || signal)
+            reject(
+              new Error(`next build failed with code/signal ${code || signal}`)
+            )
+          else resolve()
+        })
+      } catch (err) {
+        require('console').error(`Failed to run ${buildArgs.join(' ')}`, err)
+        setTimeout(() => process.exit(1), 0)
+      }
    })

    this._buildId = (
@ -95,31 +101,38 @@ export class NextStartInstance extends NextInstance {
    console.log('running', startArgs.join(' '))

    await new Promise<void>((resolve) => {
-      this.childProcess = spawn(
-        startArgs[0],
-        startArgs.slice(1),
-        this.spawnOpts
-      )
-      this.handleStdio(this.childProcess)
+      try {
+        this.childProcess = spawn(
+          startArgs[0],
+          startArgs.slice(1),
+          this.spawnOpts
+        )
+        this.handleStdio(this.childProcess)

-      this.childProcess.on('close', (code, signal) => {
-        if (this.isStopping) return
-        if (code || signal) {
-          throw new Error(
-            `next start exited unexpectedly with code/signal ${code || signal}`
-          )
-        }
-      })
+        this.childProcess.on('close', (code, signal) => {
+          if (this.isStopping) return
+          if (code || signal) {
+            throw new Error(
+              `next start exited unexpectedly with code/signal ${
+                code || signal
+              }`
+            )
+          }
+        })

-      const readyCb = (msg) => {
-        if (msg.includes('started server on') && msg.includes('url:')) {
-          this._url = msg.split('url: ').pop().trim()
-          this._parsedUrl = new URL(this._url)
-          this.off('stdout', readyCb)
-          resolve()
+        const readyCb = (msg) => {
+          if (msg.includes('started server on') && msg.includes('url:')) {
+            this._url = msg.split('url: ').pop().trim()
+            this._parsedUrl = new URL(this._url)
+            this.off('stdout', readyCb)
+            resolve()
+          }
        }
+        this.on('stdout', readyCb)
+      } catch (err) {
+        require('console').error(`Failed to run ${startArgs.join(' ')}`, err)
+        setTimeout(() => process.exit(1), 0)
      }
-      this.on('stdout', readyCb)
    })
  }