disable deploy tests for incompatible suites (#66776)

This disables tests that should not be run in a deployed environment,
because they use incompatible APIs or there's no reason to test them
outside of `next start`. Specifically disables for things like:

- Using `next.patchFile`, `next.renameFile`, etc.
- Attempting to use `next.cliOutput` to query runtime logs. When
deployed, these are only build-time logs.

[Latest Run](https://github.com/vercel/next.js/actions/runs/9483807368)

.github/workflows/test_e2e_deploy_release.yml

@@ -47,7 +47,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        group: [1/5, 2/5, 3/5, 4/5, 5/5]
+        group: [1/5, 2/5, 3/5, 4/5, 5/5, 6/6]
     with:
       afterBuild: NEXT_TEST_MODE=deploy NEXT_EXTERNAL_TESTS_FILTERS="test/deploy-tests-manifest.json" node run-tests.js --timings -g ${{ matrix.group }} -c 2 --type e2e
       skipNativeBuild: 'yes'

test/deploy-tests-manifest.json

@@ -1,72 +1,65 @@
 {
   "version": 2,
-  "suites": {},
+  "suites": {
+    "test/e2e/app-dir/app-static/app-static.test.ts": {
+      "failed": [
+        "app-dir static/dynamic handling new tags have been specified on subsequent fetch should not fetch from memory cache",
+        "app-dir static/dynamic handling new tags have been specified on subsequent fetch should not fetch from memory cache after revalidateTag is used"
+      ]
+    },
+    "test/e2e/app-dir/metadata/metadata.test.ts": {
+      "failed": [
+        "app dir - metadata navigation should render root not-found with default metadata",
+        "app dir - metadata react cache should have same title and page value on initial load",
+        "app dir - metadata react cache should have same title and page value when navigating"
+      ]
+    },
+    "test/e2e/middleware-rewrites/test/index.test.ts": {
+      "failed": ["Middleware Rewrite should handle catch-all rewrite correctly"]
+    }
+  },
   "rules": {
     "include": [
       "test/e2e/**/*.test.{t,j}s{,x}",
       "test/production/**/*.test.{t,j}s{,x}"
     ],
     "exclude": [
-      "test/e2e/app-dir/next-after-app/index.test.ts",
       "test/e2e/app-dir/scss/nm-module-nested/nm-module-nested.test.ts",
       "test/e2e/cancel-request/stream-cancel.test.ts",
       "test/e2e/edge-pages-support/edge-document.test.ts",
       "test/e2e/new-link-behavior/material-ui.test.ts",
       "test/e2e/react-dnd-compile/react-dnd-compile.test.ts",
-      "test/e2e/next-test/next-test.test.ts",
       "test/e2e/skip-trailing-slash-redirect/index.test.ts",
-      "test/e2e/tsconfig-module-preserve/index.test.ts",
       "test/e2e/app-dir/app-compilation/index.test.ts",
       "test/e2e/app-dir/parallel-route-not-found-params/parallel-route-not-found-params.test.ts",
-      "test/e2e/app-dir/ppr-navigations/loading-tsx-no-partial-rendering/loading-tsx-no-partial-rendering.test.ts",
       "test/e2e/app-dir/ppr/ppr.test.ts",
       "test/e2e/app-dir/rsc-webpack-loader/rsc-webpack-loader.test.ts",
-      "test/e2e/app-dir/scss/compilation-and-prefixing/compilation-and-prefixing.test.ts",
-      "test/e2e/app-dir/typeof-window/typeof-window.test.ts",
-      "test/e2e/app-dir/webpack-loader-conditions/webpack-loader-conditions.test.ts",
-      "test/e2e/app-dir/x-forwarded-headers/x-forwarded-headers.test.ts",
-      "test/e2e/edge-compiler-module-exports-preference/index.test.ts",
       "test/e2e/swc-warnings/index.test.ts",
       "test/e2e/third-parties/index.test.ts",
-      "test/e2e/useselectedlayoutsegment-s-in-pages-router/useselectedlayoutsegment-s-in-pages-router.test.ts",
       "test/e2e/app-dir/app-prefetch-false-loading/app-prefetch-false-loading.test.ts",
       "test/e2e/app-dir/app-routes-client-component/app-routes-client-component.test.ts",
       "test/e2e/app-dir/app-routes/app-custom-route-base-path.test.ts",
       "test/e2e/app-dir/mdx/mdx.test.ts",
       "test/e2e/app-dir/modularizeimports/modularizeimports.test.ts",
-      "test/e2e/app-dir/navigation/navigation.test.ts",
       "test/e2e/app-dir/parallel-routes-and-interception/parallel-routes-and-interception.test.ts",
-      "test/e2e/app-dir/ppr-navigations/avoid-popstate-flash/avoid-popstate-flash.test.ts",
-      "test/e2e/app-dir/server-components-externals/index.test.ts",
       "test/e2e/app-dir/third-parties/basic.test.ts",
-      "test/e2e/edge-can-read-request-body/index.test.ts",
       "test/e2e/esm-externals/esm-externals.test.ts",
-      "test/e2e/i18n-data-route/i18n-data-route.test.ts",
-      "test/e2e/next-phase/index.test.ts",
       "test/e2e/app-dir/actions-navigation/index.test.ts",
       "test/e2e/app-dir/app-static/app-static-custom-handler.test.ts",
-      "test/e2e/app-dir/conflicting-page-segments/conflicting-page-segments.test.ts",
-      "test/e2e/app-dir/interception-route-prefetch-cache/interception-route-prefetch-cache.test.ts",
-      "test/e2e/app-dir/missing-suspense-with-csr-bailout/missing-suspense-with-csr-bailout.test.ts",
       "test/e2e/app-dir/options-request/options-request.test.ts",
-      "test/e2e/app-dir/next-image/next-image-proxy.test.ts",
-      "test/e2e/app-dir/ppr-navigations/stale-prefetch-entry/stale-prefetch-entry.test.ts",
       "test/e2e/app-dir/parallel-routes-revalidation/parallel-routes-revalidation.test.ts",
       "test/e2e/app-dir/revalidate-dynamic/revalidate-dynamic.test.ts",
       "test/e2e/app-dir/scss/npm-import-nested/npm-import-nested.test.ts",
       "test/e2e/app-dir/syntax-highlighter-crash/syntax-highlighter-crash.test.ts",
-      "test/e2e/favicon-short-circuit/favicon-short-circuit.test.ts",
       "test/e2e/new-link-behavior/stitches.test.ts",
       "test/e2e/next-image-forward-ref/index.test.ts",
       "test/e2e/react-compiler/react-compiler.test.ts",
       "test/e2e/revalidate-reason/revalidate-reason.test.ts",
-      "test/e2e/app-dir/app-static/app-static.test.ts",
       "test/e2e/app-dir/actions/app-action.test.ts",
       "test/e2e/app-dir/i18n-hybrid/i18n-hybrid.test.js",
       "test/e2e/app-dir/metadata/metadata.test.ts",
       "test/e2e/app-dir/rsc-basic/rsc-basic.test.ts",
       "test/e2e/app-dir/scss/nm-module/nm-module.test.ts",
-      "test/e2e/app-dir/static-shell-debugging/static-shell-debugging.test.ts",
       "test/e2e/basepath.test.ts",
       "test/e2e/postcss-config-cjs/index.test.ts",
       "test/e2e/socket-io/index.test.js",

test/e2e/app-dir/app-compilation/index.test.ts

@@ -2,10 +2,14 @@ import { nextTestSetup } from 'e2e-utils'
 import { check, hasRedbox, waitFor } from 'next-test-utils'
 
 describe('app dir', () => {
-  const { next, isNextDev, isNextStart } = nextTestSetup({
+  const { next, isNextDev, isNextStart, skipped } = nextTestSetup({
     files: __dirname,
+    // This is skipped when deployed because there are no assertions outside of next start/next dev
+    skipDeployment: true,
   })
 
+  if (skipped) return
+
   if (isNextStart) {
     describe('Loading', () => {
       it('should render loading.js in initial html for slow page', async () => {

test/e2e/app-dir/app-static/app-static.test.ts

@@ -82,14 +82,17 @@ describe('app-dir static/dynamic handling', () => {
     expect(echoedHeaders.headers.traceparent).toEqual('C')
   })
 
-  it('should warn for too many cache tags', async () => {
+  // Runtime logs aren't queryable in deploy mode
-    const res = await next.fetch('/too-many-cache-tags')
+  if (!isNextDeploy) {
-    expect(res.status).toBe(200)
+    it('should warn for too many cache tags', async () => {
-    await retry(() => {
+      const res = await next.fetch('/too-many-cache-tags')
-      expect(next.cliOutput).toContain('exceeded max tag count for')
+      expect(res.status).toBe(200)
-      expect(next.cliOutput).toContain('tag-65')
+      await retry(() => {
+        expect(next.cliOutput).toContain('exceeded max tag count for')
+        expect(next.cliOutput).toContain('tag-65')
+      })
     })
-  })
+  }
 
   if (isNextDeploy) {
     describe('new tags have been specified on subsequent fetch', () => {
@@ -115,7 +118,7 @@ describe('app-dir static/dynamic handling', () => {
         expect(res1.status).toBe(200)
 
         const revalidateRes = await next.fetch(
-          '/api/revlidate-tag-node?tag=thankyounext'
+          '/api/revalidate-tag-node?tag=thankyounext'
         )
         expect((await revalidateRes.json()).revalidated).toBe(true)
 

test/e2e/app-dir/conflicting-page-segments/conflicting-page-segments.test.ts

@@ -4,9 +4,10 @@ import { check } from 'next-test-utils'
 describe('conflicting-page-segments', () => {
   const { next, isNextDev, skipped } = nextTestSetup({
     files: __dirname,
-    // we skip start because the build will fail and we won't be able to catch it
+    // we skip start & deploy because the build will fail and we won't be able to catch it
-    // start is re-triggered but caught in the assertions below
+    // start is re-triggered but caught in the assertions below.
     skipStart: true,
+    skipDeployment: true,
   })
 
   if (skipped) {

test/e2e/app-dir/missing-suspense-with-csr-bailout/missing-suspense-with-csr-bailout.test.ts

@@ -4,6 +4,8 @@ describe('missing-suspense-with-csr-bailout', () => {
   const { next, isNextDev, skipped } = nextTestSetup({
     files: __dirname,
     skipStart: true,
+    // This test is skipped when deployed because it's not possible to rename files after deployment.
+    skipDeployment: true,
   })
 
   if (skipped) {

test/e2e/app-dir/navigation/navigation.test.ts

@@ -176,7 +176,7 @@ describe('app dir - navigation', () => {
         )
       }
 
-      if (isNextStart) {
+      if (isNextStart || isNextDeploy) {
         await browser.waitForIdleNetwork()
         // there should be an RSC call for the prefetch
         expect(hasRscRequest).toBe(true)

test/e2e/app-dir/next-after-app/index.test.ts

@@ -15,13 +15,17 @@ describe.each(runtimes)('unstable_after() in %s runtime', (runtimeValue) => {
   const logFileDir = fs.mkdtempSync(path.join(os.tmpdir(), 'logs-'))
   const logFile = path.join(logFileDir, 'logs.jsonl')
 
-  const { next, isNextDev, isNextDeploy } = nextTestSetup({
+  const { next, isNextDev, isNextDeploy, skipped } = nextTestSetup({
     files: __dirname,
+    // `patchFile` and reading runtime logs are not supported in a deployed environment
+    skipDeployment: true,
     env: {
       PERSISTENT_LOG_FILE: logFile,
     },
   })
 
+  if (skipped) return
+
   {
     const originalContents: Record<string, string> = {}
 

test/e2e/app-dir/next-image/next-image-proxy.test.ts

@@ -10,10 +10,14 @@ let proxyPort
 let proxyServer: https.Server
 
 describe('next-image-proxy', () => {
-  const { next } = nextTestSetup({
+  const { next, skipped } = nextTestSetup({
     files: __dirname,
+    // This test is skipped when deployed because it relies on a proxy server
+    skipDeployment: true,
   })
 
+  if (skipped) return
+
   beforeAll(async () => {
     proxyPort = await findPort()
 

test/e2e/app-dir/ppr-navigations/avoid-popstate-flash/avoid-popstate-flash.test.ts

@@ -4,8 +4,10 @@ import { createTestDataServer } from 'test-data-service/writer'
 import { createTestLog } from 'test-log'
 
 describe('avoid-popstate-flash', () => {
-  if ((global as any).isNextDev) {
+  if ((global as any).isNextDev || (global as any).isNextDeploy) {
-    test('ppr is disabled in dev', () => {})
+    // this is skipped in dev because PPR is not enabled in dev
+    // and in deploy we can't rely on this test data service existing
+    test('should skip dev & deploy', () => {})
     return
   }
 

test/e2e/app-dir/ppr-navigations/loading-tsx-no-partial-rendering/loading-tsx-no-partial-rendering.test.ts

@@ -4,8 +4,10 @@ import { createTestDataServer } from 'test-data-service/writer'
 import { createTestLog } from 'test-log'
 
 describe('loading-tsx-no-partial-rendering', () => {
-  if ((global as any).isNextDev) {
+  if ((global as any).isNextDev || (global as any).isNextDeploy) {
-    test('ppr is disabled in dev', () => {})
+    // this is skipped in dev because PPR is not enabled in dev
+    // and in deploy we can't rely on this test data service existing
+    test('should skip dev & deploy', () => {})
     return
   }
 

test/e2e/app-dir/ppr-navigations/stale-prefetch-entry/stale-prefetch-entry.test.ts

@@ -4,8 +4,10 @@ import { createTestDataServer } from 'test-data-service/writer'
 import { createTestLog } from 'test-log'
 
 describe('stale-prefetch-entry', () => {
-  if ((global as any).isNextDev) {
+  if ((global as any).isNextDev || (global as any).isNextDeploy) {
-    test('ppr is disabled in dev', () => {})
+    // this is skipped in dev because PPR is not enabled in dev
+    // and in deploy we can't rely on this test data service existing
+    test('should skip dev & deploy', () => {})
     return
   }
 

test/e2e/app-dir/scss/compilation-and-prefixing/compilation-and-prefixing.test.ts

@@ -5,14 +5,17 @@ import { readdir, readFile } from 'fs-extra'
 import { join } from 'path'
 
 describe('SCSS Support', () => {
-  const { next, isNextDev } = nextTestSetup({
+  const { next, isNextDev, skipped } = nextTestSetup({
     files: __dirname,
+    // This test is skipped because it is reading files in the `.next` file which
+    // isn't available/necessary in a deployment environment.
+    skipDeployment: true,
     dependencies: {
       sass: '1.54.0',
     },
   })
 
-  // TODO: Figure out this test for dev and Turbopack
+  if (skipped) return // TODO: Figure out this test for dev and Turbopack
   ;(isNextDev ? describe.skip : describe)('Production only', () => {
     describe('CSS Compilation and Prefixing', () => {
       it(`should've compiled and prefixed`, async () => {

test/e2e/app-dir/server-components-externals/index.test.ts

@@ -2,10 +2,14 @@ import path from 'path'
 import { nextTestSetup } from 'e2e-utils'
 
 describe('app-dir - server components externals', () => {
-  const { next, isTurbopack } = nextTestSetup({
+  const { next, isTurbopack, skipped } = nextTestSetup({
+    // This test is skipped when deployed because it relies on manually patched `node_modules`
+    skipDeployment: true,
     files: __dirname,
   })
 
+  if (skipped) return
+
   it('should have externals for those in config.serverExternalPackages', async () => {
     const $ = await next.render$('/')
 

test/e2e/app-dir/static-shell-debugging/static-shell-debugging.test.ts

@@ -7,8 +7,11 @@ describe('static-shell-debugging', () => {
     { ppr: true, debugging: false },
     { ppr: false, debugging: false },
   ])('ppr = $ppr, debugging = $debugging', (context) => {
-    const { next } = nextTestSetup({
+    const { next, skipped } = nextTestSetup({
       files: __dirname,
+      // This test skips deployment because env vars that are doubled underscore prefixed
+      // are not supported. This is also intended to be used in development.
+      skipDeployment: true,
       env: {
         __NEXT_EXPERIMENTAL_STATIC_SHELL_DEBUGGING: context.debugging
           ? '1'
@@ -17,6 +20,8 @@ describe('static-shell-debugging', () => {
       nextConfig: { experimental: { ppr: context.ppr } },
     })
 
+    if (skipped) return
+
     if (context.debugging && context.ppr) {
       it('should only render the static shell', async () => {
         const res = await next.fetch('/?__nextppronly=1')

test/e2e/app-dir/typeof-window/typeof-window.test.ts

@@ -2,13 +2,18 @@ import { nextTestSetup } from 'e2e-utils'
 import path from 'path'
 
 describe('typeof-window', () => {
-  const { next } = nextTestSetup({
+  const { next, skipped } = nextTestSetup({
     files: __dirname,
+    // This test is skipped when deployed because the local tarball appears corrupted
+    // It also doesn't seem particularly useful to test when deployed
+    skipDeployment: true,
     dependencies: {
       'my-differentiated-files': `file:${path.join(__dirname, 'my-differentiated-files.tar')}`,
     },
   })
 
+  if (skipped) return
+
   it('should work using cheerio', async () => {
     const $ = await next.render$('/')
     expect($('h1').text()).toBe('Page loaded')

test/e2e/app-dir/webpack-loader-conditions/webpack-loader-conditions.test.ts

@@ -1,11 +1,13 @@
 import { nextTestSetup } from 'e2e-utils'
 
 describe('webpack-loader-conditions', () => {
-  const { next, isTurbopack } = nextTestSetup({
+  const { next, isTurbopack, skipped } = nextTestSetup({
     files: __dirname,
+    // This test is skipped because it's only expected to run in turbopack, which isn't enabled for builds
+    skipDeployment: true,
   })
 
-  if (!isTurbopack) {
+  if (!isTurbopack || skipped) {
     it('should only run the test in turbopack', () => {})
     return
   }

test/e2e/app-dir/x-forwarded-headers/x-forwarded-headers.test.ts

@@ -1,7 +1,15 @@
 import { nextTestSetup } from 'e2e-utils'
 
 describe('x-forwarded-headers', () => {
-  const { next } = nextTestSetup({ files: __dirname })
+  const { next, skipped } = nextTestSetup({
+    files: __dirname,
+    // This test is skipped because it sends requests with manipulated host headers
+    // which doesn't work in a deployed environment
+    skipDeployment: true,
+  })
+
+  if (skipped) return
+
   it('should include x-forwarded-* headers', async () => {
     const res = await next.fetch('/')
     const headers = await res.json()

test/e2e/edge-compiler-module-exports-preference/index.test.ts

@@ -5,6 +5,13 @@ import { fetchViaHTTP, shouldRunTurboDevTest } from 'next-test-utils'
 describe('Edge compiler module exports preference', () => {
   let next: NextInstance
 
+  if ((global as any).isNextDeploy) {
+    // this test is skipped when deployed because it manually creates a package in the node_modules directory
+    // which is unsupported
+    it('should skip next deploy', () => {})
+    return
+  }
+
   beforeAll(async () => {
     next = await createNext({
       files: {

test/e2e/favicon-short-circuit/favicon-short-circuit.test.ts

@@ -1,7 +1,7 @@
 import { nextTestSetup } from 'e2e-utils'
 
 describe('favicon-short-circuit', () => {
-  const { next, isNextDev, isNextStart } = nextTestSetup({
+  const { next, isNextDev } = nextTestSetup({
     files: __dirname,
   })
 
@@ -20,7 +20,7 @@ describe('favicon-short-circuit', () => {
       // Expect we didn't compile the not found route.
       expect(next.cliOutput).not.toContain('compiling /not-found')
     })
-  } else if (isNextStart) {
+  } else {
     it('should not short circuit the favicon in production', async () => {
       const res = await next.fetch('/favicon.ico')
 

test/e2e/i18n-data-route/i18n-data-route.test.ts

@@ -14,14 +14,19 @@ function checkDataRoute(data: any, page: string) {
 }
 
 describe('i18n-data-route', () => {
-  const { next } = nextTestSetup({
+  const { next, skipped } = nextTestSetup({
     files: __dirname,
+    // This test skips deployment because env vars that are doubled underscore prefixed
+    // are not supported.
+    skipDeployment: true,
     env: {
       // Disable internal header stripping so we can test the invoke output.
       __NEXT_NO_STRIP_INTERNAL_HEADERS: '1',
     },
   })
 
+  if (skipped) return
+
   describe('with locale prefix', () => {
     describe.each(i18n.locales)('/%s', (locale) => {
       const prefixed = pages.map((page) => ({

test/e2e/next-phase/index.test.ts

@@ -1,7 +1,10 @@
 import { nextTestSetup } from 'e2e-utils'
 
 describe('next-phase', () => {
-  const { next, isNextDev } = nextTestSetup({
+  const { next, isNextDev, skipped } = nextTestSetup({
+    // This test is skipped when deployed because it asserts against runtime
+    // logs that cannot be queried in a deployed environment.
+    skipDeployment: true,
     files: {
       'app/layout.js': `export default function Layout({ children }) {
         return <html><body>{children}</body></html>
@@ -17,6 +20,8 @@ describe('next-phase', () => {
     },
   })
 
+  if (skipped) return
+
   it('should render page with next phase correctly', async () => {
     const phases = {
       dev: 'phase-development-server',

test/e2e/next-test/next-test.test.ts

@@ -26,14 +26,18 @@ function createTemporaryFixture(fixtureName: string) {
 }
 
 describe('next test', () => {
-  const { next: basicExample } = nextTestSetup({
+  const { next: basicExample, skipped } = nextTestSetup({
     files: new FileRef(join(__dirname, 'basic-example')),
     dependencies: {
       '@playwright/test': '1.43.1',
     },
     skipStart: true,
+    // This doesn't need to be deployed as it's using `experimental-test` mode
+    skipDeployment: true,
   })
 
+  if (skipped) return
+
   afterAll(async () => {
     await basicExample.destroy()
   })

test/e2e/tsconfig-module-preserve/index.test.ts

@@ -3,7 +3,7 @@ import { retry } from 'next-test-utils'
 import stripAnsi from 'strip-ansi'
 
 describe('tsconfig module: preserve', () => {
-  const { next } = nextTestSetup({
+  const { next, skipped } = nextTestSetup({
     files: {
       'tsconfig.json': JSON.stringify({
         compilerOptions: { module: 'preserve' },
@@ -14,11 +14,15 @@ describe('tsconfig module: preserve', () => {
         } 
       `,
     },
+    // This test is skipped because it relies on `next.readFile`
+    skipDeployment: true,
     dependencies: {
       typescript: '5.4.4',
     },
   })
 
+  if (skipped) return
+
   it('allows you to skip moduleResolution, esModuleInterop and resolveJsonModule when using "module: preserve"', async () => {
     let output = ''