Fix recursive copy failing if more folders than concurrency (#9429)

* Fix recursive-copy function failing with more
folders than concurrency

* Make default concurrency 32 for recursive-copy

packages/next/lib/recursive-copy.ts

@@ -14,9 +14,14 @@ export async function recursiveCopy(
   source: string,
   dest: string,
   {
-    concurrency = 255,
+    concurrency = 32,
+    overwrite = false,
     filter = () => true,
-  }: { concurrency?: number; filter?(path: string): boolean } = {}
+  }: {
+    concurrency?: number
+    overwrite?: boolean
+    filter?(path: string): boolean
+  } = {}
 ) {
   const cwdPath = process.cwd()
   const from = path.resolve(cwdPath, source)
@@ -39,6 +44,7 @@ export async function recursiveCopy(
           throw err
         }
       }
+      sema.release()
       const files = await readdir(item)
       await Promise.all(files.map(file => _copy(path.join(item, file))))
     } else if (
@@ -47,11 +53,9 @@ export async function recursiveCopy(
       // we remove the base path (from) and replace \ by / (windows)
       filter(item.replace(from, '').replace(/\\/g, '/'))
     ) {
-      await copyFile(item, target, COPYFILE_EXCL)
+      await copyFile(item, target, overwrite ? undefined : COPYFILE_EXCL)
+      sema.release()
     }
-
-    sema.release()
-    return
   }
 
   await _copy(from)

test/unit/recursive-copy.test.js

@@ -1,4 +1,5 @@
 /* eslint-env jest */
+/* global jasmine */
 import { recursiveCopy } from 'next/dist/lib/recursive-copy'
 import { join } from 'path'
 import fs from 'fs-extra'
@@ -9,7 +10,18 @@ const testDir = join(__dirname, 'recursive-folder-test')
 const srcDir = join(testDir, 'src')
 const destDir = join(testDir, 'dest')
 
-beforeEach(async () => {
+jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000 * 60 * 1
+
+const setupTestDir = async (numFiles = 100) => {
+  const paths = [
+    '.hidden',
+    'file',
+    'folder1/file1',
+    'folder1/file2',
+    'link',
+    'linkfolder',
+  ]
+
   await fs.ensureDir(testDir)
 
   // create src directory structure
@@ -20,14 +32,22 @@ beforeEach(async () => {
   await fs.outputFile(join(srcDir, 'folder1', 'file2'), 'file2')
   await fs.ensureSymlink(join(srcDir, 'file'), join(srcDir, 'link'))
   await fs.ensureSymlink(join(srcDir, 'folder1'), join(srcDir, 'linkfolder'))
-})
 
-afterEach(async () => {
-  await fs.remove(testDir)
-})
+  for (let i = 0; i < numFiles - 6; i++) {
+    const path = join(`folder-${i}`, `file-${i}`)
+    await fs.outputFile(join(srcDir, path), `file-${i}`)
+    paths.push(path)
+  }
+  return paths
+}
 
 describe('recursiveCopy', () => {
+  afterAll(() => fs.remove(testDir))
+
   it('should work', async () => {
+    await fs.remove(testDir)
+    await setupTestDir(6)
+
     await recursiveCopy(srcDir, destDir, {
       filter(path) {
         return path !== '/folder1/file1'
@@ -48,4 +68,25 @@ describe('recursiveCopy', () => {
       'file1'
     )
   })
+
+  it('should work with content existing in dest', async () => {
+    await fs.remove(testDir)
+    const paths = await setupTestDir(25)
+    await recursiveCopy(srcDir, destDir)
+    await recursiveCopy(srcDir, destDir, { overwrite: true })
+
+    for (const path of paths) {
+      expect(await fs.pathExists(join(destDir, path))).toBe(true)
+    }
+  })
+
+  it('should handle more files than concurrency', async () => {
+    await fs.remove(testDir)
+    const paths = await setupTestDir(100)
+    await recursiveCopy(srcDir, destDir, { concurrency: 50 })
+
+    for (const path of paths) {
+      expect(await fs.pathExists(join(destDir, path))).toBe(true)
+    }
+  })
 })