Run and report benchmarks (#66851)

Using `@vercel/devlow-bench`, this benchmarks changes landed on canary
and reports results to Datadog.

.github/workflows/build_and_test.yml

@@ -152,6 +152,25 @@ jobs:
       stepName: 'rust-doc-check'
     secrets: inherit
 
+  devlow-bench:
+    name: Run devlow benchmarks
+    needs: ['changes', 'build-next', 'build-native']
+    if: ${{ needs.changes.outputs.docs-only == 'false' }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        mode:
+          - '--turbopack=false'
+          - '--turbopack=true'
+        selector:
+          - '--scenario=heavy-npm-deps --page=homepage'
+    uses: ./.github/workflows/build_reusable.yml
+    with:
+      afterBuild: pnpm install && ./node_modules/.bin/devlow-bench ./scripts/devlow-bench.mjs --datadog=ubuntu-latest-16-core ${{ matrix.mode }} ${{ matrix.selector }}
+      stepName: 'devlow-bench-${{ matrix.group }}'
+    secrets: inherit
+
   test-turbopack-dev:
     name: test turbopack dev
     needs: ['changes', 'build-next', 'build-native']

bench/heavy-npm-deps/package.json

@@ -3,7 +3,8 @@
   "version": "0.1.0",
   "private": true,
   "scripts": {
-    "dev-application": "next dev --turbo",
+    "dev-turbopack": "next dev --turbo",
+    "dev-webpack": "next dev",
     "build-application": "next build",
     "start-application": "next start"
   },

package.json

@@ -114,6 +114,7 @@
     "@types/trusted-types": "2.0.3",
     "@typescript-eslint/eslint-plugin": "6.14.0",
     "@typescript-eslint/parser": "6.14.0",
+    "@vercel/devlow-bench": "0.3.1",
     "@vercel/fetch": "6.1.1",
     "@vercel/og": "0.6.2",
     "abort-controller": "3.0.0",

pnpm-lock.yaml

@@ -191,6 +191,9 @@ importers:
       '@typescript-eslint/parser':
         specifier: 6.14.0
         version: 6.14.0(eslint@8.56.0)(typescript@5.3.3)
+      '@vercel/devlow-bench':
+        specifier: 0.3.1
+        version: 0.3.1
       '@vercel/fetch':
         specifier: 6.1.1
         version: 6.1.1(@types/node-fetch@2.6.1)(node-fetch@2.6.7)
@@ -3538,6 +3541,24 @@ packages:
       postcss-value-parser: 4.2.0
     dev: true
 
+  /@datadog/datadog-api-client@1.25.0:
+    resolution: {integrity: sha512-q0InWeL9RDTnwqeeDaBWM7tNomTPQBpIYfAYfoDtWgOcEPQyq7e3pIxCQUFGmLwUTudv2Ar4kvAh0rGaZw5/Zw==}
+    engines: {node: '>=12.0.0'}
+    dependencies:
+      '@types/buffer-from': 1.1.3
+      '@types/node': 20.12.3
+      '@types/pako': 1.0.7
+      buffer-from: 1.1.2
+      cross-fetch: 3.1.8
+      es6-promise: 4.2.8
+      form-data: 4.0.0
+      loglevel: 1.9.1
+      pako: 2.1.0
+      url-parse: 1.5.10
+    transitivePeerDependencies:
+      - encoding
+    dev: true
+
   /@datadog/native-appsec@3.2.0:
     resolution: {integrity: sha512-biAa7EFfuavjSWgSQaCit9CqGzr6Af5nhzfNNGJ38Y/Y387hDvLivAR374kK1z6XoxGZEOa+XPbVogmV/2Bcjw==}
     engines: {node: '>=12'}
@@ -4438,6 +4459,11 @@ packages:
     engines: {node: '>=18'}
     dev: true
 
+  /@inquirer/figures@1.0.3:
+    resolution: {integrity: sha512-ErXXzENMH5pJt5/ssXV0DfWUZqly8nGzf0UcBV9xTnP+KyffE2mqyxIMBrZ8ijQck2nU0TQm40EQB53YreyWHw==}
+    engines: {node: '>=18'}
+    dev: true
+
   /@inquirer/type@1.3.2:
     resolution: {integrity: sha512-5Frickan9c89QbPkSu6I6y8p+9eR6hZkdPahGmNDsTFX8FHLPAozyzCZMKUeW8FyYwnlCKUjqIEqxY+UctARiw==}
     engines: {node: '>=18'}
@@ -5601,6 +5627,13 @@ packages:
       write-file-atomic: 3.0.3
     dev: true
 
+  /@ljharb/through@2.3.13:
+    resolution: {integrity: sha512-/gKJun8NNiWGZJkGzI/Ragc53cOdcLNdzjLaIa+GEjguQs0ulsurx8WN0jijdK9yPqDvziX995sMRLyLt1uZMQ==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      call-bind: 1.0.7
+    dev: true
+
   /@mantine/core@7.10.1(@mantine/hooks@7.10.1)(react-dom@19.0.0-rc-6230622a1a-20240610)(react@19.0.0-rc-6230622a1a-20240610)(types-react@19.0.0-rc.0):
     resolution: {integrity: sha512-l9ypojKN3PjwO1CSLIsqxi7mA25+7w+xc71Q+JuCCREI0tuGwkZsKbIOpuTATIJOjPh8ycLiW7QxX1LYsRTq6w==}
     peerDependencies:
@@ -7155,6 +7188,12 @@ packages:
     resolution: {integrity: sha512-wJsiX1tosQ+J5+bY5LrSahHxr2wT+uME5UDwdN1kg4frt40euqA+wzECkmq4t5QbveHiJepfdThgQrPw6KiSlg==}
     dev: true
 
+  /@types/buffer-from@1.1.3:
+    resolution: {integrity: sha512-2lq4YC9uLUMGHkl2IDtX4tCXSo2+hwMpOJcY1qiIk1kybc31rIlPyM1HCVJhkPFIo75a/pOVxqyvwuf5TpCG/w==}
+    dependencies:
+      '@types/node': 20.12.3
+    dev: true
+
   /@types/busboy@1.5.3:
     resolution: {integrity: sha512-YMBLFN/xBD8bnqywIlGyYqsNFXu6bsiY7h3Ae0kO17qEuTjsqeyYMRPSUDacIKIquws2Y6KjmxAyNx8xB3xQbw==}
     dependencies:
@@ -7464,6 +7503,10 @@ packages:
   /@types/normalize-package-data@2.4.0:
     resolution: {integrity: sha512-f5j5b/Gf71L+dbqxIpQ4Z2WlmI/mPJ0fOkGGmFgtb6sAu97EPczzbS3/tJKxmcYDj55OX6ssqwDAWOHIYDRDGA==}
 
+  /@types/pako@1.0.7:
+    resolution: {integrity: sha512-YBtzT2ztNF6R/9+UXj2wTGFnC9NklAnASt3sC0h2m1bbH7G6FyBIkt4AN8ThZpNfxUo1b2iMVO0UawiJymEt8A==}
+    dev: true
+
   /@types/parse-json@4.0.0:
     resolution: {integrity: sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==}
     dev: true
@@ -7910,6 +7953,22 @@ packages:
     resolution: {integrity: sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==}
     dev: true
 
+  /@vercel/devlow-bench@0.3.1:
+    resolution: {integrity: sha512-RbKloNKAj55/JYqhx2QtqVFWKr1f7zvuBVGs8oPksCO6oZkSnvgJhcgjftOpFAzJKUSQ+oUOYIO9oWdjDUjgFw==}
+    hasBin: true
+    dependencies:
+      '@datadog/datadog-api-client': 1.25.0
+      chalk: 2.4.2
+      inquirer: 9.2.23
+      minimist: 1.2.8
+      pidusage-tree: 2.0.5
+      playwright-chromium: 1.41.2
+      split2: 4.2.0
+      tree-kill: 1.2.2
+    transitivePeerDependencies:
+      - encoding
+    dev: true
+
   /@vercel/fetch-cached-dns@2.0.2(node-fetch@2.6.7):
     resolution: {integrity: sha512-gDqKEV8CeY2YmCdZpP1rn3tFK1L07Vw2+HYkCK8zpRHOVGr/sP8yhBsW+C/yqGVj0i9z/rIvqIHe5emvRvxwgw==}
     peerDependencies:
@@ -9304,6 +9363,10 @@ packages:
   /buffer-from@1.1.1:
     resolution: {integrity: sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==}
 
+  /buffer-from@1.1.2:
+    resolution: {integrity: sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==}
+    dev: true
+
   /buffer-xor@1.0.3:
     resolution: {integrity: sha512-571s0T7nZWK6vB67HI5dyUF7wXiNcfaPPPTl6zYCNApANjIvYJTg7hlud/+cJpdAhS7dVzqMLmfhfHR3rAcOjQ==}
     dev: true
@@ -9429,9 +9492,20 @@ packages:
   /call-bind@1.0.2:
     resolution: {integrity: sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==}
     dependencies:
-      function-bind: 1.1.1
+      function-bind: 1.1.2
       get-intrinsic: 1.2.1
 
+  /call-bind@1.0.7:
+    resolution: {integrity: sha512-GHTSNSYICQ7scH7sZ+M2rFopRoLh8t2bLSW6BbgrtLsahOIB5iyAVJf9GjWK3cYTDaMj4XdBpM1cA6pIS0Kv2w==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      es-define-property: 1.0.0
+      es-errors: 1.3.0
+      function-bind: 1.1.2
+      get-intrinsic: 1.2.4
+      set-function-length: 1.2.2
+    dev: true
+
   /caller-callsite@2.0.0:
     resolution: {integrity: sha512-JuG3qI4QOftFsZyOn1qq87fq5grLIyk1JYd5lJmdA+fG7aQ9pA/i3JIJGcO3q0MrRcHlOt1U+ZeHW8Dq9axALQ==}
     engines: {node: '>=4'}
@@ -9997,27 +10071,18 @@ packages:
     resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==}
     requiresBuild: true
 
-  /color-string@1.5.4:
-    resolution: {integrity: sha512-57yF5yt8Xa3czSEW1jfQDE79Idk0+AkN/4KWad6tbdxUmAs3MvjxlWSWD4deYytcRfoZ9nhKyFl1kj5tBvidbw==}
-    dependencies:
-      color-name: 1.1.4
-      simple-swizzle: 0.2.2
-    dev: true
-
   /color-string@1.9.1:
     resolution: {integrity: sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==}
     requiresBuild: true
     dependencies:
       color-name: 1.1.4
       simple-swizzle: 0.2.2
-    dev: false
-    optional: true
 
   /color@3.1.3:
     resolution: {integrity: sha512-xgXAcTHa2HeFCGLE9Xs/R82hujGtu9Jd9x4NW3T34+OMs7VoPsjwzRczKHvTAHeJwWFwX5j15+MgAppE8ztObQ==}
     dependencies:
       color-convert: 1.9.3
-      color-string: 1.5.4
+      color-string: 1.9.1
     dev: true
 
   /color@4.2.3:
@@ -10544,6 +10609,14 @@ packages:
       node-fetch: 2.6.1
     dev: true
 
+  /cross-fetch@3.1.8:
+    resolution: {integrity: sha512-cvA+JwZoU0Xq+h6WkMvAUqPEYy92Obet6UdKLfW60qn99ftItKjB5T+BkyWOFWe2pUyfQ+IJHmpOTznqk1M6Kg==}
+    dependencies:
+      node-fetch: 2.7.0
+    transitivePeerDependencies:
+      - encoding
+    dev: true
+
   /cross-spawn-async@2.2.5:
     resolution: {integrity: sha512-snteb3aVrxYYOX9e8BabYFK9WhCDhTlw1YQktfTthBogxri4/2r9U2nQc0ffY73ZAxezDc+U8gvHAeU1wy1ubQ==}
     deprecated: cross-spawn no longer requires a build toolchain, use it instead
@@ -11509,6 +11582,15 @@ packages:
     engines: {node: '>=10'}
     dev: false
 
+  /define-data-property@1.1.4:
+    resolution: {integrity: sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      es-define-property: 1.0.0
+      es-errors: 1.3.0
+      gopd: 1.0.1
+    dev: true
+
   /define-lazy-prop@2.0.0:
     resolution: {integrity: sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==}
     engines: {node: '>=8'}
@@ -12112,9 +12194,9 @@ packages:
     dependencies:
       call-bind: 1.0.2
       es-to-primitive: 1.2.1
-      function-bind: 1.1.1
+      function-bind: 1.1.2
       function.prototype.name: 1.1.5
-      get-intrinsic: 1.1.2
+      get-intrinsic: 1.2.1
       get-symbol-description: 1.0.0
       has: 1.0.3
       has-property-descriptors: 1.0.0
@@ -12179,6 +12261,18 @@ packages:
       unbox-primitive: 1.0.2
       which-typed-array: 1.1.11
 
+  /es-define-property@1.0.0:
+    resolution: {integrity: sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      get-intrinsic: 1.2.4
+    dev: true
+
+  /es-errors@1.3.0:
+    resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==}
+    engines: {node: '>= 0.4'}
+    dev: true
+
   /es-iterator-helpers@1.0.13:
     resolution: {integrity: sha512-LK3VGwzvaPWobO8xzXXGRUOGw8Dcjyfk62CsY/wfHN75CwsJPbuypOYJxK6g5RyEL8YDjIWcl6jgd8foO6mmrA==}
     dependencies:
@@ -13765,7 +13859,6 @@ packages:
 
   /function-bind@1.1.2:
     resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==}
-    dev: true
 
   /function.prototype.name@1.1.5:
     resolution: {integrity: sha512-uN7m/BzVKQnCUF/iW8jYea67v++2u7m5UgENbHRtdDVclOUP+FMPlCNdmk0h/ysGyo2tavMJEDqJAkJdRa1vMA==}
@@ -13815,14 +13908,6 @@ packages:
     engines: {node: '>=18'}
     dev: true
 
-  /get-intrinsic@1.1.2:
-    resolution: {integrity: sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA==}
-    dependencies:
-      function-bind: 1.1.1
-      has: 1.0.3
-      has-symbols: 1.0.3
-    dev: true
-
   /get-intrinsic@1.2.1:
     resolution: {integrity: sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==}
     dependencies:
@@ -13831,6 +13916,17 @@ packages:
       has-proto: 1.0.1
       has-symbols: 1.0.3
 
+  /get-intrinsic@1.2.4:
+    resolution: {integrity: sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      es-errors: 1.3.0
+      function-bind: 1.1.2
+      has-proto: 1.0.1
+      has-symbols: 1.0.3
+      hasown: 2.0.0
+    dev: true
+
   /get-nonce@1.0.1:
     resolution: {integrity: sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==}
     engines: {node: '>=6'}
@@ -14338,6 +14434,12 @@ packages:
     dependencies:
       get-intrinsic: 1.2.1
 
+  /has-property-descriptors@1.0.2:
+    resolution: {integrity: sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==}
+    dependencies:
+      es-define-property: 1.0.0
+    dev: true
+
   /has-proto@1.0.1:
     resolution: {integrity: sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==}
     engines: {node: '>= 0.4'}
@@ -15066,11 +15168,32 @@ packages:
       run-async: 2.4.1
       rxjs: 7.8.1
       string-width: 4.2.3
-      strip-ansi: 6.0.1
+      strip-ansi: 6.0.0
       through: 2.3.8
       wrap-ansi: 6.2.0
     dev: true
 
+  /inquirer@9.2.23:
+    resolution: {integrity: sha512-kod5s+FBPIDM2xiy9fu+6wdU/SkK5le5GS9lh4FEBjBHqiMgD9lLFbCbuqFNAjNL2ZOy9Wd9F694IOzN9pZHBA==}
+    engines: {node: '>=18'}
+    dependencies:
+      '@inquirer/figures': 1.0.3
+      '@ljharb/through': 2.3.13
+      ansi-escapes: 4.3.2
+      chalk: 5.3.0
+      cli-cursor: 3.1.0
+      cli-width: 4.1.0
+      external-editor: 3.1.0
+      lodash: 4.17.21
+      mute-stream: 1.0.0
+      ora: 5.4.1
+      run-async: 3.0.0
+      rxjs: 7.8.1
+      string-width: 4.2.3
+      strip-ansi: 6.0.1
+      wrap-ansi: 6.2.0
+    dev: true
+
   /int64-buffer@0.1.10:
     resolution: {integrity: sha512-v7cSY1J8ydZ0GyjUHqF+1bshJ6cnEVLo9EnjB8p+4HDRPZc9N5jjmvUV7NvEsqQOKyH0pmIBFWXVQbiS0+OBbA==}
     dev: true
@@ -16836,7 +16959,7 @@ packages:
     engines: {node: '>=6'}
     hasBin: true
     dependencies:
-      minimist: 1.2.6
+      minimist: 1.2.8
 
   /json5@2.2.3:
     resolution: {integrity: sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==}
@@ -17465,6 +17588,11 @@ packages:
       wrap-ansi: 9.0.0
     dev: true
 
+  /loglevel@1.9.1:
+    resolution: {integrity: sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==}
+    engines: {node: '>= 0.6.0'}
+    dev: true
+
   /long@4.0.0:
     resolution: {integrity: sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==}
     dev: true
@@ -18481,6 +18609,9 @@ packages:
   /minimist@1.2.6:
     resolution: {integrity: sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==}
 
+  /minimist@1.2.8:
+    resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==}
+
   /minipass-collect@1.0.2:
     resolution: {integrity: sha512-6T6lH0H8OG9kITm/Jm6tdooIbogG9e0tLgpY6mphXSm/A9u8Nq1ryBG+Qspiub9LjWlBPsPS3tWQ/Botq4FdxA==}
     engines: {node: '>= 8'}
@@ -19528,7 +19659,7 @@ packages:
       bl: 4.1.0
       chalk: 4.1.2
       cli-cursor: 3.1.0
-      cli-spinners: 2.6.1
+      cli-spinners: 2.9.2
       is-interactive: 1.0.0
       is-unicode-supported: 0.1.0
       log-symbols: 4.1.0
@@ -19751,6 +19882,10 @@ packages:
     resolution: {integrity: sha512-4hLB8Py4zZce5s4yd9XzopqwVv/yGNhV1Bl8NTmCq1763HeK2+EwVTv+leGeL13Dnh2wfbqowVPXCIO0z4taYw==}
     dev: true
 
+  /pako@2.1.0:
+    resolution: {integrity: sha512-w+eufiZ1WuJYgPXbV/PO3NCMEc3xqylkKHzp8bxp1uW4qaSNQUkwmLLEc3kKsfz8lpV1F8Ht3U1Cm+9Srog2ug==}
+    dev: true
+
   /param-case@3.0.4:
     resolution: {integrity: sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==}
     dependencies:
@@ -20084,6 +20219,20 @@ packages:
     hasBin: true
     dev: true
 
+  /pidusage-tree@2.0.5:
+    resolution: {integrity: sha512-4J9SkX1IorF9srgzbTrXpfO2xA4JHESDn5AGGDtCHXvVAGNvP4KzZpWwXhLDKlB+dC5rcERIKS5Z7JktjzCcCA==}
+    dependencies:
+      pidtree: 0.3.0
+      pidusage: 2.0.21
+    dev: true
+
+  /pidusage@2.0.21:
+    resolution: {integrity: sha512-cv3xAQos+pugVX+BfXpHsbyz/dLzX+lr44zNMsYiGxUw+kV5sgQCIcLd1z+0vq+KyC7dJ+/ts2PsfgWfSC3WXA==}
+    engines: {node: '>=8'}
+    dependencies:
+      safe-buffer: 5.2.1
+    dev: true
+
   /pify@2.3.0:
     resolution: {integrity: sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==}
     engines: {node: '>=0.10.0'}
@@ -22961,6 +23110,11 @@ packages:
     resolution: {integrity: sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==}
     engines: {node: '>=0.12.0'}
 
+  /run-async@3.0.0:
+    resolution: {integrity: sha512-540WwVDOMxA6dN6We19EcT9sc3hkXPw5mzRNGM3FkdN/vtE9NFvj5lFAPNwUDmJjXidm3v7TC1cTE7t17Ulm1Q==}
+    engines: {node: '>=0.12.0'}
+    dev: true
+
   /run-parallel@1.2.0:
     resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==}
     dependencies:
@@ -23250,6 +23404,18 @@ packages:
     resolution: {integrity: sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==}
     dev: true
 
+  /set-function-length@1.2.2:
+    resolution: {integrity: sha512-pgRc4hJ4/sNjWCSS9AmnS40x3bNMDTknHgL5UaMBTMyJnU90EgWh1Rz+MC9eFu4BuN/UwZjKQuY/1v3rM7HMfg==}
+    engines: {node: '>= 0.4'}
+    dependencies:
+      define-data-property: 1.1.4
+      es-errors: 1.3.0
+      function-bind: 1.1.2
+      get-intrinsic: 1.2.4
+      gopd: 1.0.1
+      has-property-descriptors: 1.0.2
+    dev: true
+
   /set-value@2.0.1:
     resolution: {integrity: sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==}
     engines: {node: '>=0.10.0'}
@@ -23709,6 +23875,11 @@ packages:
       readable-stream: 3.6.0
     dev: true
 
+  /split2@4.2.0:
+    resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==}
+    engines: {node: '>= 10.x'}
+    dev: true
+
   /split@0.2.10:
     resolution: {integrity: sha512-e0pKq+UUH2Xq/sXbYpZBZc3BawsfDZ7dgv+JtRTUPNcvF5CMR4Y9cvJqkMY0MoxWzTHvZuz1beg6pNEKlszPiQ==}
     dependencies:
@@ -23914,7 +24085,7 @@ packages:
       call-bind: 1.0.2
       define-properties: 1.1.4
       es-abstract: 1.20.2
-      get-intrinsic: 1.1.2
+      get-intrinsic: 1.2.1
       has-symbols: 1.0.3
       internal-slot: 1.0.3
       regexp.prototype.flags: 1.4.3

scripts/devlow-bench.mjs

@@ -0,0 +1,402 @@
+import { rm, writeFile, readFile } from 'node:fs/promises'
+import { join, resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { describe } from '@vercel/devlow-bench'
+import * as devlow from '@vercel/devlow-bench'
+import { newBrowserSession } from '@vercel/devlow-bench/browser'
+import { command } from '@vercel/devlow-bench/shell'
+import { waitForFile } from '@vercel/devlow-bench/file'
+
+const REPO_ROOT = fileURLToPath(new URL('..', import.meta.url))
+
+const GIT_SHA =
+  process.env.GITHUB_SHA ??
+  (await (async () => {
+    const cmd = command('git', ['rev-parse', 'HEAD'])
+    await cmd.ok()
+    return cmd.output
+  })())
+
+const GIT_BRANCH =
+  process.env.GITHUB_REF_NAME ??
+  (await (async () => {
+    const cmd = command('git', ['rev-parse', '--abbrev-ref', 'HEAD'])
+    await cmd.ok()
+    return cmd.output
+  })())
+
+const nextDevWorkflow =
+  (benchmarkName, pages) =>
+  async ({ turbopack, page }) => {
+    const pageConfig =
+      typeof pages[page] === 'string' ? { url: pages[page] } : pages[page]
+    const cleanupTasks = []
+    try {
+      const benchmarkDir = resolve(REPO_ROOT, 'bench', benchmarkName)
+
+      // cleanup .next directory to remove persistent cache
+      await retry(() =>
+        rm(join(benchmarkDir, '.next'), { recursive: true, force: true })
+      )
+
+      await measureTime('cleanup', {
+        scenario: benchmarkName,
+        props: { turbopack: null, page: null },
+      })
+
+      // startup browser
+      let session = await newBrowserSession({})
+      const closeSession = async () => {
+        if (session) {
+          await session.close()
+          session = null
+        }
+      }
+      cleanupTasks.push(closeSession)
+      await measureTime('browser startup', {
+        props: { turbopack: null, page: null },
+      })
+
+      const env = {
+        PATH: process.env.PATH,
+        NODE: process.env.NODE,
+        HOSTNAME: process.env.HOSTNAME,
+        PWD: process.env.PWD,
+        NODE_ENV: 'development',
+        // Disable otel initialization to prevent pending / hanging request to otel collector
+        OTEL_SDK_DISABLED: 'true',
+        NEXT_PUBLIC_OTEL_SENTRY: 'true',
+        NEXT_PUBLIC_OTEL_DEV_DISABLED: 'true',
+        NEXT_TRACE_UPLOAD_DISABLED: 'true',
+        // Enable next.js test mode to get HMR events
+        __NEXT_TEST_MODE: '1',
+      }
+
+      // run command to start dev server
+      const args = [turbopack ? 'dev-turbopack' : 'dev-webpack']
+      let shell = command('pnpm', args, {
+        cwd: benchmarkDir,
+        env,
+      })
+      const killShell = async () => {
+        if (shell) {
+          await shell.kill()
+          shell = null
+        }
+      }
+      cleanupTasks.push(killShell)
+
+      // wait for server to be ready
+      const START_SERVER_REGEXP = /Local:\s+(?<url>.+)\n/
+      const {
+        groups: { url },
+      } = await shell.waitForOutput(START_SERVER_REGEXP)
+      await measureTime('server startup', { props: { page: null } })
+      await shell.reportMemUsage('mem usage after startup', {
+        props: { page: null },
+      })
+
+      // open page
+      const pageInstance = await session.hardNavigation(
+        'open page',
+        url + pageConfig.url
+      )
+      await shell.reportMemUsage('mem usage after open page')
+
+      let status = 0
+      try {
+        if (
+          await pageInstance.evaluate(
+            '!next.appDir && __NEXT_DATA__.page === "/404"'
+          )
+        ) {
+          status = 2
+        }
+      } catch (e) {
+        status = 2
+      }
+
+      try {
+        if (
+          !(await pageInstance.evaluate(
+            'next.appDir || __NEXT_DATA__.page && !__NEXT_DATA__.err'
+          ))
+        ) {
+          status = 1
+        }
+      } catch (e) {
+        status = 1
+      }
+
+      await reportMeasurement('page status', status, 'status code')
+
+      // reload page
+      await session.reload('reload page')
+
+      await reportMeasurement(
+        'console output',
+        shell.output.split(/\n/).length,
+        'lines'
+      )
+
+      // HMR
+      if (pageConfig.hmr) {
+        let hmrEvent = () => {}
+        pageInstance.exposeBinding(
+          'TURBOPACK_HMR_EVENT',
+          (_source, latency) => {
+            hmrEvent(latency)
+          }
+        )
+        const { file, before, after } = pageConfig.hmr
+        const path = resolve(benchmarkDir, file)
+        const content = await readFile(path, 'utf8')
+        cleanupTasks.push(async () => {
+          await writeFile(path, content, 'utf8')
+        })
+        let currentContent = content
+        /* eslint-disable no-await-in-loop */
+        for (let hmrAttempt = 0; hmrAttempt < 10; hmrAttempt++) {
+          if (hmrAttempt > 0) {
+            await new Promise((resolve) => {
+              setTimeout(resolve, 1000)
+            })
+          }
+          const linesStart = shell.output.split(/\n/).length
+          let reportedName
+          if (hmrAttempt < 3) {
+            reportedName = 'hmr/warmup'
+          } else {
+            reportedName = 'hmr'
+          }
+          await pageInstance.evaluate(
+            'window.__NEXT_HMR_CB = (arg) => TURBOPACK_HMR_EVENT(arg); window.__NEXT_HMR_LATENCY_CB = (arg) => TURBOPACK_HMR_EVENT(arg);'
+          )
+          // eslint-disable-next-line no-loop-func
+          const hmrDone = new Promise((resolve) => {
+            let once = true
+            const end = async (code) => {
+              const success = code <= 1
+              if (!success && !reportedName) reportedName = 'hmr'
+              if (reportedName) {
+                await reportMeasurement(
+                  `${reportedName}/status`,
+                  code,
+                  'status code'
+                )
+              }
+              clearTimeout(timeout)
+              resolve(success)
+            }
+            cleanupTasks.push(async () => {
+              if (!once) return
+              once = false
+              await end(3)
+            })
+            const timeout = setTimeout(async () => {
+              if (!once) return
+              once = false
+              await end(2)
+            }, 60000)
+            hmrEvent = async (latency) => {
+              if (!once) return
+              once = false
+              if (reportedName) {
+                if (typeof latency === 'number') {
+                  await reportMeasurement(
+                    `${reportedName}/reported latency`,
+                    latency,
+                    'ms'
+                  )
+                }
+                await measureTime(reportedName, {
+                  relativeTo: `${reportedName}/start`,
+                })
+              }
+              await end(0)
+            }
+            pageInstance.once('load', async () => {
+              if (!once) return
+              once = false
+              if (reportedName) {
+                await measureTime(reportedName, {
+                  relativeTo: `${reportedName}/start`,
+                })
+              }
+              await end(1)
+            })
+          })
+          const idx = before
+            ? currentContent.indexOf(before)
+            : currentContent.indexOf(after) + after.length
+
+          let newContent = `${currentContent}\n\n/* HMR */`
+          if (file.endsWith('.tsx')) {
+            newContent = `${currentContent.slice(
+              0,
+              idx
+            )}<div id="hmr-test">HMR</div>${currentContent.slice(idx)}`
+          } else if (file.endsWith('.css')) {
+            newContent = `${currentContent.slice(
+              0,
+              idx
+            )}\n--hmr-test-${hmrAttempt}: 0;\n${currentContent.slice(idx)}`
+          } else if (file.endsWith('.mdx')) {
+            newContent = `${currentContent.slice(
+              0,
+              idx
+            )}\n\nHMR\n\n${currentContent.slice(idx)}`
+          }
+
+          if (reportedName) {
+            await measureTime(`${reportedName}/start`)
+          }
+
+          if (currentContent === newContent) {
+            throw new Error("HMR didn't change content")
+          }
+          await writeFile(path, newContent, 'utf8')
+          currentContent = newContent
+          const success = await hmrDone
+
+          if (reportedName) {
+            await reportMeasurement(
+              `console output/${reportedName}`,
+              shell.output.split(/\n/).length - linesStart,
+              'lines'
+            )
+          }
+
+          if (!success) break
+        }
+        /* eslint-enable no-await-in-loop */
+      }
+
+      if (turbopack) {
+        // close dev server and browser
+        await killShell()
+        await closeSession()
+      } else {
+        // wait for persistent cache to be written
+        const waitPromise = new Promise((resolve) => {
+          setTimeout(resolve, 5000)
+        })
+        const cacheLocation = join(
+          benchmarkDir,
+          '.next',
+          'cache',
+          'webpack',
+          'client-development'
+        )
+        await Promise.race([
+          waitForFile(join(cacheLocation, 'index.pack')),
+          waitForFile(join(cacheLocation, 'index.pack.gz')),
+        ])
+        await measureTime('cache created')
+        await waitPromise
+        await measureTime('waiting')
+
+        // close dev server and browser
+        await killShell()
+        await closeSession()
+      }
+
+      // startup new browser
+      session = await newBrowserSession({})
+      await measureTime('browser startup', {
+        props: { turbopack: null, page: null },
+      })
+
+      // run command to start dev server
+      shell = command('pnpm', args, {
+        cwd: benchmarkDir,
+        env,
+      })
+
+      // wait for server to be ready
+      const {
+        groups: { url: url2 },
+      } = await shell.waitForOutput(START_SERVER_REGEXP)
+      await shell.reportMemUsage('mem usage after startup with cache')
+
+      // open page
+      await session.hardNavigation(
+        'open page with cache',
+        url2 + pageConfig.url
+      )
+
+      await reportMeasurement(
+        'console output with cache',
+        shell.output.split(/\n/).length,
+        'lines'
+      )
+      await shell.reportMemUsage('mem usage after open page with cache')
+    } catch (e) {
+      console.log('CAUGHT', e)
+      throw e
+    } finally {
+      // This must run in order
+      // eslint-disable-next-line no-await-in-loop
+      for (const task of cleanupTasks.reverse()) await task()
+      await measureTime('shutdown')
+    }
+  }
+
+const pages = {
+  homepage: {
+    url: '/',
+    hmr: {
+      file: 'components/lodash.js',
+      before: '<h1>Client Component</h1>',
+    },
+  },
+}
+
+describe(
+  'heavy-npm-deps dev test',
+  {
+    turbopack: true,
+    page: Object.keys(pages),
+  },
+  nextDevWorkflow('heavy-npm-deps', pages)
+)
+
+async function retry(fn) {
+  let lastError
+  for (let i = 100; i < 2000; i += 100) {
+    try {
+      // eslint-disable-next-line no-await-in-loop
+      await fn()
+      return
+    } catch (e) {
+      lastError = e
+      // eslint-disable-next-line no-await-in-loop
+      await new Promise((resolve) => {
+        setTimeout(resolve, i)
+      })
+    }
+  }
+  throw lastError
+}
+
+function measureTime(name, options) {
+  return devlow.measureTime(name, {
+    props: {
+      git_sha: GIT_SHA,
+      git_branch: GIT_BRANCH,
+      ...options?.props,
+    },
+    ...options,
+  })
+}
+
+function reportMeasurement(name, value, unit, options) {
+  return devlow.reportMeasurement(name, value, unit, {
+    props: {
+      git_sha: GIT_SHA,
+      git_branch: GIT_BRANCH,
+      ...options?.props,
+    },
+    ...options,
+  })
+}