Update swc runners config (#53939)

This fixes unexpected cache hits occurring causing latest next-swc changes to not be built for the docker based builds. Noticed in our turbo run summaries for these builds that no input files were detected most likely due to older Node.js/pnpm versions being used from the old Docker image. This also updates to leverage new runner labels for better build perf cutting build times down from over an hour down to around 20 min. Fixes build issue on aarch64-linux-musl due to `TurboMalloc` not being able to resolve. Test build done here https://github.com/vercel/next.js/actions/runs/5850893681/job/15860929304 x-ref: https://github.com/vercel/next.js/actions/runs/5843580924/job/15845848384 x-ref: https://github.com/vercel/next.js/actions/runs/5837652747/job/15833506858 x-ref: https://github.com/vercel/next.js/actions/runs/5837652747/job/15833506925 x-ref: https://github.com/vercel/next.js/actions/runs/5837652747/job/15833507067 x-ref: https://github.com/vercel/next.js/actions/runs/5837652747/job/15833507135
2023-08-13 19:02:48 -07:00 · 2023-08-13 19:02:48 -07:00 · 9a6ae144a9
commit 9a6ae144a9
parent 261db496f7
6 changed files with 119 additions and 75 deletions
--- a/.github/workflows/build_and_deploy.yml
+++ b/.github/workflows/build_and_deploy.yml
@ -6,7 +6,7 @@ on:
  workflow_dispatch:

 env:
-  NAPI_CLI_VERSION: 2.14.7
+  NAPI_CLI_VERSION: 2.16.2
  TURBO_VERSION: 1.10.9
  NODE_MAINTENANCE_VERSION: 16
  NODE_LTS_VERSION: 18.16.1
@ -70,33 +70,58 @@ jobs:

  # Build binaries for publishing
  build-native:
+    defaults:
+      run:
+        shell: bash -leo pipefail {0}
+
    strategy:
      fail-fast: false
      matrix:
        settings:
          # pnpm is aliased here temporarily until the build docker
          # image is updated past Node.js v14.19 (current 14.18.1)
-          - host: macos-latest
+          - host:
+              - 'self-hosted'
+              - 'macos'
+              - 'arm64'
+
            target: 'x86_64-apple-darwin'
            build: |
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable
              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-apple-darwin --release
              strip -x packages/next-swc/native/next-swc.*.node
-          - host: windows-latest
+
+          - host:
+              - 'self-hosted'
+              - 'windows'
+              - 'x64'
+
            build: |
              corepack enable
              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}"
              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-pc-windows-msvc
            target: 'x86_64-pc-windows-msvc'
-          - host: windows-latest
+
+          - host:
+              - 'self-hosted'
+              - 'windows'
+              - 'x64'
+
            build: |
              corepack enable
              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}"
              turbo run build-native-no-plugin --remote-cache-timeout 90 --summarize -- --release --target i686-pc-windows-msvc
            target: 'i686-pc-windows-msvc'
-          - host: ubuntu-latest
+
+          - host:
+              - 'self-hosted'
+              - 'linux'
+              - 'x64'
+              - 'metal'
+              - 'docker'
+
            target: 'x86_64-unknown-linux-gnu'
-            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-x64
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
            build: >-
              set -e &&
              apt update &&
@ -104,36 +129,54 @@ jobs:
              rustup toolchain install "${RUST_TOOLCHAIN}" &&
              rustup default "${RUST_TOOLCHAIN}" &&
              rustup target add x86_64-unknown-linux-gnu &&
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable && pnpm -v && node -v &&
              unset CC_x86_64_unknown_linux_gnu && unset CC &&
              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-unknown-linux-gnu &&
              strip packages/next-swc/native/next-swc.*.node
-          - host: ubuntu-latest
+
+          - host:
+              - 'self-hosted'
+              - 'linux'
+              - 'x64'
+              - 'metal'
+              - 'docker'
+
            target: 'x86_64-unknown-linux-musl'
-            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
            build: >-
              set -e &&
              apk add --no-cache libc6-compat pkgconfig &&
              rustup toolchain install "${RUST_TOOLCHAIN}" &&
              rustup default "${RUST_TOOLCHAIN}" &&
              rustup target add x86_64-unknown-linux-musl &&
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable && pnpm -v && node -v &&
              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-unknown-linux-musl &&
              strip packages/next-swc/native/next-swc.*.node
-          - host: macos-latest
+
+          - host:
+              - 'self-hosted'
+              - 'macos'
+              - 'arm64'
+
            target: 'aarch64-apple-darwin'
            build: |
-              sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/*;
              export CC=$(xcrun -f clang);
              export CXX=$(xcrun -f clang++);
              SYSROOT=$(xcrun --sdk macosx --show-sdk-path);
              export CFLAGS="-isysroot $SYSROOT -isystem $SYSROOT";
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable
              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-apple-darwin
              strip -x packages/next-swc/native/next-swc.*.node
-          - host: ubuntu-latest
+
+          - host:
+              - 'self-hosted'
+              - 'linux'
+              - 'x64'
+              - 'metal'
+              - 'docker'
+
            target: 'aarch64-unknown-linux-gnu'
-            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-aarch64
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
            build: >-
              set -e &&
              apt update &&
@ -142,29 +185,42 @@ jobs:
              rustup toolchain install "${RUST_TOOLCHAIN}" &&
              rustup default "${RUST_TOOLCHAIN}" &&
              rustup target add aarch64-unknown-linux-gnu &&
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable &&
              export CC_aarch64_unknown_linux_gnu=/usr/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc &&
              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-unknown-linux-gnu --features plugin,rustls-tls,tracing/release_max_level_info &&
              llvm-strip -x packages/next-swc/native/next-swc.*.node
-          - host: ubuntu-latest
+
+          - host:
+              - 'self-hosted'
+              - 'linux'
+              - 'x64'
+              - 'metal'
+              - 'docker'
+
            target: 'aarch64-unknown-linux-musl'
-            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
+            docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
            build: >-
              set -e &&
              apk add --no-cache libc6-compat pkgconfig &&
              export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
-              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
+              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable &&
              rustup toolchain install "${RUST_TOOLCHAIN}" &&
              rustup default "${RUST_TOOLCHAIN}" &&
              rustup target add aarch64-unknown-linux-musl &&
              turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-unknown-linux-musl &&
              llvm-strip -x packages/next-swc/native/next-swc.*.node
-          - host: windows-latest
+
+          - host:
+              - 'self-hosted'
+              - 'windows'
+              - 'x64'
+
            target: 'aarch64-pc-windows-msvc'
            build: |
              corepack enable
              npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}"
              turbo run build-native-no-plugin-woa-release --remote-cache-timeout 90 --summarize -- --target aarch64-pc-windows-msvc
+
    name: stable - ${{ matrix.settings.target }} - node@16
    runs-on: ${{ matrix.settings.host }}
    env:
@ -202,27 +258,12 @@ jobs:
          targets: ${{ matrix.settings.target }}
          skip-install: ${{ matrix.settings.docker }}

-      - name: Cache cargo registry
-        uses: actions/cache@v3
-        timeout-minutes: 5
-        with:
-          path: ~/.cargo/registry
-          key: ${{ matrix.settings.target }}-cargo-registry
-
-      - name: Cache cargo index
-        uses: actions/cache@v3
-        timeout-minutes: 5
-        with:
-          path: ~/.cargo/git
-          key: ${{ matrix.settings.target }}-cargo-index
-
      - name: normalize versions
        run: node scripts/normalize-version-bump.js

      - name: Setup toolchain
        run: ${{ matrix.settings.setup }}
        if: ${{ matrix.settings.setup }}
-        shell: bash

      - name: Build in docker
        uses: addnab/docker-run-action@v3
@ -250,17 +291,14 @@ jobs:
      - name: 'Build'
        run: ${{ matrix.settings.build }}
        if: ${{ !matrix.settings.docker }}
-        shell: bash

      - name: 'check build cache status'
        id: check-did-build
-        shell: bash
        run: if [[ ! -z $(ls .turbo/runs) ]]; then echo "DID_BUILD=yup" >> $GITHUB_OUTPUT; fi

      # Trying to upload metrics for the Turbopack to datadog's CI pipeline execution
      - name: 'Collect turbopack build metrics'
        id: check-turbopack-bytesize
-        shell: bash
        if: ${{ steps.check-did-build.outputs.DID_BUILD == 'yup' }}
        continue-on-error: true
        run: |
@ -299,7 +337,13 @@ jobs:
    strategy:
      matrix:
        target: [web, nodejs]
-    runs-on: macos-latest
+
+    runs-on:
+      - 'self-hosted'
+      - 'linux'
+      - 'x64'
+      - 'metal'
+
    env:
      TURBO_TEAM: 'vercel'
      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
--- a/packages/next-swc/crates/next-dev/src/lib.rs
+++ b/packages/next-swc/crates/next-dev/src/lib.rs
@ -39,12 +39,11 @@ use next_core::{
 use owo_colors::OwoColorize;
 use tracing_subscriber::{prelude::*, EnvFilter, Registry};
 use turbo_tasks::{
-    util::{FormatBytes, FormatDuration},
-    StatsType, TransientInstance, TurboTasks, TurboTasksBackendApi, UpdateInfo, Value, Vc,
+    util::FormatDuration, StatsType, TransientInstance, TurboTasks, TurboTasksBackendApi,
+    UpdateInfo, Value, Vc,
 };
 use turbopack_binding::{
    turbo::{
-        malloc::TurboMalloc,
        tasks_env::{CustomProcessEnv, ProcessEnv},
        tasks_fs::{DiskFileSystem, FileSystem},
        tasks_memory::MemoryBackend,
@ -591,10 +590,9 @@ pub async fn start_server(options: &DevServerOptions) -> Result<()> {
    let stats_future = async move {
        if options.log_detail {
            println!(
-                "{event_type} - startup {start} ({memory})",
+                "{event_type} - startup {start}",
                event_type = "event".purple(),
                start = FormatDuration(start.elapsed()),
-                memory = FormatBytes(TurboMalloc::memory_usage())
            );
        }

@ -616,20 +614,18 @@ pub async fn start_server(options: &DevServerOptions) -> Result<()> {
                match (options.log_detail, !reasons.is_empty()) {
                    (true, true) => {
                        println!(
-                            "\x1b[2K{event_type} - {reasons} {elapsed} ({tasks} tasks, {memory})",
+                            "\x1b[2K{event_type} - {reasons} {elapsed} ({tasks} tasks)",
                            event_type = "event".purple(),
                            elapsed = FormatDuration(elapsed),
                            tasks = count,
-                            memory = FormatBytes(TurboMalloc::memory_usage())
                        );
                    }
                    (true, false) => {
                        println!(
-                            "\x1b[2K{event_type} - compilation {elapsed} ({tasks} tasks, {memory})",
+                            "\x1b[2K{event_type} - compilation {elapsed} ({tasks} tasks)",
                            event_type = "event".purple(),
                            elapsed = FormatDuration(elapsed),
                            tasks = count,
-                            memory = FormatBytes(TurboMalloc::memory_usage())
                        );
                    }
                    (false, true) => {
@ -651,18 +647,10 @@ pub async fn start_server(options: &DevServerOptions) -> Result<()> {
                }
            } else {
                progress_counter += 1;
-                if options.log_detail {
-                    print!(
-                        "\x1b[2K{event_type} - {progress_counter}s... ({memory})\r",
-                        event_type = "event".purple(),
-                        memory = FormatBytes(TurboMalloc::memory_usage())
-                    );
-                } else {
-                    print!(
-                        "\x1b[2K{event_type} - {progress_counter}s...\r",
-                        event_type = "event".purple(),
-                    );
-                }
+                print!(
+                    "\x1b[2K{event_type} - {progress_counter}s...\r",
+                    event_type = "event".purple(),
+                );
                let _ = stdout().lock().flush();
            }
        }
--- a/packages/next-swc/package.json
+++ b/packages/next-swc/package.json
@ -31,7 +31,7 @@
    }
  },
  "devDependencies": {
-    "@napi-rs/cli": "2.14.7",
+    "@napi-rs/cli": "2.16.2",
    "cross-env": "6.0.3"
  }
 }
--- a/packages/next/package.json
+++ b/packages/next/package.json
@ -135,7 +135,7 @@
    "@hapi/accept": "5.0.2",
    "@jest/transform": "29.5.0",
    "@jest/types": "29.5.0",
-    "@napi-rs/cli": "2.14.7",
+    "@napi-rs/cli": "2.16.2",
    "@napi-rs/triples": "1.1.0",
    "@next/polyfill-module": "13.4.14-canary.3",
    "@next/polyfill-nomodule": "13.4.14-canary.3",
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@ -815,8 +815,8 @@ importers:
        specifier: 29.5.0
        version: 29.5.0
      '@napi-rs/cli':
-        specifier: 2.14.7
-        version: 2.14.7
+        specifier: 2.16.2
+        version: 2.16.2
      '@napi-rs/triples':
        specifier: 1.1.0
        version: 1.1.0
@ -1410,8 +1410,8 @@ importers:
  packages/next-swc:
    devDependencies:
      '@napi-rs/cli':
-        specifier: 2.14.7
-        version: 2.14.7
+        specifier: 2.16.2
+        version: 2.16.2
      cross-env:
        specifier: 6.0.3
        version: 6.0.3
@ -6649,8 +6649,8 @@ packages:
      '@types/react': 18.2.7
      react: 18.2.0

-  /@napi-rs/cli@2.14.7:
-    resolution: {integrity: sha512-H+YCgdbBNMWn8c878C3RsRYcMxamWI7rB0H+1TqOy9n6u31isikcRyW1VDyoNHp7ApfmzV0+tpPopVi0W7WHAA==}
+  /@napi-rs/cli@2.16.2:
+    resolution: {integrity: sha512-U2aZfnr0s9KkXpZlYC0l5WxWCXL7vJUNpCnWMwq3T9GG9rhYAAUM9CTZsi1Z+0iR2LcHbfq9EfMgoqnuTyUjfg==}
    engines: {node: '>= 10'}
    hasBin: true
    dev: true
--- a/turbo.json
+++ b/turbo.json
@ -6,7 +6,9 @@
        "../../.cargo/**",
        "../../packages/next-swc/crates/**",
        "../../**/Cargo.toml",
-        "../../**/Cargo.lock"
+        "../../**/Cargo.lock",
+        "../../.github/workflows/build_and_deploy.yml",
+        "../../rust-toolchain"
      ],
      "dependsOn": ["^build-native"],
      "outputs": ["native/*.node"]
@ -16,7 +18,9 @@
        "../../.cargo/**",
        "../../packages/next-swc/crates/**",
        "../../**/Cargo.toml",
-        "../../**/Cargo.lock"
+        "../../**/Cargo.lock",
+        "../../.github/workflows/build_and_deploy.yml",
+        "../../rust-toolchain"
      ],
      "dependsOn": ["^build-native-release"],
      "outputs": ["native/*.node"]
@ -26,7 +30,9 @@
        "../../.cargo/**",
        "../../packages/next-swc/crates/**",
        "../../**/Cargo.toml",
-        "../../**/Cargo.lock"
+        "../../**/Cargo.lock",
+        "../../.github/workflows/build_and_deploy.yml",
+        "../../rust-toolchain"
      ],
      "dependsOn": ["^build-native-no-plugin"],
      "outputs": ["native/*.node"]
@ -36,7 +42,9 @@
        "../../.cargo/**",
        "../../packages/next-swc/crates/**",
        "../../**/Cargo.toml",
-        "../../**/Cargo.lock"
+        "../../**/Cargo.lock",
+        "../../.github/workflows/build_and_deploy.yml",
+        "../../rust-toolchain"
      ],
      "dependsOn": ["^build-native-no-plugin-woa"],
      "outputs": ["native/*.node"]
@ -46,7 +54,9 @@
        "../../.cargo/**",
        "../../packages/next-swc/crates/**",
        "../../**/Cargo.toml",
-        "../../**/Cargo.lock"
+        "../../**/Cargo.lock",
+        "../../.github/workflows/build_and_deploy.yml",
+        "../../rust-toolchain"
      ],
      "dependsOn": ["^build-native-no-plugin-woa-release"],
      "outputs": ["native/*.node"]
@ -56,7 +66,9 @@
        "../../.cargo/**",
        "../../packages/next-swc/crates/**",
        "../../**/Cargo.toml",
-        "../../**/Cargo.lock"
+        "../../**/Cargo.lock",
+        "../../.github/workflows/build_and_deploy.yml",
+        "../../rust-toolchain"
      ],
      "dependsOn": ["^build-wasm"],
      "outputs": ["crates/wasm/pkg/*"]