Update swc runners config (#53939)
This fixes unexpected cache hits occurring causing latest next-swc changes to not be built for the docker based builds. Noticed in our turbo run summaries for these builds that no input files were detected most likely due to older Node.js/pnpm versions being used from the old Docker image. This also updates to leverage new runner labels for better build perf cutting build times down from over an hour down to around 20 min. Fixes build issue on aarch64-linux-musl due to `TurboMalloc` not being able to resolve. Test build done here https://github.com/vercel/next.js/actions/runs/5850893681/job/15860929304 x-ref: https://github.com/vercel/next.js/actions/runs/5843580924/job/15845848384 x-ref: https://github.com/vercel/next.js/actions/runs/5837652747/job/15833506858 x-ref: https://github.com/vercel/next.js/actions/runs/5837652747/job/15833506925 x-ref: https://github.com/vercel/next.js/actions/runs/5837652747/job/15833507067 x-ref: https://github.com/vercel/next.js/actions/runs/5837652747/job/15833507135
This commit is contained in:
parent
261db496f7
commit
9a6ae144a9
6 changed files with 119 additions and 75 deletions
124
.github/workflows/build_and_deploy.yml
vendored
124
.github/workflows/build_and_deploy.yml
vendored
|
@ -6,7 +6,7 @@ on:
|
|||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
NAPI_CLI_VERSION: 2.14.7
|
||||
NAPI_CLI_VERSION: 2.16.2
|
||||
TURBO_VERSION: 1.10.9
|
||||
NODE_MAINTENANCE_VERSION: 16
|
||||
NODE_LTS_VERSION: 18.16.1
|
||||
|
@ -70,33 +70,58 @@ jobs:
|
|||
|
||||
# Build binaries for publishing
|
||||
build-native:
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -leo pipefail {0}
|
||||
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
settings:
|
||||
# pnpm is aliased here temporarily until the build docker
|
||||
# image is updated past Node.js v14.19 (current 14.18.1)
|
||||
- host: macos-latest
|
||||
- host:
|
||||
- 'self-hosted'
|
||||
- 'macos'
|
||||
- 'arm64'
|
||||
|
||||
target: 'x86_64-apple-darwin'
|
||||
build: |
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable
|
||||
turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-apple-darwin --release
|
||||
strip -x packages/next-swc/native/next-swc.*.node
|
||||
- host: windows-latest
|
||||
|
||||
- host:
|
||||
- 'self-hosted'
|
||||
- 'windows'
|
||||
- 'x64'
|
||||
|
||||
build: |
|
||||
corepack enable
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}"
|
||||
turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-pc-windows-msvc
|
||||
target: 'x86_64-pc-windows-msvc'
|
||||
- host: windows-latest
|
||||
|
||||
- host:
|
||||
- 'self-hosted'
|
||||
- 'windows'
|
||||
- 'x64'
|
||||
|
||||
build: |
|
||||
corepack enable
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}"
|
||||
turbo run build-native-no-plugin --remote-cache-timeout 90 --summarize -- --release --target i686-pc-windows-msvc
|
||||
target: 'i686-pc-windows-msvc'
|
||||
- host: ubuntu-latest
|
||||
|
||||
- host:
|
||||
- 'self-hosted'
|
||||
- 'linux'
|
||||
- 'x64'
|
||||
- 'metal'
|
||||
- 'docker'
|
||||
|
||||
target: 'x86_64-unknown-linux-gnu'
|
||||
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-x64
|
||||
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian
|
||||
build: >-
|
||||
set -e &&
|
||||
apt update &&
|
||||
|
@ -104,36 +129,54 @@ jobs:
|
|||
rustup toolchain install "${RUST_TOOLCHAIN}" &&
|
||||
rustup default "${RUST_TOOLCHAIN}" &&
|
||||
rustup target add x86_64-unknown-linux-gnu &&
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable && pnpm -v && node -v &&
|
||||
unset CC_x86_64_unknown_linux_gnu && unset CC &&
|
||||
turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-unknown-linux-gnu &&
|
||||
strip packages/next-swc/native/next-swc.*.node
|
||||
- host: ubuntu-latest
|
||||
|
||||
- host:
|
||||
- 'self-hosted'
|
||||
- 'linux'
|
||||
- 'x64'
|
||||
- 'metal'
|
||||
- 'docker'
|
||||
|
||||
target: 'x86_64-unknown-linux-musl'
|
||||
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
|
||||
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
|
||||
build: >-
|
||||
set -e &&
|
||||
apk add --no-cache libc6-compat pkgconfig &&
|
||||
rustup toolchain install "${RUST_TOOLCHAIN}" &&
|
||||
rustup default "${RUST_TOOLCHAIN}" &&
|
||||
rustup target add x86_64-unknown-linux-musl &&
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable && pnpm -v && node -v &&
|
||||
turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target x86_64-unknown-linux-musl &&
|
||||
strip packages/next-swc/native/next-swc.*.node
|
||||
- host: macos-latest
|
||||
|
||||
- host:
|
||||
- 'self-hosted'
|
||||
- 'macos'
|
||||
- 'arm64'
|
||||
|
||||
target: 'aarch64-apple-darwin'
|
||||
build: |
|
||||
sudo rm -Rf /Library/Developer/CommandLineTools/SDKs/*;
|
||||
export CC=$(xcrun -f clang);
|
||||
export CXX=$(xcrun -f clang++);
|
||||
SYSROOT=$(xcrun --sdk macosx --show-sdk-path);
|
||||
export CFLAGS="-isysroot $SYSROOT -isystem $SYSROOT";
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable
|
||||
turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-apple-darwin
|
||||
strip -x packages/next-swc/native/next-swc.*.node
|
||||
- host: ubuntu-latest
|
||||
|
||||
- host:
|
||||
- 'self-hosted'
|
||||
- 'linux'
|
||||
- 'x64'
|
||||
- 'metal'
|
||||
- 'docker'
|
||||
|
||||
target: 'aarch64-unknown-linux-gnu'
|
||||
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-aarch64
|
||||
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-debian-aarch64
|
||||
build: >-
|
||||
set -e &&
|
||||
apt update &&
|
||||
|
@ -142,29 +185,42 @@ jobs:
|
|||
rustup toolchain install "${RUST_TOOLCHAIN}" &&
|
||||
rustup default "${RUST_TOOLCHAIN}" &&
|
||||
rustup target add aarch64-unknown-linux-gnu &&
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable &&
|
||||
export CC_aarch64_unknown_linux_gnu=/usr/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc &&
|
||||
turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-unknown-linux-gnu --features plugin,rustls-tls,tracing/release_max_level_info &&
|
||||
llvm-strip -x packages/next-swc/native/next-swc.*.node
|
||||
- host: ubuntu-latest
|
||||
|
||||
- host:
|
||||
- 'self-hosted'
|
||||
- 'linux'
|
||||
- 'x64'
|
||||
- 'metal'
|
||||
- 'docker'
|
||||
|
||||
target: 'aarch64-unknown-linux-musl'
|
||||
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:stable-2022-10-24-alpine
|
||||
docker: ghcr.io/napi-rs/napi-rs/nodejs-rust:lts-alpine
|
||||
build: >-
|
||||
set -e &&
|
||||
apk add --no-cache libc6-compat pkgconfig &&
|
||||
export JEMALLOC_SYS_WITH_LG_PAGE=16 &&
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && corepack enable &&
|
||||
rustup toolchain install "${RUST_TOOLCHAIN}" &&
|
||||
rustup default "${RUST_TOOLCHAIN}" &&
|
||||
rustup target add aarch64-unknown-linux-musl &&
|
||||
turbo run build-native-release --remote-cache-timeout 90 --summarize -- --target aarch64-unknown-linux-musl &&
|
||||
llvm-strip -x packages/next-swc/native/next-swc.*.node
|
||||
- host: windows-latest
|
||||
|
||||
- host:
|
||||
- 'self-hosted'
|
||||
- 'windows'
|
||||
- 'x64'
|
||||
|
||||
target: 'aarch64-pc-windows-msvc'
|
||||
build: |
|
||||
corepack enable
|
||||
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}"
|
||||
turbo run build-native-no-plugin-woa-release --remote-cache-timeout 90 --summarize -- --target aarch64-pc-windows-msvc
|
||||
|
||||
name: stable - ${{ matrix.settings.target }} - node@16
|
||||
runs-on: ${{ matrix.settings.host }}
|
||||
env:
|
||||
|
@ -202,27 +258,12 @@ jobs:
|
|||
targets: ${{ matrix.settings.target }}
|
||||
skip-install: ${{ matrix.settings.docker }}
|
||||
|
||||
- name: Cache cargo registry
|
||||
uses: actions/cache@v3
|
||||
timeout-minutes: 5
|
||||
with:
|
||||
path: ~/.cargo/registry
|
||||
key: ${{ matrix.settings.target }}-cargo-registry
|
||||
|
||||
- name: Cache cargo index
|
||||
uses: actions/cache@v3
|
||||
timeout-minutes: 5
|
||||
with:
|
||||
path: ~/.cargo/git
|
||||
key: ${{ matrix.settings.target }}-cargo-index
|
||||
|
||||
- name: normalize versions
|
||||
run: node scripts/normalize-version-bump.js
|
||||
|
||||
- name: Setup toolchain
|
||||
run: ${{ matrix.settings.setup }}
|
||||
if: ${{ matrix.settings.setup }}
|
||||
shell: bash
|
||||
|
||||
- name: Build in docker
|
||||
uses: addnab/docker-run-action@v3
|
||||
|
@ -250,17 +291,14 @@ jobs:
|
|||
- name: 'Build'
|
||||
run: ${{ matrix.settings.build }}
|
||||
if: ${{ !matrix.settings.docker }}
|
||||
shell: bash
|
||||
|
||||
- name: 'check build cache status'
|
||||
id: check-did-build
|
||||
shell: bash
|
||||
run: if [[ ! -z $(ls .turbo/runs) ]]; then echo "DID_BUILD=yup" >> $GITHUB_OUTPUT; fi
|
||||
|
||||
# Trying to upload metrics for the Turbopack to datadog's CI pipeline execution
|
||||
- name: 'Collect turbopack build metrics'
|
||||
id: check-turbopack-bytesize
|
||||
shell: bash
|
||||
if: ${{ steps.check-did-build.outputs.DID_BUILD == 'yup' }}
|
||||
continue-on-error: true
|
||||
run: |
|
||||
|
@ -299,7 +337,13 @@ jobs:
|
|||
strategy:
|
||||
matrix:
|
||||
target: [web, nodejs]
|
||||
runs-on: macos-latest
|
||||
|
||||
runs-on:
|
||||
- 'self-hosted'
|
||||
- 'linux'
|
||||
- 'x64'
|
||||
- 'metal'
|
||||
|
||||
env:
|
||||
TURBO_TEAM: 'vercel'
|
||||
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
|
||||
|
|
|
@ -39,12 +39,11 @@ use next_core::{
|
|||
use owo_colors::OwoColorize;
|
||||
use tracing_subscriber::{prelude::*, EnvFilter, Registry};
|
||||
use turbo_tasks::{
|
||||
util::{FormatBytes, FormatDuration},
|
||||
StatsType, TransientInstance, TurboTasks, TurboTasksBackendApi, UpdateInfo, Value, Vc,
|
||||
util::FormatDuration, StatsType, TransientInstance, TurboTasks, TurboTasksBackendApi,
|
||||
UpdateInfo, Value, Vc,
|
||||
};
|
||||
use turbopack_binding::{
|
||||
turbo::{
|
||||
malloc::TurboMalloc,
|
||||
tasks_env::{CustomProcessEnv, ProcessEnv},
|
||||
tasks_fs::{DiskFileSystem, FileSystem},
|
||||
tasks_memory::MemoryBackend,
|
||||
|
@ -591,10 +590,9 @@ pub async fn start_server(options: &DevServerOptions) -> Result<()> {
|
|||
let stats_future = async move {
|
||||
if options.log_detail {
|
||||
println!(
|
||||
"{event_type} - startup {start} ({memory})",
|
||||
"{event_type} - startup {start}",
|
||||
event_type = "event".purple(),
|
||||
start = FormatDuration(start.elapsed()),
|
||||
memory = FormatBytes(TurboMalloc::memory_usage())
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -616,20 +614,18 @@ pub async fn start_server(options: &DevServerOptions) -> Result<()> {
|
|||
match (options.log_detail, !reasons.is_empty()) {
|
||||
(true, true) => {
|
||||
println!(
|
||||
"\x1b[2K{event_type} - {reasons} {elapsed} ({tasks} tasks, {memory})",
|
||||
"\x1b[2K{event_type} - {reasons} {elapsed} ({tasks} tasks)",
|
||||
event_type = "event".purple(),
|
||||
elapsed = FormatDuration(elapsed),
|
||||
tasks = count,
|
||||
memory = FormatBytes(TurboMalloc::memory_usage())
|
||||
);
|
||||
}
|
||||
(true, false) => {
|
||||
println!(
|
||||
"\x1b[2K{event_type} - compilation {elapsed} ({tasks} tasks, {memory})",
|
||||
"\x1b[2K{event_type} - compilation {elapsed} ({tasks} tasks)",
|
||||
event_type = "event".purple(),
|
||||
elapsed = FormatDuration(elapsed),
|
||||
tasks = count,
|
||||
memory = FormatBytes(TurboMalloc::memory_usage())
|
||||
);
|
||||
}
|
||||
(false, true) => {
|
||||
|
@ -651,18 +647,10 @@ pub async fn start_server(options: &DevServerOptions) -> Result<()> {
|
|||
}
|
||||
} else {
|
||||
progress_counter += 1;
|
||||
if options.log_detail {
|
||||
print!(
|
||||
"\x1b[2K{event_type} - {progress_counter}s... ({memory})\r",
|
||||
event_type = "event".purple(),
|
||||
memory = FormatBytes(TurboMalloc::memory_usage())
|
||||
);
|
||||
} else {
|
||||
print!(
|
||||
"\x1b[2K{event_type} - {progress_counter}s...\r",
|
||||
event_type = "event".purple(),
|
||||
);
|
||||
}
|
||||
print!(
|
||||
"\x1b[2K{event_type} - {progress_counter}s...\r",
|
||||
event_type = "event".purple(),
|
||||
);
|
||||
let _ = stdout().lock().flush();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -31,7 +31,7 @@
|
|||
}
|
||||
},
|
||||
"devDependencies": {
|
||||
"@napi-rs/cli": "2.14.7",
|
||||
"@napi-rs/cli": "2.16.2",
|
||||
"cross-env": "6.0.3"
|
||||
}
|
||||
}
|
||||
|
|
|
@ -135,7 +135,7 @@
|
|||
"@hapi/accept": "5.0.2",
|
||||
"@jest/transform": "29.5.0",
|
||||
"@jest/types": "29.5.0",
|
||||
"@napi-rs/cli": "2.14.7",
|
||||
"@napi-rs/cli": "2.16.2",
|
||||
"@napi-rs/triples": "1.1.0",
|
||||
"@next/polyfill-module": "13.4.14-canary.3",
|
||||
"@next/polyfill-nomodule": "13.4.14-canary.3",
|
||||
|
|
|
@ -815,8 +815,8 @@ importers:
|
|||
specifier: 29.5.0
|
||||
version: 29.5.0
|
||||
'@napi-rs/cli':
|
||||
specifier: 2.14.7
|
||||
version: 2.14.7
|
||||
specifier: 2.16.2
|
||||
version: 2.16.2
|
||||
'@napi-rs/triples':
|
||||
specifier: 1.1.0
|
||||
version: 1.1.0
|
||||
|
@ -1410,8 +1410,8 @@ importers:
|
|||
packages/next-swc:
|
||||
devDependencies:
|
||||
'@napi-rs/cli':
|
||||
specifier: 2.14.7
|
||||
version: 2.14.7
|
||||
specifier: 2.16.2
|
||||
version: 2.16.2
|
||||
cross-env:
|
||||
specifier: 6.0.3
|
||||
version: 6.0.3
|
||||
|
@ -6649,8 +6649,8 @@ packages:
|
|||
'@types/react': 18.2.7
|
||||
react: 18.2.0
|
||||
|
||||
/@napi-rs/cli@2.14.7:
|
||||
resolution: {integrity: sha512-H+YCgdbBNMWn8c878C3RsRYcMxamWI7rB0H+1TqOy9n6u31isikcRyW1VDyoNHp7ApfmzV0+tpPopVi0W7WHAA==}
|
||||
/@napi-rs/cli@2.16.2:
|
||||
resolution: {integrity: sha512-U2aZfnr0s9KkXpZlYC0l5WxWCXL7vJUNpCnWMwq3T9GG9rhYAAUM9CTZsi1Z+0iR2LcHbfq9EfMgoqnuTyUjfg==}
|
||||
engines: {node: '>= 10'}
|
||||
hasBin: true
|
||||
dev: true
|
||||
|
|
24
turbo.json
24
turbo.json
|
@ -6,7 +6,9 @@
|
|||
"../../.cargo/**",
|
||||
"../../packages/next-swc/crates/**",
|
||||
"../../**/Cargo.toml",
|
||||
"../../**/Cargo.lock"
|
||||
"../../**/Cargo.lock",
|
||||
"../../.github/workflows/build_and_deploy.yml",
|
||||
"../../rust-toolchain"
|
||||
],
|
||||
"dependsOn": ["^build-native"],
|
||||
"outputs": ["native/*.node"]
|
||||
|
@ -16,7 +18,9 @@
|
|||
"../../.cargo/**",
|
||||
"../../packages/next-swc/crates/**",
|
||||
"../../**/Cargo.toml",
|
||||
"../../**/Cargo.lock"
|
||||
"../../**/Cargo.lock",
|
||||
"../../.github/workflows/build_and_deploy.yml",
|
||||
"../../rust-toolchain"
|
||||
],
|
||||
"dependsOn": ["^build-native-release"],
|
||||
"outputs": ["native/*.node"]
|
||||
|
@ -26,7 +30,9 @@
|
|||
"../../.cargo/**",
|
||||
"../../packages/next-swc/crates/**",
|
||||
"../../**/Cargo.toml",
|
||||
"../../**/Cargo.lock"
|
||||
"../../**/Cargo.lock",
|
||||
"../../.github/workflows/build_and_deploy.yml",
|
||||
"../../rust-toolchain"
|
||||
],
|
||||
"dependsOn": ["^build-native-no-plugin"],
|
||||
"outputs": ["native/*.node"]
|
||||
|
@ -36,7 +42,9 @@
|
|||
"../../.cargo/**",
|
||||
"../../packages/next-swc/crates/**",
|
||||
"../../**/Cargo.toml",
|
||||
"../../**/Cargo.lock"
|
||||
"../../**/Cargo.lock",
|
||||
"../../.github/workflows/build_and_deploy.yml",
|
||||
"../../rust-toolchain"
|
||||
],
|
||||
"dependsOn": ["^build-native-no-plugin-woa"],
|
||||
"outputs": ["native/*.node"]
|
||||
|
@ -46,7 +54,9 @@
|
|||
"../../.cargo/**",
|
||||
"../../packages/next-swc/crates/**",
|
||||
"../../**/Cargo.toml",
|
||||
"../../**/Cargo.lock"
|
||||
"../../**/Cargo.lock",
|
||||
"../../.github/workflows/build_and_deploy.yml",
|
||||
"../../rust-toolchain"
|
||||
],
|
||||
"dependsOn": ["^build-native-no-plugin-woa-release"],
|
||||
"outputs": ["native/*.node"]
|
||||
|
@ -56,7 +66,9 @@
|
|||
"../../.cargo/**",
|
||||
"../../packages/next-swc/crates/**",
|
||||
"../../**/Cargo.toml",
|
||||
"../../**/Cargo.lock"
|
||||
"../../**/Cargo.lock",
|
||||
"../../.github/workflows/build_and_deploy.yml",
|
||||
"../../rust-toolchain"
|
||||
],
|
||||
"dependsOn": ["^build-wasm"],
|
||||
"outputs": ["crates/wasm/pkg/*"]
|
||||
|
|
Loading…
Reference in a new issue