archive/rsnext
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update caching for swc turbo builds (#42929)

Browse source 
Updates to leverage turbo remote caching for the swc builds and
normalize the package versions to prevent repo version bumps from
un-necessarily invalidating the cache.

Full cache hits can be seen in this test run
https://github.com/vercel/next.js/actions/runs/3469932131/jobs/5797559609
This commit is contained in:
JJ Kasper 2022-11-15 08:24:04 -08:00 committed by GitHub
parent a0e51781e8
commit ea9c5aac5f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 171 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

108
.github/workflows/build_test_deploy.yml vendored
View file

@ -1030,6 +1030,9 @@ jobs:
turbo-${{ github.job }}-${{ github.ref_name }}-${{ steps.get-week.outputs.WEEK }}-
turbo-${{ github.job }}-canary-${{ steps.get-week.outputs.WEEK }}-
- name: normalize versions
run: node scripts/normalize-version-bump.js
# We use restore-key to pick latest cache.
# We will not get exact match, but doc says
# "If there are multiple partial matches for a restore key, the action returns the most recently created cache."
@ -1153,17 +1156,17 @@ jobs:
target: 'x86_64-apple-darwin'
build: |
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi
turbo run build-native --cache-dir=".turbo" -- --release
turbo run build-native -- --release
strip -x packages/next-swc/native/next-swc.*.node
- host: windows-latest
build: |
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}"
turbo run build-native --cache-dir=".turbo" -- --release
turbo run build-native -- --release
target: 'x86_64-pc-windows-msvc'
- host: windows-latest
build: |
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}"
turbo run build-native-no-plugin --cache-dir=".turbo" -- --release --target i686-pc-windows-msvc
turbo run build-native-no-plugin -- --release --target i686-pc-windows-msvc
target: 'i686-pc-windows-msvc'
- host: ubuntu-latest
target: 'x86_64-unknown-linux-gnu'
@ -1175,7 +1178,7 @@ jobs:
rustup target add x86_64-unknown-linux-gnu &&
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
unset CC_x86_64_unknown_linux_gnu && unset CC &&
turbo run build-native --cache-dir=".turbo" -- --release --target x86_64-unknown-linux-gnu &&
turbo run build-native -- --release --target x86_64-unknown-linux-gnu &&
strip packages/next-swc/native/next-swc.*.node
- host: ubuntu-latest
target: 'x86_64-unknown-linux-musl'
@ -1187,7 +1190,7 @@ jobs:
rustup default "${RUST_TOOLCHAIN}" &&
rustup target add x86_64-unknown-linux-musl &&
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
turbo run build-native --cache-dir=".turbo" -- --release --target x86_64-unknown-linux-musl &&
turbo run build-native -- --release --target x86_64-unknown-linux-musl &&
strip packages/next-swc/native/next-swc.*.node
- host: macos-latest
target: 'aarch64-apple-darwin'
@ -1198,7 +1201,7 @@ jobs:
SYSROOT=$(xcrun --sdk macosx --show-sdk-path);
export CFLAGS="-isysroot $SYSROOT -isystem $SYSROOT";
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi
turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-apple-darwin
turbo run build-native -- --release --target aarch64-apple-darwin
strip -x packages/next-swc/native/next-swc.*.node
- host: ubuntu-latest
target: 'aarch64-unknown-linux-gnu'
@ -1211,7 +1214,7 @@ jobs:
rustup target add aarch64-unknown-linux-gnu &&
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi &&
export CC_aarch64_unknown_linux_gnu=/usr/aarch64-unknown-linux-gnu/bin/aarch64-unknown-linux-gnu-gcc &&
turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-unknown-linux-gnu &&
turbo run build-native -- --release --target aarch64-unknown-linux-gnu &&
llvm-strip -x packages/next-swc/native/next-swc.*.node
- host: ubuntu-latest
target: 'armv7-unknown-linux-gnueabihf'
@ -1220,7 +1223,7 @@ jobs:
sudo apt-get install gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf -y
build: |
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi
turbo run build-native-no-plugin --cache-dir=".turbo" -- --release --target armv7-unknown-linux-gnueabihf
turbo run build-native-no-plugin -- --release --target armv7-unknown-linux-gnueabihf
arm-linux-gnueabihf-strip packages/next-swc/native/next-swc.*.node
- host: ubuntu-latest
target: aarch64-linux-android
@ -1235,7 +1238,7 @@ jobs:
chmod 777 "${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/${CLANG_VERSION}/lib/linux/aarch64/libgcc.a"
echo "INPUT(-lunwind)" > "${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/${CLANG_VERSION}/lib/linux/aarch64/libgcc.a"
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" && if [ ! -f $(dirname $(which yarn))/pnpm ]; then ln -s $(which yarn) $(dirname $(which yarn))/pnpm;fi
turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-linux-android
turbo run build-native -- --release --target aarch64-linux-android
${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip packages/next-swc/native/next-swc.*.node
- host: ubuntu-latest
target: armv7-linux-androideabi
@ -1250,7 +1253,7 @@ jobs:
chmod 777 "${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/${CLANG_VERSION}/lib/linux/arm/libgcc.a"
echo "INPUT(-lunwind)" > "${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/lib64/clang/${CLANG_VERSION}/lib/linux/arm/libgcc.a"
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}"
turbo run build-native-no-plugin --cache-dir=".turbo" -- --release --target armv7-linux-androideabi
turbo run build-native-no-plugin -- --release --target armv7-linux-androideabi
${ANDROID_NDK_LATEST_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip packages/next-swc/native/next-swc.*.node
- host: ubuntu-latest
target: 'aarch64-unknown-linux-musl'
@ -1263,17 +1266,21 @@ jobs:
rustup toolchain install "${RUST_TOOLCHAIN}" &&
rustup default "${RUST_TOOLCHAIN}" &&
rustup target add aarch64-unknown-linux-musl &&
turbo run build-native --cache-dir=".turbo" -- --release --target aarch64-unknown-linux-musl &&
turbo run build-native -- --release --target aarch64-unknown-linux-musl &&
llvm-strip -x packages/next-swc/native/next-swc.*.node
- host: windows-latest
target: 'aarch64-pc-windows-msvc'
build: |
npm i -g "@napi-rs/cli@${NAPI_CLI_VERSION}" "turbo@${TURBO_VERSION}" "pnpm@${PNPM_VERSION}"
turbo run build-native-no-plugin --cache-dir=".turbo" -- --release --target aarch64-pc-windows-msvc --cargo-flags=--no-default-features
turbo run build-native-no-plugin -- --release --target aarch64-pc-windows-msvc --cargo-flags=--no-default-features
if: ${{ needs.build.outputs.isRelease == 'true' }}
needs: build
name: stable - ${{ matrix.settings.target }} - node@16
runs-on: ${{ matrix.settings.host }}
env:
TURBO_TEAM: 'vercel'
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
TURBO_REMOTE_ONLY: 'true'
steps:
# https://github.com/actions/virtual-environments/issues/1187
- name: tune linux network
@ -1302,16 +1309,6 @@ jobs:
key: next-swc-cargo-cache-${{ matrix.settings.target }}--${{ hashFiles('**/Cargo.lock') }}
restore-keys: |
next-swc-cargo-cache-${{ matrix.settings.target }}
- name: Turbo Cache
id: turbo-cache
uses: actions/cache@v3
with:
path: .turbo
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ matrix.settings.target }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }}
restore-keys: |
turbo-${{ github.job }}-${{ github.ref_name }}-${{ matrix.settings.target }}
turbo-${{ github.job }}-${{ github.ref_name }}-${{ matrix.settings.target }}-${{ needs.build.outputs.weekNum }}-
turbo-${{ github.job }}-canary-${{ matrix.settings.target }}-${{ needs.build.outputs.weekNum }}-
- name: Setup node
uses: actions/setup-node@v3
@ -1341,6 +1338,9 @@ jobs:
path: ~/.cargo/git
key: ${{ matrix.settings.target }}-cargo-index
- name: normalize versions
run: node scripts/normalize-version-bump.js
- name: Setup toolchain
run: ${{ matrix.settings.setup }}
if: ${{ matrix.settings.setup }}
@ -1351,7 +1351,7 @@ jobs:
if: ${{ matrix.settings.docker }}
with:
image: ${{ matrix.settings.docker }}
options: -e RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} -e NAPI_CLI_VERSION=${{ env.NAPI_CLI_VERSION }} -e TURBO_VERSION=${{ env.TURBO_VERSION }} -v ${{ env.HOME }}/.cargo/git:/root/.cargo/git -v ${{ env.HOME }}/.cargo/registry:/root/.cargo/registry -v ${{ github.workspace }}:/build -w /build
options: -e RUST_TOOLCHAIN=${{ env.RUST_TOOLCHAIN }} -e NAPI_CLI_VERSION=${{ env.NAPI_CLI_VERSION }} -e TURBO_VERSION=${{ env.TURBO_VERSION }} -e TURBO_TEAM=vercel -e TURBO_TOKEN=${{ secrets.TURBO_TOKEN }} -e TURBO_REMOTE_ONLY=true -v ${{ env.HOME }}/.cargo/git:/root/.cargo/git -v ${{ env.HOME }}/.cargo/registry:/root/.cargo/registry -v ${{ github.workspace }}:/build -w /build
run: ${{ matrix.settings.build }}
- name: 'Build'
@ -1370,21 +1370,48 @@ jobs:
needs: build
name: stable - x86_64-unknown-freebsd - node@16
runs-on: macos-12
env:
TURBO_TEAM: 'vercel'
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
TURBO_REMOTE_ONLY: 'true'
steps:
- name: tune mac network
run: sudo sysctl -w net.link.generic.system.hwcksum_tx=0 && sudo sysctl -w net.link.generic.system.hwcksum_rx=0
- uses: actions/checkout@v3
- name: Setup node
uses: actions/setup-node@v3
with:
node-version: 16
check-latest: true
- run: npm i -g turbo@${{ env.TURBO_VERSION }} pnpm@${PNPM_VERSION}
- name: Delete useless files
run: |
rm -rf bench
rm -rf docs
rm -rf errors
rm -rf examples
rm -rf scripts
rm -rf test
- run: node scripts/normalize-version-bump.js
# attempt pulling existing turbo cache, this must be done
# outside of the freeBSD vm as there isn't a turbo build
# for freeBSD currently
- run: node ./scripts/pull-freebsd-cache.js
- name: check build exists
run: if [ -f packages/next-swc/native/next-swc.freebsd-x64.node ]; then echo "::set-output name=BUILD_EXISTS::yes"; else echo "::set-output name=BUILD_EXISTS::no"; fi
id: build-exists
- run: echo ${{steps.build-exists.outputs.BUILD_EXISTS}}
- name: Build
id: build
uses: vmactions/freebsd-vm@v0.3.0
uses: vmactions/freebsd-vm@v0
if: ${{ steps.build-exists.outputs.BUILD_EXISTS == 'no' }}
env:
DEBUG: napi:*
RUSTUP_HOME: /usr/local/rustup
@ -1393,7 +1420,7 @@ jobs:
# Disable LTO, or the lld may crash with OOM
CARGO_PROFILE_RELEASE_LTO: false
with:
envs: DEBUG RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS CARGO_PROFILE_RELEASE_LTO NAPI_CLI_VERSION TURBO_VERSION RUST_TOOLCHAIN PNPM_VERSION
envs: DEBUG RUSTUP_HOME CARGO_HOME RUSTUP_IO_THREADS CARGO_PROFILE_RELEASE_LTO NAPI_CLI_VERSION RUST_TOOLCHAIN PNPM_VERSION
usesh: true
mem: 6000
prepare: |
@ -1417,6 +1444,11 @@ jobs:
pnpm --filter=@next/swc run build-native-no-plugin --platform --release --target x86_64-unknown-freebsd
rm -rf node_modules
rm -rf packages/next-swc/target
- name: cache build
if: ${{ steps.build-exists.outputs.BUILD_EXISTS == 'no' }}
run: pnpm turbo run cache-build-native -- --platform --release --target x86_64-unknown-freebsd
- name: Upload artifact
uses: actions/upload-artifact@v3
with:
@ -1431,6 +1463,10 @@ jobs:
matrix:
target: [web, nodejs]
runs-on: macos-latest
env:
TURBO_TEAM: 'vercel'
TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
TURBO_REMOTE_ONLY: 'true'
steps:
- uses: actions/cache@v3
id: restore-build
@ -1454,22 +1490,14 @@ jobs:
- run: npm i -g turbo@${{ env.TURBO_VERSION }} pnpm@${PNPM_VERSION}
- name: Turbo cache
id: turbo-cache
uses: actions/cache@v3
with:
path: .turbo
key: turbo-${{ github.job }}-${{ matrix.target }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-${{ github.sha }}
restore-keys: |
turbo-${{ github.job }}-${{ matrix.target }}-
turbo-${{ github.job }}-${{ matrix.target }}-${{ github.ref_name }}-${{ needs.build.outputs.weekNum }}-
turbo-${{ github.job }}-${{ matrix.target }}-canary-${{ needs.build.outputs.weekNum }}-
- name: Install wasm-pack
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
- name: normalize versions
run: node scripts/normalize-version-bump.js
- name: Build
run: turbo run build-wasm --cache-dir=".turbo" -- --target ${{ matrix.target }}
run: turbo run build-wasm -- --target ${{ matrix.target }}
- name: Add target to folder name
run: '[[ -d "packages/next-swc/crates/wasm/pkg" ]] && mv packages/next-swc/crates/wasm/pkg packages/next-swc/crates/wasm/pkg-${{ matrix.target }} || ls packages/next-swc/crates/wasm'
@ -1513,7 +1541,6 @@ jobs:
id: turbo-cache
uses: actions/cache@v3
if: ${{needs.build.outputs.docsChange == 'nope'}}
with:
path: .turbo
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ steps.get-week.outputs.WEEK }}-${{ github.sha }}
@ -1526,6 +1553,9 @@ jobs:
if: ${{needs.build.outputs.docsChange == 'nope'}}
run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh
- name: normalize versions
run: node scripts/normalize-version-bump.js
- name: Build
if: ${{needs.build.outputs.docsChange == 'nope'}}
run: turbo run build-wasm --cache-dir=".turbo" -- --target nodejs --dev

3
packages/next-swc/package.json
View file

@ -5,7 +5,8 @@
"scripts": {
"build-native": "napi build --platform -p next-swc-napi --cargo-name next_swc_napi --features plugin --js false native",
"build-native-no-plugin": "napi build --platform -p next-swc-napi --cargo-name next_swc_napi --js false native",
"build-wasm": "wasm-pack build crates/wasm --scope=next"
"build-wasm": "wasm-pack build crates/wasm --scope=next",
"cache-build-native": "echo $(ls native)"
},
"napi": {
"name": "next-swc",

66
scripts/normalize-version-bump.js Executable file
View file

@ -0,0 +1,66 @@
#!/usr/bin/env node
// @ts-check
/*
This prevents busting the turbo cache un-necessarily due
to bumping the version in the repo's package.json files
*/
const path = require('path')
const fs = require('fs/promises')
const cwd = process.cwd()
const NORMALIZED_VERSION = '0.0.0'
const readJson = async (filePath) =>
JSON.parse(await fs.readFile(filePath, 'utf8'))
const writeJson = async (filePath, data) =>
fs.writeFile(filePath, JSON.stringify(data, null, 2) + '\n')
;(async function () {
const packages = await fs.readdir(path.join(cwd, 'packages'))
const pkgJsonData = new Map()
const pkgNames = []
await Promise.all(
packages.map(async (pkgDir) => {
const data = await readJson(
path.join(cwd, 'packages', pkgDir, 'package.json')
)
pkgNames.push(data.name)
pkgJsonData.set(pkgDir, data)
})
)
const normalizeVersions = async (filePath, data) => {
data = data || (await readJson(filePath))
const version = data.version
if (version) {
data.version = NORMALIZED_VERSION
const normalizeEntry = (type, key) => {
const pkgVersion = data[type][key]
if (pkgNames.includes(key) && pkgVersion === version) {
data[type][key] = NORMALIZED_VERSION
}
}
for (const key of Object.keys(data.dependencies || {})) {
normalizeEntry('dependencies', key)
}
for (const key of Object.keys(data.devDependencies || {})) {
normalizeEntry('devDependencies', key)
}
await writeJson(filePath, data)
}
}
await Promise.all(
packages.map((pkgDir) =>
normalizeVersions(
path.join(cwd, 'packages', pkgDir, 'package.json'),
pkgJsonData.get(pkgDir)
)
)
)
await normalizeVersions(path.join(cwd, 'lerna.json'))
await fs.unlink(path.join(cwd, 'pnpm-lock.yaml'))
await fs.writeFile(path.join(cwd, 'pnpm-lock.yaml'), '')
})()

30
scripts/pull-freebsd-cache.js Executable file
View file

@ -0,0 +1,30 @@
#!/usr/bin/env node
// @ts-check
const { execSync } = require('child_process')
;(async function () {
const turboResult = execSync(
`pnpm turbo run cache-build-native --dry=json -- --platform --release --target x86_64-unknown-freebsd`
).toString()
const turboData = JSON.parse(turboResult)
const task = turboData.tasks.find((t) => t.command !== '<NONEXISTENT>')
if (!task) {
console.warn(`Failed to find related turbo task`, turboResult)
return
}
// pull cache if it was available
if (task.cacheState.local || task.cacheState.remote) {
const pullResult = execSync(
`pnpm turbo run cache-build-native -- --platform --release --target x86_64-unknown-freebsd`
).toString()
console.log(pullResult)
} else {
console.warn(`No turbo cache was available, continuing...`)
console.warn(task)
}
})()

4
turbo.json
View file

@ -13,6 +13,10 @@
"dependsOn": ["^build-wasm"],
"outputs": ["crates/wasm/pkg/*"]
},
"cache-build-native": {
"dependsOn": ["^cache-build-native"],
"outputs": ["native/*.node"]
},
"build": {
"dependsOn": ["^build"],
"outputs": ["dist/**"]