Implement granular rust caching (#54582)
This fixes our caching for the docker builds as they were missing inputs the other jobs had also enables caching the rust target cache which improves build times when only changing our package and the lockfile isn't invalidated. Tested here https://github.com/vercel/next.js/actions/runs/5987764387
This commit is contained in:
parent
05dbd9a8a2
commit
8997ab0812
4 changed files with 69 additions and 8 deletions
7
.github/workflows/build_and_deploy.yml
vendored
7
.github/workflows/build_and_deploy.yml
vendored
|
@ -268,6 +268,13 @@ jobs:
|
||||||
run: ${{ matrix.settings.setup }}
|
run: ${{ matrix.settings.setup }}
|
||||||
if: ${{ matrix.settings.setup }}
|
if: ${{ matrix.settings.setup }}
|
||||||
|
|
||||||
|
- name: Cache on ${{ github.ref_name }}
|
||||||
|
uses: ijjk/rust-cache@turbo-cache-v1.0.7
|
||||||
|
with:
|
||||||
|
shared-key: build-${{ matrix.settings.target }}
|
||||||
|
save-if: 'true'
|
||||||
|
cache-provider: 'turbo'
|
||||||
|
|
||||||
# we only need custom caching for docker builds
|
# we only need custom caching for docker builds
|
||||||
# as they are on an older Node.js version and have
|
# as they are on an older Node.js version and have
|
||||||
# issues with turbo caching
|
# issues with turbo caching
|
||||||
|
|
12
.github/workflows/build_reusable.yml
vendored
12
.github/workflows/build_reusable.yml
vendored
|
@ -49,6 +49,10 @@ on:
|
||||||
required: false
|
required: false
|
||||||
description: 'if test trace needs uploading'
|
description: 'if test trace needs uploading'
|
||||||
type: string
|
type: string
|
||||||
|
rustCacheKey:
|
||||||
|
required: false
|
||||||
|
description: 'rustCacheKey to cache shared target assets'
|
||||||
|
type: string
|
||||||
|
|
||||||
env:
|
env:
|
||||||
NAPI_CLI_VERSION: 2.14.7
|
NAPI_CLI_VERSION: 2.14.7
|
||||||
|
@ -114,6 +118,14 @@ jobs:
|
||||||
|
|
||||||
- run: corepack prepare --activate yarn@1.22.19 && npm i -g "turbo@${TURBO_VERSION}" "@napi-rs/cli@${NAPI_CLI_VERSION}"
|
- run: corepack prepare --activate yarn@1.22.19 && npm i -g "turbo@${TURBO_VERSION}" "@napi-rs/cli@${NAPI_CLI_VERSION}"
|
||||||
|
|
||||||
|
- name: Cache on ${{ github.ref_name }}
|
||||||
|
uses: ijjk/rust-cache@turbo-cache-v1.0.7
|
||||||
|
if: ${{ inputs.rustCacheKey }}
|
||||||
|
with:
|
||||||
|
shared-key: ${{ inputs.rustCacheKey }}-x86_64-unknown-linux-gnu
|
||||||
|
save-if: ${{ github.ref_name == 'canary' }}
|
||||||
|
cache-provider: 'turbo'
|
||||||
|
|
||||||
# clean up any previous artifacts to avoid hitting disk space limits
|
# clean up any previous artifacts to avoid hitting disk space limits
|
||||||
- run: git clean -xdf && rm -rf /tmp/next-repo-*; rm -rf /tmp/next-install-* /tmp/yarn-* /tmp/ncc-cache target
|
- run: git clean -xdf && rm -rf /tmp/next-repo-*; rm -rf /tmp/next-install-* /tmp/yarn-* /tmp/ncc-cache target
|
||||||
|
|
||||||
|
|
|
@ -1,14 +1,35 @@
|
||||||
#!/usr/bin/env node
|
#!/usr/bin/env node
|
||||||
// @ts-check
|
// @ts-check
|
||||||
|
|
||||||
const { execSync } = require('child_process')
|
const { spawn } = require('child_process')
|
||||||
|
|
||||||
;(async function () {
|
;(async function () {
|
||||||
const target = process.argv[process.argv.length - 1]
|
const target = process.argv[process.argv.length - 1]
|
||||||
|
|
||||||
const turboResult = execSync(
|
let turboResult = ''
|
||||||
`pnpm turbo run cache-build-native --dry=json -- ${target}`
|
|
||||||
).toString()
|
await new Promise((resolve, reject) => {
|
||||||
|
const child = spawn(
|
||||||
|
'/bin/bash',
|
||||||
|
['-c', `pnpm turbo run cache-build-native --dry=json -- ${target}`],
|
||||||
|
{
|
||||||
|
stdio: 'pipe',
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
child.stdout.on('data', (data) => {
|
||||||
|
turboResult += data.toString()
|
||||||
|
})
|
||||||
|
|
||||||
|
child.on('exit', (code, signal) => {
|
||||||
|
if (code || signal) {
|
||||||
|
return reject(
|
||||||
|
new Error(`invalid exit code ${code} or signal ${signal}`)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
resolve(0)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
const turboData = JSON.parse(turboResult)
|
const turboData = JSON.parse(turboResult)
|
||||||
|
|
||||||
|
@ -21,10 +42,23 @@ const { execSync } = require('child_process')
|
||||||
|
|
||||||
// pull cache if it was available
|
// pull cache if it was available
|
||||||
if (task.cache.local || task.cache.remote) {
|
if (task.cache.local || task.cache.remote) {
|
||||||
const pullResult = execSync(
|
await new Promise((resolve, reject) => {
|
||||||
`pnpm turbo run cache-build-native -- ${target}`
|
const child = spawn(
|
||||||
).toString()
|
'/bin/bash',
|
||||||
console.log(pullResult)
|
['-c', `pnpm turbo run cache-build-native -- ${target}`],
|
||||||
|
{
|
||||||
|
stdio: 'inherit',
|
||||||
|
}
|
||||||
|
)
|
||||||
|
child.on('exit', (code, signal) => {
|
||||||
|
if (code || signal) {
|
||||||
|
return reject(
|
||||||
|
new Error(`invalid exit code ${code} or signal ${signal}`)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
resolve(0)
|
||||||
|
})
|
||||||
|
})
|
||||||
} else {
|
} else {
|
||||||
console.warn(`No turbo cache was available, continuing...`)
|
console.warn(`No turbo cache was available, continuing...`)
|
||||||
console.warn(task)
|
console.warn(task)
|
||||||
|
|
|
@ -74,6 +74,14 @@
|
||||||
"outputs": ["crates/wasm/pkg/*"]
|
"outputs": ["crates/wasm/pkg/*"]
|
||||||
},
|
},
|
||||||
"cache-build-native": {
|
"cache-build-native": {
|
||||||
|
"inputs": [
|
||||||
|
"../../.cargo/**",
|
||||||
|
"../../packages/next-swc/crates/**",
|
||||||
|
"../../**/Cargo.toml",
|
||||||
|
"../../**/Cargo.lock",
|
||||||
|
"../../.github/workflows/build_and_deploy.yml",
|
||||||
|
"../../rust-toolchain"
|
||||||
|
],
|
||||||
"dependsOn": ["^cache-build-native"],
|
"dependsOn": ["^cache-build-native"],
|
||||||
"outputs": ["native/*.node"]
|
"outputs": ["native/*.node"]
|
||||||
},
|
},
|
||||||
|
|
Loading…
Reference in a new issue