Persist package tarballs as GitHub Actions artifacts (#66445)
This commit is contained in:
Sebastian Silbermann
GitHub
parent
a22858d6e7
commit
f1488d5d68
4 changed files with 273 additions and 179 deletions
95
.github/workflows/build_and_deploy.yml
vendored
95
.github/workflows/build_and_deploy.yml
vendored
|
|
@ -1,8 +1,10 @@
|
|||
# Update all mentions of this name in vercel-packages when changing.
|
||||
name: build-and-deploy
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ['canary']
|
||||
# TODO: Run only on canary pushes but PR syncs.
|
||||
# Requires checking if CI is approved
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
|
|
@ -14,6 +16,41 @@ env:
|
|||
TURBO_REMOTE_ONLY: 'true'
|
||||
|
||||
jobs:
|
||||
deploy-target:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
value: ${{ steps.deploy-target.outputs.value }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 1
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: ${{ env.NODE_LTS_VERSION }}
|
||||
check-latest: true
|
||||
- run: corepack enable
|
||||
- name: Determine deploy target
|
||||
# 'force-preview' performs a full preview build but only if acknowledged i.e. workflow_dispatch
|
||||
# 'automated-preview' for pushes on branches other than 'canary' for integration testing.
|
||||
# 'staging' for canary branch since that will eventually be published i.e. become the production build.
|
||||
id: deploy-target
|
||||
run: |
|
||||
if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]];
|
||||
then
|
||||
echo "value=production" >> $GITHUB_OUTPUT
|
||||
elif [ '${{ github.ref }}' == 'refs/heads/canary' ]
|
||||
then
|
||||
echo "value=staging" >> $GITHUB_OUTPUT
|
||||
elif [ '${{ github.event_name }}' == 'workflow_dispatch' ]
|
||||
then
|
||||
echo "value=force-preview" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "value=automated-preview" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
- name: Print deploy target
|
||||
run: echo "Deploy target is '${{ steps.deploy-target.outputs.value }}'"
|
||||
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
|
|
@ -21,8 +58,6 @@ jobs:
|
|||
# we build a dev binary for use in CI so skip downloading
|
||||
# canary next-swc binaries in the monorepo
|
||||
NEXT_SKIP_NATIVE_POSTINSTALL: 1
|
||||
outputs:
|
||||
isRelease: ${{ steps.check-release.outputs.IS_RELEASE }}
|
||||
steps:
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v4
|
||||
|
|
@ -52,15 +87,6 @@ jobs:
|
|||
|
||||
- run: pnpm run build
|
||||
|
||||
- id: check-release
|
||||
run: |
|
||||
if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]];
|
||||
then
|
||||
echo "IS_RELEASE=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "IS_RELEASE=false" >> $GITHUB_OUTPUT
|
||||
fi
|
||||
|
||||
- uses: actions/cache@v4
|
||||
timeout-minutes: 5
|
||||
id: cache-build
|
||||
|
|
@ -70,6 +96,8 @@ jobs:
|
|||
|
||||
# Build binaries for publishing
|
||||
build-native:
|
||||
needs:
|
||||
- deploy-target
|
||||
defaults:
|
||||
run:
|
||||
shell: bash -leo pipefail {0}
|
||||
|
|
@ -77,6 +105,17 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
exclude:
|
||||
# Exclude slow builds for automated previews
|
||||
# These are rarely needed for the standard preview usage (e.g. Front sync)
|
||||
- settings:
|
||||
target: ${{ needs.deploy-target.outputs.value == 'automated-preview' && 'i686-pc-windows-msvc' }}
|
||||
- settings:
|
||||
target: ${{ needs.deploy-target.outputs.value == 'automated-preview' && 'x86_64-pc-windows-msvc' }}
|
||||
- settings:
|
||||
target: ${{ needs.deploy-target.outputs.value == 'automated-preview' && 'aarch64-unknown-linux-musl' }}
|
||||
- settings:
|
||||
target: ${{ needs.deploy-target.outputs.value == 'automated-preview' && 'x86_64-unknown-linux-musl' }}
|
||||
settings:
|
||||
- host:
|
||||
- 'self-hosted'
|
||||
|
|
@ -405,16 +444,14 @@ jobs:
|
|||
path: packages/next-swc/crates/wasm/pkg-*
|
||||
|
||||
deployTarball:
|
||||
if: ${{ needs.build.outputs.isRelease != 'true' }}
|
||||
name: Deploy tarball
|
||||
if: ${{ needs.deploy-target.outputs.value != 'production' }}
|
||||
name: Deploy preview tarball
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- deploy-target
|
||||
- build
|
||||
- build-wasm
|
||||
- build-native
|
||||
env:
|
||||
VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }}
|
||||
VERCEL_TEST_TEAM: vtest314-next-e2e-tests
|
||||
steps:
|
||||
- name: Setup node
|
||||
uses: actions/setup-node@v4
|
||||
|
|
@ -451,15 +488,22 @@ jobs:
|
|||
merge-multiple: true
|
||||
path: packages/next-swc/crates/wasm
|
||||
|
||||
- run: npm i -g vercel@latest
|
||||
- name: Create tarballs
|
||||
run: node scripts/create-preview-tarballs.js "${{ github.sha }}" "${{ runner.temp }}/preview-tarballs"
|
||||
|
||||
- run: node ./scripts/deploy-tarball.js
|
||||
- name: Upload tarballs
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
# Update all mentions of this name in vercel-packages when changing.
|
||||
name: preview-tarballs
|
||||
path: ${{ runner.temp }}/preview-tarballs/*
|
||||
|
||||
publishRelease:
|
||||
if: ${{ needs.build.outputs.isRelease == 'true' }}
|
||||
if: ${{ needs.deploy-target.outputs.value == 'production' }}
|
||||
name: Potentially publish release
|
||||
runs-on: ubuntu-latest
|
||||
needs:
|
||||
- deploy-target
|
||||
- build
|
||||
- build-wasm
|
||||
- build-native
|
||||
|
|
@ -519,23 +563,25 @@ jobs:
|
|||
path: /home/runner/.npm/_logs/*
|
||||
|
||||
deployExamples:
|
||||
if: ${{ needs.deploy-target.outputs.value != 'automated-preview' }}
|
||||
name: Deploy examples
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
needs: [build, deploy-target]
|
||||
steps:
|
||||
- run: echo '${{ needs.deploy-target.outputs.value }}'
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 25
|
||||
- name: Install Vercel CLI
|
||||
run: npm i -g vercel@latest
|
||||
- name: Deploy preview examples
|
||||
if: ${{ needs.build.outputs.isRelease != 'true' }}
|
||||
if: ${{ needs.deploy-target.outputs.value != 'production' }}
|
||||
run: ./scripts/deploy-examples.sh
|
||||
env:
|
||||
VERCEL_API_TOKEN: ${{ secrets.VERCEL_API_TOKEN }}
|
||||
DEPLOY_ENVIRONMENT: preview
|
||||
- name: Deploy production examples
|
||||
if: ${{ needs.build.outputs.isRelease == 'true' }}
|
||||
if: ${{ needs.deploy-target.outputs.value == 'production' }}
|
||||
run: ./scripts/deploy-examples.sh
|
||||
env:
|
||||
VERCEL_API_TOKEN: ${{ secrets.VERCEL_API_TOKEN }}
|
||||
|
|
@ -570,9 +616,10 @@ jobs:
|
|||
NEXT_SKIP_NATIVE_POSTINSTALL: 1
|
||||
|
||||
upload_turbopack_bytesize:
|
||||
if: ${{ needs.deploy-target.outputs.value != 'automated-preview'}}
|
||||
name: Upload Turbopack Bytesize metrics to Datadog
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build-native]
|
||||
needs: [build-native, deploy-target]
|
||||
env:
|
||||
DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }}
|
||||
steps:
|
||||
|
|
|
|||
|
|
@ -125,3 +125,23 @@ To run the test suite using Turbopack, you can use the `TURBOPACK=1` environment
|
|||
```sh
|
||||
TURBOPACK=1 pnpm test-dev test/e2e/app-dir/app/
|
||||
```
|
||||
|
||||
## Integration testing outside the repository with preview builds
|
||||
|
||||
Every branch build will create a tarball for each package in this repository<sup>1</sup> that can be used in external repositories.
|
||||
|
||||
You can use this preview build in other packages by using a https://vercel-packages.vercel.app URL instead of a version in the `package.json` e.g.
|
||||
|
||||
```json
|
||||
{
|
||||
"dependencies": {
|
||||
"next": "https://vercel-packages.vercel.app/next/commits/abcd/next"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
You can refer to builds only by commit SHAs at the moment.
|
||||
|
||||
<sup>1</sup> Not all native packages are built automatically.
|
||||
`build-and-deploy` excludes slow, rarely used native variants of `next-swc`.
|
||||
To force a build of all packages, you can trigger `build-and-deploy` manually (i.e. `workflow_dispatch`).
|
||||
|
|
|
|||
182
scripts/create-preview-tarballs.js
Normal file
182
scripts/create-preview-tarballs.js
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
// @ts-check
|
||||
const execa = require('execa')
|
||||
const fs = require('node:fs/promises')
|
||||
const os = require('node:os')
|
||||
const path = require('node:path')
|
||||
|
||||
async function main() {
|
||||
const [
|
||||
commitSha,
|
||||
tarballDirectory = path.join(os.tmpdir(), 'vercel-nextjs-preview-tarballs'),
|
||||
] = process.argv.slice(2)
|
||||
const repoRoot = path.resolve(__dirname, '..')
|
||||
|
||||
await fs.mkdir(tarballDirectory, { recursive: true })
|
||||
|
||||
const [{ stdout: shortSha }, { stdout: dateString }] = await Promise.all([
|
||||
execa('git', ['rev-parse', '--short', commitSha]),
|
||||
// Source: https://github.com/facebook/react/blob/767f52237cf7892ad07726f21e3e8bacfc8af839/scripts/release/utils.js#L114
|
||||
execa(`git`, [
|
||||
'show',
|
||||
'-s',
|
||||
'--no-show-signature',
|
||||
'--format=%cd',
|
||||
'--date=format:%Y%m%d',
|
||||
commitSha,
|
||||
]),
|
||||
])
|
||||
|
||||
const lernaConfig = JSON.parse(
|
||||
await fs.readFile(path.join(repoRoot, 'lerna.json'), 'utf8')
|
||||
)
|
||||
|
||||
// 15.0.0-canary.17 -> 15.0.0
|
||||
// 15.0.0 -> 15.0.0
|
||||
const [semverStableVersion] = lernaConfig.version.split('-')
|
||||
const version = `${semverStableVersion}-preview-${shortSha}-${dateString}`
|
||||
console.info(`Designated version: ${version}`)
|
||||
|
||||
const nativePackagesDir = path.join(
|
||||
repoRoot,
|
||||
'packages/next-swc/crates/napi/npm'
|
||||
)
|
||||
const platforms = (await fs.readdir(nativePackagesDir)).filter(
|
||||
(name) => !name.startsWith('.')
|
||||
)
|
||||
|
||||
console.info(`Creating tarballs for next-swc packages`)
|
||||
const nextSwcPackageNames = new Set()
|
||||
await Promise.all(
|
||||
platforms.map(async (platform) => {
|
||||
const binaryName = `next-swc.${platform}.node`
|
||||
try {
|
||||
await fs.cp(
|
||||
path.join(repoRoot, 'packages/next-swc/native', binaryName),
|
||||
path.join(nativePackagesDir, platform, binaryName)
|
||||
)
|
||||
} catch (error) {
|
||||
if (error.code === 'ENOENT') {
|
||||
console.warn(
|
||||
`Skipping next-swc platform '${platform}' tarball creation because ${binaryName} was never built.`
|
||||
)
|
||||
return
|
||||
}
|
||||
throw error
|
||||
}
|
||||
const manifest = JSON.parse(
|
||||
await fs.readFile(
|
||||
path.join(nativePackagesDir, platform, 'package.json'),
|
||||
'utf8'
|
||||
)
|
||||
)
|
||||
manifest.version = version
|
||||
await fs.writeFile(
|
||||
path.join(nativePackagesDir, platform, 'package.json'),
|
||||
JSON.stringify(manifest, null, 2) + '\n'
|
||||
)
|
||||
// By encoding the package name in the directory, vercel-packages can later extract the package name of a tarball from its path when `tarballDirectory` is zipped.
|
||||
const packDestination = path.join(tarballDirectory, manifest.name)
|
||||
await fs.mkdir(packDestination, { recursive: true })
|
||||
const { stdout } = await execa(
|
||||
'npm',
|
||||
['pack', '--pack-destination', packDestination],
|
||||
{
|
||||
cwd: path.join(nativePackagesDir, platform),
|
||||
}
|
||||
)
|
||||
// tarball name is printed as the last line of npm-pack
|
||||
const tarballName = stdout.trim().split('\n').pop()
|
||||
console.info(`Created tarball ${path.join(packDestination, tarballName)}`)
|
||||
|
||||
nextSwcPackageNames.add(manifest.name)
|
||||
})
|
||||
)
|
||||
|
||||
const lernaListJson = await execa('pnpm', [
|
||||
'--silent',
|
||||
'lerna',
|
||||
'list',
|
||||
'--json',
|
||||
])
|
||||
const packages = JSON.parse(lernaListJson.stdout)
|
||||
const packagesByVersion = new Map()
|
||||
for (const packageInfo of packages) {
|
||||
packagesByVersion.set(
|
||||
packageInfo.name,
|
||||
`https://vercel-packages.vercel.app/next/commits/${commitSha}/${packageInfo.name}`
|
||||
)
|
||||
}
|
||||
for (const nextSwcPackageName of nextSwcPackageNames) {
|
||||
packagesByVersion.set(
|
||||
nextSwcPackageName,
|
||||
`https://vercel-packages.vercel.app/next/commits/${commitSha}/${nextSwcPackageName}`
|
||||
)
|
||||
}
|
||||
|
||||
console.info(`Creating tarballs for regular packages`)
|
||||
for (const packageInfo of packages) {
|
||||
if (packageInfo.private) {
|
||||
continue
|
||||
}
|
||||
|
||||
const packageJsonPath = path.join(packageInfo.location, 'package.json')
|
||||
const packageJson = await fs.readFile(packageJsonPath, 'utf8')
|
||||
const manifest = JSON.parse(packageJson)
|
||||
|
||||
manifest.version = version
|
||||
|
||||
if (packageInfo.name === 'next') {
|
||||
manifest.optionalDependencies ??= {}
|
||||
for (const nextSwcPackageName of nextSwcPackageNames) {
|
||||
manifest.optionalDependencies[nextSwcPackageName] =
|
||||
packagesByVersion.get(nextSwcPackageName)
|
||||
}
|
||||
}
|
||||
|
||||
// ensure it depends on packages from this release.
|
||||
for (const [dependencyName, version] of packagesByVersion) {
|
||||
if (manifest.dependencies?.[dependencyName] !== undefined) {
|
||||
manifest.dependencies[dependencyName] = version
|
||||
}
|
||||
if (manifest.devDependencies?.[dependencyName] !== undefined) {
|
||||
manifest.devDependencies[dependencyName] = version
|
||||
}
|
||||
if (manifest.peerDependencies?.[dependencyName] !== undefined) {
|
||||
manifest.peerDependencies[dependencyName] = version
|
||||
}
|
||||
if (manifest.optionalDependencies?.[dependencyName] !== undefined) {
|
||||
manifest.optionalDependencies[dependencyName] = version
|
||||
}
|
||||
}
|
||||
|
||||
await fs.writeFile(
|
||||
packageJsonPath,
|
||||
JSON.stringify(manifest, null, 2) +
|
||||
// newline will be added by Prettier
|
||||
'\n'
|
||||
)
|
||||
|
||||
// By encoding the package name in the directory, vercel-packages can later extract the package name of a tarball from its path when `tarballDirectory` is zipped.
|
||||
const packDestination = path.join(tarballDirectory, manifest.name)
|
||||
await fs.mkdir(packDestination, { recursive: true })
|
||||
const { stdout } = await execa(
|
||||
'npm',
|
||||
['pack', '--pack-destination', packDestination],
|
||||
{
|
||||
cwd: packageInfo.location,
|
||||
}
|
||||
)
|
||||
// tarball name is printed as the last line of npm-pack
|
||||
const tarballName = stdout.trim().split('\n').pop()
|
||||
console.info(`Created tarball ${path.join(packDestination, tarballName)}`)
|
||||
}
|
||||
|
||||
console.info(
|
||||
`When this job is completed, a Next.js preview build will be available under ${packagesByVersion.get('next')}`
|
||||
)
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
|
|
@ -1,155 +0,0 @@
|
|||
// @ts-check
|
||||
const path = require('path')
|
||||
const execa = require('execa')
|
||||
const fs = require('fs/promises')
|
||||
|
||||
const cwd = process.cwd()
|
||||
|
||||
async function main() {
|
||||
const deployDir = path.join(cwd, 'files')
|
||||
const publicDir = path.join(deployDir, 'public')
|
||||
await fs.mkdir(publicDir, { recursive: true })
|
||||
await fs.writeFile(
|
||||
path.join(deployDir, 'package.json'),
|
||||
JSON.stringify({
|
||||
name: 'files',
|
||||
dependencies: {},
|
||||
scripts: {
|
||||
build: 'node inject-deploy-url.js',
|
||||
},
|
||||
})
|
||||
)
|
||||
await fs.copyFile(
|
||||
path.join(cwd, 'scripts/inject-deploy-url.js'),
|
||||
path.join(deployDir, 'inject-deploy-url.js')
|
||||
)
|
||||
|
||||
let nativePackagesDir = path.join(cwd, 'packages/next-swc/crates/napi/npm')
|
||||
let platforms = (await fs.readdir(nativePackagesDir)).filter(
|
||||
(name) => !name.startsWith('.')
|
||||
)
|
||||
|
||||
const optionalDeps = {}
|
||||
const { version } = JSON.parse(
|
||||
await fs.readFile(path.join(cwd, 'lerna.json'), 'utf8')
|
||||
)
|
||||
|
||||
await Promise.all(
|
||||
platforms.map(async (platform) => {
|
||||
let binaryName = `next-swc.${platform}.node`
|
||||
await fs.cp(
|
||||
path.join(cwd, 'packages/next-swc/native', binaryName),
|
||||
path.join(nativePackagesDir, platform, binaryName)
|
||||
)
|
||||
let pkg = JSON.parse(
|
||||
await fs.readFile(
|
||||
path.join(nativePackagesDir, platform, 'package.json'),
|
||||
'utf8'
|
||||
)
|
||||
)
|
||||
pkg.version = version
|
||||
await fs.writeFile(
|
||||
path.join(nativePackagesDir, platform, 'package.json'),
|
||||
JSON.stringify(pkg, null, 2)
|
||||
)
|
||||
const { stdout } = await execa(`npm`, [
|
||||
`pack`,
|
||||
`${path.join(nativePackagesDir, platform)}`,
|
||||
])
|
||||
process.stdout.write(stdout)
|
||||
const tarballName = stdout.split('\n').pop()?.trim() || ''
|
||||
await fs.rename(
|
||||
path.join(cwd, tarballName),
|
||||
path.join(publicDir, tarballName)
|
||||
)
|
||||
optionalDeps[pkg.name] = `https://DEPLOY_URL/${tarballName}`
|
||||
})
|
||||
)
|
||||
|
||||
const nextPkgJsonPath = path.join(cwd, 'packages/next/package.json')
|
||||
const nextPkg = JSON.parse(await fs.readFile(nextPkgJsonPath, 'utf8'))
|
||||
|
||||
nextPkg.optionalDependencies = optionalDeps
|
||||
|
||||
await fs.writeFile(nextPkgJsonPath, JSON.stringify(nextPkg, null, 2))
|
||||
|
||||
const { stdout: nextPackStdout } = await execa(`npm`, [
|
||||
`pack`,
|
||||
`${path.join(cwd, 'packages/next')}`,
|
||||
])
|
||||
process.stdout.write(nextPackStdout)
|
||||
const nextTarballName = nextPackStdout.split('\n').pop()?.trim() || ''
|
||||
await fs.rename(
|
||||
path.join(cwd, nextTarballName),
|
||||
path.join(publicDir, nextTarballName)
|
||||
)
|
||||
|
||||
await fs.writeFile(
|
||||
path.join(deployDir, 'vercel.json'),
|
||||
JSON.stringify(
|
||||
{
|
||||
version: 2,
|
||||
rewrites: [
|
||||
{
|
||||
source: '/next.tgz',
|
||||
destination: `/${nextTarballName}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
)
|
||||
const vercelConfigDir = path.join(cwd, '.vercel')
|
||||
|
||||
if (process.env.VERCEL_TEST_TOKEN) {
|
||||
await fs.mkdir(vercelConfigDir)
|
||||
await fs.writeFile(
|
||||
path.join(vercelConfigDir, 'auth.json'),
|
||||
JSON.stringify({
|
||||
token: process.env.VERCEL_TEST_TOKEN,
|
||||
})
|
||||
)
|
||||
await fs.writeFile(
|
||||
path.join(vercelConfigDir, 'config.json'),
|
||||
JSON.stringify({})
|
||||
)
|
||||
console.log('wrote config to', vercelConfigDir)
|
||||
}
|
||||
|
||||
const child = execa(
|
||||
'vercel',
|
||||
[
|
||||
'--scope',
|
||||
process.env.VERCEL_TEST_TEAM || '',
|
||||
'--global-config',
|
||||
vercelConfigDir,
|
||||
'-y',
|
||||
],
|
||||
{
|
||||
cwd: deployDir,
|
||||
}
|
||||
)
|
||||
let deployOutput = ''
|
||||
const handleData = (type) => (chunk) => {
|
||||
process[type].write(chunk)
|
||||
|
||||
// only want stdout since that's where deployment URL
|
||||
// is sent to
|
||||
if (type === 'stdout') {
|
||||
deployOutput += chunk.toString()
|
||||
}
|
||||
}
|
||||
child.stdout?.on('data', handleData('stdout'))
|
||||
child.stderr?.on('data', handleData('stderr'))
|
||||
|
||||
await child
|
||||
|
||||
const deployUrl = deployOutput.trim()
|
||||
console.log(`\n\nNext.js tarball: ${deployUrl.trim()}/next.tgz`)
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
Loading…
Reference in a new issue