From f1488d5d68d71f2cda71ff93aaa11dea4d62accd Mon Sep 17 00:00:00 2001 From: Sebastian Silbermann Date: Tue, 2 Jul 2024 17:18:45 +0200 Subject: [PATCH] Persist package tarballs as GitHub Actions artifacts (#66445) --- .github/workflows/build_and_deploy.yml | 95 +++++++++---- contributing/core/testing.md | 20 +++ scripts/create-preview-tarballs.js | 182 +++++++++++++++++++++++++ scripts/deploy-tarball.js | 155 --------------------- 4 files changed, 273 insertions(+), 179 deletions(-) create mode 100644 scripts/create-preview-tarballs.js delete mode 100644 scripts/deploy-tarball.js diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml index 008229f182..978d7e5a30 100644 --- a/.github/workflows/build_and_deploy.yml +++ b/.github/workflows/build_and_deploy.yml @@ -1,8 +1,10 @@ +# Update all mentions of this name in vercel-packages when changing. name: build-and-deploy on: push: - branches: ['canary'] + # TODO: Run only on canary pushes but PR syncs. + # Requires checking if CI is approved workflow_dispatch: env: @@ -14,6 +16,41 @@ env: TURBO_REMOTE_ONLY: 'true' jobs: + deploy-target: + runs-on: ubuntu-latest + outputs: + value: ${{ steps.deploy-target.outputs.value }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 1 + - name: Setup node + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_LTS_VERSION }} + check-latest: true + - run: corepack enable + - name: Determine deploy target + # 'force-preview' performs a full preview build but only if acknowledged i.e. workflow_dispatch + # 'automated-preview' for pushes on branches other than 'canary' for integration testing. + # 'staging' for canary branch since that will eventually be published i.e. become the production build. + id: deploy-target + run: | + if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]]; + then + echo "value=production" >> $GITHUB_OUTPUT + elif [ '${{ github.ref }}' == 'refs/heads/canary' ] + then + echo "value=staging" >> $GITHUB_OUTPUT + elif [ '${{ github.event_name }}' == 'workflow_dispatch' ] + then + echo "value=force-preview" >> $GITHUB_OUTPUT + else + echo "value=automated-preview" >> $GITHUB_OUTPUT + fi + - name: Print deploy target + run: echo "Deploy target is '${{ steps.deploy-target.outputs.value }}'" + build: runs-on: ubuntu-latest env: @@ -21,8 +58,6 @@ jobs: # we build a dev binary for use in CI so skip downloading # canary next-swc binaries in the monorepo NEXT_SKIP_NATIVE_POSTINSTALL: 1 - outputs: - isRelease: ${{ steps.check-release.outputs.IS_RELEASE }} steps: - name: Setup node uses: actions/setup-node@v4 @@ -52,15 +87,6 @@ jobs: - run: pnpm run build - - id: check-release - run: | - if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]]; - then - echo "IS_RELEASE=true" >> $GITHUB_OUTPUT - else - echo "IS_RELEASE=false" >> $GITHUB_OUTPUT - fi - - uses: actions/cache@v4 timeout-minutes: 5 id: cache-build @@ -70,6 +96,8 @@ jobs: # Build binaries for publishing build-native: + needs: + - deploy-target defaults: run: shell: bash -leo pipefail {0} @@ -77,6 +105,17 @@ jobs: strategy: fail-fast: false matrix: + exclude: + # Exclude slow builds for automated previews + # These are rarely needed for the standard preview usage (e.g. Front sync) + - settings: + target: ${{ needs.deploy-target.outputs.value == 'automated-preview' && 'i686-pc-windows-msvc' }} + - settings: + target: ${{ needs.deploy-target.outputs.value == 'automated-preview' && 'x86_64-pc-windows-msvc' }} + - settings: + target: ${{ needs.deploy-target.outputs.value == 'automated-preview' && 'aarch64-unknown-linux-musl' }} + - settings: + target: ${{ needs.deploy-target.outputs.value == 'automated-preview' && 'x86_64-unknown-linux-musl' }} settings: - host: - 'self-hosted' @@ -405,16 +444,14 @@ jobs: path: packages/next-swc/crates/wasm/pkg-* deployTarball: - if: ${{ needs.build.outputs.isRelease != 'true' }} - name: Deploy tarball + if: ${{ needs.deploy-target.outputs.value != 'production' }} + name: Deploy preview tarball runs-on: ubuntu-latest needs: + - deploy-target - build - build-wasm - build-native - env: - VERCEL_TEST_TOKEN: ${{ secrets.VERCEL_TEST_TOKEN }} - VERCEL_TEST_TEAM: vtest314-next-e2e-tests steps: - name: Setup node uses: actions/setup-node@v4 @@ -451,15 +488,22 @@ jobs: merge-multiple: true path: packages/next-swc/crates/wasm - - run: npm i -g vercel@latest + - name: Create tarballs + run: node scripts/create-preview-tarballs.js "${{ github.sha }}" "${{ runner.temp }}/preview-tarballs" - - run: node ./scripts/deploy-tarball.js + - name: Upload tarballs + uses: actions/upload-artifact@v4 + with: + # Update all mentions of this name in vercel-packages when changing. + name: preview-tarballs + path: ${{ runner.temp }}/preview-tarballs/* publishRelease: - if: ${{ needs.build.outputs.isRelease == 'true' }} + if: ${{ needs.deploy-target.outputs.value == 'production' }} name: Potentially publish release runs-on: ubuntu-latest needs: + - deploy-target - build - build-wasm - build-native @@ -519,23 +563,25 @@ jobs: path: /home/runner/.npm/_logs/* deployExamples: + if: ${{ needs.deploy-target.outputs.value != 'automated-preview' }} name: Deploy examples runs-on: ubuntu-latest - needs: [build] + needs: [build, deploy-target] steps: + - run: echo '${{ needs.deploy-target.outputs.value }}' - uses: actions/checkout@v4 with: fetch-depth: 25 - name: Install Vercel CLI run: npm i -g vercel@latest - name: Deploy preview examples - if: ${{ needs.build.outputs.isRelease != 'true' }} + if: ${{ needs.deploy-target.outputs.value != 'production' }} run: ./scripts/deploy-examples.sh env: VERCEL_API_TOKEN: ${{ secrets.VERCEL_API_TOKEN }} DEPLOY_ENVIRONMENT: preview - name: Deploy production examples - if: ${{ needs.build.outputs.isRelease == 'true' }} + if: ${{ needs.deploy-target.outputs.value == 'production' }} run: ./scripts/deploy-examples.sh env: VERCEL_API_TOKEN: ${{ secrets.VERCEL_API_TOKEN }} @@ -570,9 +616,10 @@ jobs: NEXT_SKIP_NATIVE_POSTINSTALL: 1 upload_turbopack_bytesize: + if: ${{ needs.deploy-target.outputs.value != 'automated-preview'}} name: Upload Turbopack Bytesize metrics to Datadog runs-on: ubuntu-latest - needs: [build-native] + needs: [build-native, deploy-target] env: DATADOG_API_KEY: ${{ secrets.DATA_DOG_API_KEY }} steps: diff --git a/contributing/core/testing.md b/contributing/core/testing.md index be9e1264a3..0b59624830 100644 --- a/contributing/core/testing.md +++ b/contributing/core/testing.md @@ -125,3 +125,23 @@ To run the test suite using Turbopack, you can use the `TURBOPACK=1` environment ```sh TURBOPACK=1 pnpm test-dev test/e2e/app-dir/app/ ``` + +## Integration testing outside the repository with preview builds + +Every branch build will create a tarball for each package in this repository1 that can be used in external repositories. + +You can use this preview build in other packages by using a https://vercel-packages.vercel.app URL instead of a version in the `package.json` e.g. + +```json +{ + "dependencies": { + "next": "https://vercel-packages.vercel.app/next/commits/abcd/next" + } +} +``` + +You can refer to builds only by commit SHAs at the moment. + +1 Not all native packages are built automatically. +`build-and-deploy` excludes slow, rarely used native variants of `next-swc`. +To force a build of all packages, you can trigger `build-and-deploy` manually (i.e. `workflow_dispatch`). diff --git a/scripts/create-preview-tarballs.js b/scripts/create-preview-tarballs.js new file mode 100644 index 0000000000..10d341c59e --- /dev/null +++ b/scripts/create-preview-tarballs.js @@ -0,0 +1,182 @@ +// @ts-check +const execa = require('execa') +const fs = require('node:fs/promises') +const os = require('node:os') +const path = require('node:path') + +async function main() { + const [ + commitSha, + tarballDirectory = path.join(os.tmpdir(), 'vercel-nextjs-preview-tarballs'), + ] = process.argv.slice(2) + const repoRoot = path.resolve(__dirname, '..') + + await fs.mkdir(tarballDirectory, { recursive: true }) + + const [{ stdout: shortSha }, { stdout: dateString }] = await Promise.all([ + execa('git', ['rev-parse', '--short', commitSha]), + // Source: https://github.com/facebook/react/blob/767f52237cf7892ad07726f21e3e8bacfc8af839/scripts/release/utils.js#L114 + execa(`git`, [ + 'show', + '-s', + '--no-show-signature', + '--format=%cd', + '--date=format:%Y%m%d', + commitSha, + ]), + ]) + + const lernaConfig = JSON.parse( + await fs.readFile(path.join(repoRoot, 'lerna.json'), 'utf8') + ) + + // 15.0.0-canary.17 -> 15.0.0 + // 15.0.0 -> 15.0.0 + const [semverStableVersion] = lernaConfig.version.split('-') + const version = `${semverStableVersion}-preview-${shortSha}-${dateString}` + console.info(`Designated version: ${version}`) + + const nativePackagesDir = path.join( + repoRoot, + 'packages/next-swc/crates/napi/npm' + ) + const platforms = (await fs.readdir(nativePackagesDir)).filter( + (name) => !name.startsWith('.') + ) + + console.info(`Creating tarballs for next-swc packages`) + const nextSwcPackageNames = new Set() + await Promise.all( + platforms.map(async (platform) => { + const binaryName = `next-swc.${platform}.node` + try { + await fs.cp( + path.join(repoRoot, 'packages/next-swc/native', binaryName), + path.join(nativePackagesDir, platform, binaryName) + ) + } catch (error) { + if (error.code === 'ENOENT') { + console.warn( + `Skipping next-swc platform '${platform}' tarball creation because ${binaryName} was never built.` + ) + return + } + throw error + } + const manifest = JSON.parse( + await fs.readFile( + path.join(nativePackagesDir, platform, 'package.json'), + 'utf8' + ) + ) + manifest.version = version + await fs.writeFile( + path.join(nativePackagesDir, platform, 'package.json'), + JSON.stringify(manifest, null, 2) + '\n' + ) + // By encoding the package name in the directory, vercel-packages can later extract the package name of a tarball from its path when `tarballDirectory` is zipped. + const packDestination = path.join(tarballDirectory, manifest.name) + await fs.mkdir(packDestination, { recursive: true }) + const { stdout } = await execa( + 'npm', + ['pack', '--pack-destination', packDestination], + { + cwd: path.join(nativePackagesDir, platform), + } + ) + // tarball name is printed as the last line of npm-pack + const tarballName = stdout.trim().split('\n').pop() + console.info(`Created tarball ${path.join(packDestination, tarballName)}`) + + nextSwcPackageNames.add(manifest.name) + }) + ) + + const lernaListJson = await execa('pnpm', [ + '--silent', + 'lerna', + 'list', + '--json', + ]) + const packages = JSON.parse(lernaListJson.stdout) + const packagesByVersion = new Map() + for (const packageInfo of packages) { + packagesByVersion.set( + packageInfo.name, + `https://vercel-packages.vercel.app/next/commits/${commitSha}/${packageInfo.name}` + ) + } + for (const nextSwcPackageName of nextSwcPackageNames) { + packagesByVersion.set( + nextSwcPackageName, + `https://vercel-packages.vercel.app/next/commits/${commitSha}/${nextSwcPackageName}` + ) + } + + console.info(`Creating tarballs for regular packages`) + for (const packageInfo of packages) { + if (packageInfo.private) { + continue + } + + const packageJsonPath = path.join(packageInfo.location, 'package.json') + const packageJson = await fs.readFile(packageJsonPath, 'utf8') + const manifest = JSON.parse(packageJson) + + manifest.version = version + + if (packageInfo.name === 'next') { + manifest.optionalDependencies ??= {} + for (const nextSwcPackageName of nextSwcPackageNames) { + manifest.optionalDependencies[nextSwcPackageName] = + packagesByVersion.get(nextSwcPackageName) + } + } + + // ensure it depends on packages from this release. + for (const [dependencyName, version] of packagesByVersion) { + if (manifest.dependencies?.[dependencyName] !== undefined) { + manifest.dependencies[dependencyName] = version + } + if (manifest.devDependencies?.[dependencyName] !== undefined) { + manifest.devDependencies[dependencyName] = version + } + if (manifest.peerDependencies?.[dependencyName] !== undefined) { + manifest.peerDependencies[dependencyName] = version + } + if (manifest.optionalDependencies?.[dependencyName] !== undefined) { + manifest.optionalDependencies[dependencyName] = version + } + } + + await fs.writeFile( + packageJsonPath, + JSON.stringify(manifest, null, 2) + + // newline will be added by Prettier + '\n' + ) + + // By encoding the package name in the directory, vercel-packages can later extract the package name of a tarball from its path when `tarballDirectory` is zipped. + const packDestination = path.join(tarballDirectory, manifest.name) + await fs.mkdir(packDestination, { recursive: true }) + const { stdout } = await execa( + 'npm', + ['pack', '--pack-destination', packDestination], + { + cwd: packageInfo.location, + } + ) + // tarball name is printed as the last line of npm-pack + const tarballName = stdout.trim().split('\n').pop() + console.info(`Created tarball ${path.join(packDestination, tarballName)}`) + } + + console.info( + `When this job is completed, a Next.js preview build will be available under ${packagesByVersion.get('next')}` + ) +} + +main().catch((err) => { + console.error(err) + process.exit(1) +}) diff --git a/scripts/deploy-tarball.js b/scripts/deploy-tarball.js deleted file mode 100644 index a6a35cdefd..0000000000 --- a/scripts/deploy-tarball.js +++ /dev/null @@ -1,155 +0,0 @@ -// @ts-check -const path = require('path') -const execa = require('execa') -const fs = require('fs/promises') - -const cwd = process.cwd() - -async function main() { - const deployDir = path.join(cwd, 'files') - const publicDir = path.join(deployDir, 'public') - await fs.mkdir(publicDir, { recursive: true }) - await fs.writeFile( - path.join(deployDir, 'package.json'), - JSON.stringify({ - name: 'files', - dependencies: {}, - scripts: { - build: 'node inject-deploy-url.js', - }, - }) - ) - await fs.copyFile( - path.join(cwd, 'scripts/inject-deploy-url.js'), - path.join(deployDir, 'inject-deploy-url.js') - ) - - let nativePackagesDir = path.join(cwd, 'packages/next-swc/crates/napi/npm') - let platforms = (await fs.readdir(nativePackagesDir)).filter( - (name) => !name.startsWith('.') - ) - - const optionalDeps = {} - const { version } = JSON.parse( - await fs.readFile(path.join(cwd, 'lerna.json'), 'utf8') - ) - - await Promise.all( - platforms.map(async (platform) => { - let binaryName = `next-swc.${platform}.node` - await fs.cp( - path.join(cwd, 'packages/next-swc/native', binaryName), - path.join(nativePackagesDir, platform, binaryName) - ) - let pkg = JSON.parse( - await fs.readFile( - path.join(nativePackagesDir, platform, 'package.json'), - 'utf8' - ) - ) - pkg.version = version - await fs.writeFile( - path.join(nativePackagesDir, platform, 'package.json'), - JSON.stringify(pkg, null, 2) - ) - const { stdout } = await execa(`npm`, [ - `pack`, - `${path.join(nativePackagesDir, platform)}`, - ]) - process.stdout.write(stdout) - const tarballName = stdout.split('\n').pop()?.trim() || '' - await fs.rename( - path.join(cwd, tarballName), - path.join(publicDir, tarballName) - ) - optionalDeps[pkg.name] = `https://DEPLOY_URL/${tarballName}` - }) - ) - - const nextPkgJsonPath = path.join(cwd, 'packages/next/package.json') - const nextPkg = JSON.parse(await fs.readFile(nextPkgJsonPath, 'utf8')) - - nextPkg.optionalDependencies = optionalDeps - - await fs.writeFile(nextPkgJsonPath, JSON.stringify(nextPkg, null, 2)) - - const { stdout: nextPackStdout } = await execa(`npm`, [ - `pack`, - `${path.join(cwd, 'packages/next')}`, - ]) - process.stdout.write(nextPackStdout) - const nextTarballName = nextPackStdout.split('\n').pop()?.trim() || '' - await fs.rename( - path.join(cwd, nextTarballName), - path.join(publicDir, nextTarballName) - ) - - await fs.writeFile( - path.join(deployDir, 'vercel.json'), - JSON.stringify( - { - version: 2, - rewrites: [ - { - source: '/next.tgz', - destination: `/${nextTarballName}`, - }, - ], - }, - null, - 2 - ) - ) - const vercelConfigDir = path.join(cwd, '.vercel') - - if (process.env.VERCEL_TEST_TOKEN) { - await fs.mkdir(vercelConfigDir) - await fs.writeFile( - path.join(vercelConfigDir, 'auth.json'), - JSON.stringify({ - token: process.env.VERCEL_TEST_TOKEN, - }) - ) - await fs.writeFile( - path.join(vercelConfigDir, 'config.json'), - JSON.stringify({}) - ) - console.log('wrote config to', vercelConfigDir) - } - - const child = execa( - 'vercel', - [ - '--scope', - process.env.VERCEL_TEST_TEAM || '', - '--global-config', - vercelConfigDir, - '-y', - ], - { - cwd: deployDir, - } - ) - let deployOutput = '' - const handleData = (type) => (chunk) => { - process[type].write(chunk) - - // only want stdout since that's where deployment URL - // is sent to - if (type === 'stdout') { - deployOutput += chunk.toString() - } - } - child.stdout?.on('data', handleData('stdout')) - child.stderr?.on('data', handleData('stderr')) - - await child - - const deployUrl = deployOutput.trim() - console.log(`\n\nNext.js tarball: ${deployUrl.trim()}/next.tgz`) -} - -main().catch((err) => { - console.error(err) - process.exit(1) -})