Apply publish step optimizations (#43620)
Follow-up to https://github.com/vercel/next.js/pull/32337 this removes the un-necessary step where we fetch all of the tags which requires pulling a lot of un-necessary git history inflating cache size and publish times. The only reason these tags were needing to be fetched is due to an issue in how the `actions/checkout` step works (https://github.com/actions/checkout/issues/882). This reduces the publish times by at least 4 minutes by removing the tags fetching step https://github.com/vercel/next.js/actions/runs/3598569786/jobs/6061449995#step:16:14 As a further optimization this adds concurrency to the `npm publish` calls themselves to hopefully reduce time spent there as well.
This commit is contained in:
parent
2063ff3338
commit
6b863fe294
5 changed files with 166 additions and 129 deletions
5
.github/workflows/build_test_deploy.yml
vendored
5
.github/workflows/build_test_deploy.yml
vendored
|
@ -84,11 +84,10 @@ jobs:
|
|||
- run: pnpm install
|
||||
- run: pnpm run build
|
||||
- run: node run-tests.js --timings --write-timings -g 1/1
|
||||
- run: node ./scripts/fetch-tags.mjs ${{ github.sha }}
|
||||
|
||||
- id: check-release
|
||||
run: |
|
||||
if [[ $(git describe --exact-match 2> /dev/null || :) = v* ]];
|
||||
if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]];
|
||||
then
|
||||
echo "::set-output name=IS_RELEASE::true"
|
||||
else
|
||||
|
@ -893,7 +892,7 @@ jobs:
|
|||
|
||||
- run: npm i -g pnpm@${PNPM_VERSION}
|
||||
- run: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
|
||||
- run: ./scripts/publish-native.js $GITHUB_REF
|
||||
- run: ./scripts/publish-native.js
|
||||
- run: ./scripts/publish-release.js
|
||||
|
||||
testDeployE2E:
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import { execSync } from 'child_process'
|
||||
import execa from 'execa'
|
||||
;(async () => {
|
||||
const { execSync } = require('child_process')
|
||||
|
||||
const checkIsRelease = async () => {
|
||||
let commitId = process.argv[2] || ''
|
||||
|
||||
// parse only the last string which should be version if
|
||||
|
@ -14,21 +14,13 @@ import execa from 'execa'
|
|||
const versionString = commitMsg.split(' ').pop().trim()
|
||||
const publishMsgRegex = /^v\d{1,}\.\d{1,}\.\d{1,}(-\w{1,}\.\d{1,})?$/
|
||||
|
||||
console.log({ commitId, commitMsg, versionString })
|
||||
|
||||
if (publishMsgRegex.test(versionString)) {
|
||||
console.log('publish commit, fetching tags')
|
||||
|
||||
const result = await execa(
|
||||
'git',
|
||||
['fetch', '--depth=1', 'origin', '+refs/tags/*:refs/tags/*'],
|
||||
{
|
||||
stdio: ['ignore', 'inherit', 'inherit'],
|
||||
}
|
||||
)
|
||||
|
||||
process.exit(result.exitCode)
|
||||
console.log(versionString)
|
||||
process.exit(0)
|
||||
} else {
|
||||
console.log('not publish commit')
|
||||
console.log('not publish commit', { commitId, commitMsg, versionString })
|
||||
process.exit(1)
|
||||
}
|
||||
})()
|
||||
}
|
||||
|
||||
checkIsRelease()
|
|
@ -1,18 +1,20 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
const path = require('path')
|
||||
const { readFile, readdir, writeFile } = require('fs/promises')
|
||||
const execa = require('execa')
|
||||
const { copy } = require('fs-extra')
|
||||
const { execSync } = require('child_process')
|
||||
const { Sema } = require('async-sema')
|
||||
const { readFile, readdir, writeFile } = require('fs/promises')
|
||||
|
||||
const cwd = process.cwd()
|
||||
|
||||
;(async function () {
|
||||
try {
|
||||
const publishSema = new Sema(2)
|
||||
|
||||
let version = JSON.parse(
|
||||
await readFile(path.join(cwd, 'lerna.json'))
|
||||
).version
|
||||
let gitref = process.argv.slice(2)[0]
|
||||
|
||||
// Copy binaries to package folders, update version, and publish
|
||||
let nativePackagesDir = path.join(cwd, 'packages/next-swc/crates/napi/npm')
|
||||
|
@ -20,93 +22,116 @@ const cwd = process.cwd()
|
|||
(name) => !name.startsWith('.')
|
||||
)
|
||||
|
||||
for (let platform of platforms) {
|
||||
try {
|
||||
let binaryName = `next-swc.${platform}.node`
|
||||
await copy(
|
||||
path.join(cwd, 'packages/next-swc/native', binaryName),
|
||||
path.join(nativePackagesDir, platform, binaryName)
|
||||
)
|
||||
let pkg = JSON.parse(
|
||||
await readFile(path.join(nativePackagesDir, platform, 'package.json'))
|
||||
)
|
||||
pkg.version = version
|
||||
await writeFile(
|
||||
path.join(nativePackagesDir, platform, 'package.json'),
|
||||
JSON.stringify(pkg, null, 2)
|
||||
)
|
||||
execSync(
|
||||
`npm publish ${path.join(
|
||||
nativePackagesDir,
|
||||
platform
|
||||
)} --access public ${
|
||||
gitref.includes('canary') ? ' --tag canary' : ''
|
||||
}`
|
||||
)
|
||||
} catch (err) {
|
||||
// don't block publishing other versions on single platform error
|
||||
console.error(`Failed to publish`, platform)
|
||||
await Promise.all(
|
||||
platforms.map(async (platform) => {
|
||||
await publishSema.acquire()
|
||||
|
||||
if (
|
||||
err.message &&
|
||||
err.message.includes(
|
||||
'You cannot publish over the previously published versions'
|
||||
try {
|
||||
let binaryName = `next-swc.${platform}.node`
|
||||
await copy(
|
||||
path.join(cwd, 'packages/next-swc/native', binaryName),
|
||||
path.join(nativePackagesDir, platform, binaryName)
|
||||
)
|
||||
) {
|
||||
console.error('Ignoring already published error', platform)
|
||||
} else {
|
||||
// throw err
|
||||
let pkg = JSON.parse(
|
||||
await readFile(
|
||||
path.join(nativePackagesDir, platform, 'package.json')
|
||||
)
|
||||
)
|
||||
pkg.version = version
|
||||
await writeFile(
|
||||
path.join(nativePackagesDir, platform, 'package.json'),
|
||||
JSON.stringify(pkg, null, 2)
|
||||
)
|
||||
await execa(
|
||||
`npm`,
|
||||
[
|
||||
`publish`,
|
||||
`${path.join(nativePackagesDir, platform)}`,
|
||||
`--access`,
|
||||
`public`,
|
||||
...(version.includes('canary') ? ['--tag', 'canary'] : []),
|
||||
],
|
||||
{ stdio: 'inherit' }
|
||||
)
|
||||
} catch (err) {
|
||||
// don't block publishing other versions on single platform error
|
||||
console.error(`Failed to publish`, platform, err)
|
||||
|
||||
if (
|
||||
err.message &&
|
||||
err.message.includes(
|
||||
'You cannot publish over the previously published versions'
|
||||
)
|
||||
) {
|
||||
console.error('Ignoring already published error', platform, err)
|
||||
} else {
|
||||
// throw err
|
||||
}
|
||||
} finally {
|
||||
publishSema.release()
|
||||
}
|
||||
}
|
||||
// lerna publish in next step sill fail if git status is not clean
|
||||
execSync(
|
||||
`git update-index --skip-worktree ${path.join(
|
||||
nativePackagesDir,
|
||||
platform,
|
||||
'package.json'
|
||||
)}`
|
||||
)
|
||||
}
|
||||
// lerna publish in next step sill fail if git status is not clean
|
||||
await execa(
|
||||
`git`,
|
||||
[
|
||||
'update-index',
|
||||
'--skip-worktree',
|
||||
`${path.join(nativePackagesDir, platform, 'package.json')}`,
|
||||
],
|
||||
{ stdio: 'inherit' }
|
||||
)
|
||||
})
|
||||
)
|
||||
|
||||
// Update name/version of wasm packages and publish
|
||||
let wasmDir = path.join(cwd, 'packages/next-swc/crates/wasm')
|
||||
for (let wasmTarget of ['web', 'nodejs']) {
|
||||
let wasmPkg = JSON.parse(
|
||||
await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`))
|
||||
)
|
||||
wasmPkg.name = `@next/swc-wasm-${wasmTarget}`
|
||||
wasmPkg.version = version
|
||||
|
||||
await writeFile(
|
||||
path.join(wasmDir, `pkg-${wasmTarget}/package.json`),
|
||||
JSON.stringify(wasmPkg, null, 2)
|
||||
)
|
||||
await Promise.all(
|
||||
['web', 'nodejs'].map(async (wasmTarget) => {
|
||||
await publishSema.acquire()
|
||||
|
||||
try {
|
||||
execSync(
|
||||
`npm publish ${path.join(
|
||||
wasmDir,
|
||||
`pkg-${wasmTarget}`
|
||||
)} --access public ${
|
||||
gitref.includes('canary') ? ' --tag canary' : ''
|
||||
}`
|
||||
let wasmPkg = JSON.parse(
|
||||
await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`))
|
||||
)
|
||||
} catch (err) {
|
||||
// don't block publishing other versions on single platform error
|
||||
console.error(`Failed to publish`, wasmTarget)
|
||||
wasmPkg.name = `@next/swc-wasm-${wasmTarget}`
|
||||
wasmPkg.version = version
|
||||
|
||||
if (
|
||||
err.message &&
|
||||
err.message.includes(
|
||||
'You cannot publish over the previously published versions'
|
||||
await writeFile(
|
||||
path.join(wasmDir, `pkg-${wasmTarget}/package.json`),
|
||||
JSON.stringify(wasmPkg, null, 2)
|
||||
)
|
||||
|
||||
try {
|
||||
await execa(
|
||||
`npm`,
|
||||
[
|
||||
'publish',
|
||||
`${path.join(wasmDir, `pkg-${wasmTarget}`)}`,
|
||||
'--access',
|
||||
'public',
|
||||
...(version.includes('canary') ? ['--tag', 'canary'] : []),
|
||||
],
|
||||
{ stdio: 'inherit' }
|
||||
)
|
||||
) {
|
||||
console.error('Ignoring already published error', wasmTarget)
|
||||
} else {
|
||||
// throw err
|
||||
} catch (err) {
|
||||
// don't block publishing other versions on single platform error
|
||||
console.error(`Failed to publish`, wasmTarget, err)
|
||||
|
||||
if (
|
||||
err.message &&
|
||||
err.message.includes(
|
||||
'You cannot publish over the previously published versions'
|
||||
)
|
||||
) {
|
||||
console.error('Ignoring already published error', wasmTarget)
|
||||
} else {
|
||||
// throw err
|
||||
}
|
||||
} finally {
|
||||
publishSema.release()
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
)
|
||||
|
||||
// Update optional dependencies versions
|
||||
let nextPkg = JSON.parse(
|
||||
|
@ -122,7 +147,13 @@ const cwd = process.cwd()
|
|||
JSON.stringify(nextPkg, null, 2)
|
||||
)
|
||||
// lerna publish in next step will fail if git status is not clean
|
||||
execSync('git update-index --skip-worktree packages/next/package.json')
|
||||
await execa(
|
||||
'git',
|
||||
['update-index', '--skip-worktree', 'packages/next/package.json'],
|
||||
{
|
||||
stdio: 'inherit',
|
||||
}
|
||||
)
|
||||
} catch (err) {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
|
|
|
@ -2,22 +2,20 @@
|
|||
// @ts-check
|
||||
|
||||
const path = require('path')
|
||||
const { readdir } = require('fs/promises')
|
||||
const execa = require('execa')
|
||||
const { Sema } = require('async-sema')
|
||||
const { execSync } = require('child_process')
|
||||
const { readJson } = require('fs-extra')
|
||||
const { readJson, readdir } = require('fs-extra')
|
||||
|
||||
const cwd = process.cwd()
|
||||
|
||||
;(async function () {
|
||||
let isCanary = true
|
||||
|
||||
if (!process.env.NPM_TOKEN) {
|
||||
console.log('No NPM_TOKEN, exiting...')
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const tagOutput = execSync('git describe --exact-match').toString()
|
||||
const tagOutput = execSync(
|
||||
`node ${path.join(__dirname, 'check-is-release.js')}`
|
||||
).toString()
|
||||
console.log(tagOutput)
|
||||
|
||||
if (tagOutput.trim().startsWith('v')) {
|
||||
|
@ -34,15 +32,28 @@ const cwd = process.cwd()
|
|||
}
|
||||
console.log(`Publishing ${isCanary ? 'canary' : 'stable'}`)
|
||||
|
||||
if (!process.env.NPM_TOKEN) {
|
||||
console.log('No NPM_TOKEN, exiting...')
|
||||
return
|
||||
}
|
||||
|
||||
const packagesDir = path.join(cwd, 'packages')
|
||||
const packageDirs = await readdir(packagesDir)
|
||||
const publishSema = new Sema(2)
|
||||
|
||||
const publish = async (pkg, retry = 0) => {
|
||||
try {
|
||||
execSync(
|
||||
`npm publish ${path.join(packagesDir, pkg)} --access public${
|
||||
isCanary ? ' --tag canary' : ''
|
||||
}`
|
||||
await publishSema.acquire()
|
||||
await execa(
|
||||
`npm`,
|
||||
[
|
||||
'publish',
|
||||
`${path.join(packagesDir, pkg)}`,
|
||||
'--access',
|
||||
'public',
|
||||
...(isCanary ? ['--tag', 'canary'] : []),
|
||||
],
|
||||
{ stdio: 'inherit' }
|
||||
)
|
||||
} catch (err) {
|
||||
console.error(`Failed to publish ${pkg}`, err)
|
||||
|
@ -66,18 +77,22 @@ const cwd = process.cwd()
|
|||
await publish(pkg, retry + 1)
|
||||
}
|
||||
throw err
|
||||
} finally {
|
||||
publishSema.release()
|
||||
}
|
||||
}
|
||||
|
||||
for (const packageDir of packageDirs) {
|
||||
const pkgJson = await readJson(
|
||||
path.join(packagesDir, packageDir, 'package.json')
|
||||
)
|
||||
await Promise.all(
|
||||
packageDirs.map(async (packageDir) => {
|
||||
const pkgJson = await readJson(
|
||||
path.join(packagesDir, packageDir, 'package.json')
|
||||
)
|
||||
|
||||
if (pkgJson.private) {
|
||||
console.log(`Skipping private package ${packageDir}`)
|
||||
continue
|
||||
}
|
||||
await publish(packageDir)
|
||||
}
|
||||
if (pkgJson.private) {
|
||||
console.log(`Skipping private package ${packageDir}`)
|
||||
return
|
||||
}
|
||||
await publish(packageDir)
|
||||
})
|
||||
)
|
||||
})()
|
||||
|
|
|
@ -1,11 +1,12 @@
|
|||
#!/bin/bash
|
||||
|
||||
git describe --exact-match
|
||||
|
||||
if [[ ! $? -eq 0 ]];then
|
||||
echo "Nothing to publish, exiting.."
|
||||
touch .github/actions/next-stats-action/SKIP_NEXT_STATS.txt
|
||||
exit 0;
|
||||
if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]];
|
||||
then
|
||||
echo "Publish occurred, running release stats..."
|
||||
else
|
||||
echo "Not publish commit, exiting..."
|
||||
touch .github/actions/next-stats-action/SKIP_NEXT_STATS.txt
|
||||
exit 0;
|
||||
fi
|
||||
|
||||
if [[ -z "$NPM_TOKEN" ]];then
|
||||
|
@ -13,6 +14,5 @@ if [[ -z "$NPM_TOKEN" ]];then
|
|||
exit 0;
|
||||
fi
|
||||
|
||||
echo "Publish occurred, running release stats..."
|
||||
echo "Waiting 30 seconds to allow publish to finalize"
|
||||
sleep 30
|
||||
|
|
Loading…
Reference in a new issue