Apply publish step optimizations (#43620)
Follow-up to https://github.com/vercel/next.js/pull/32337 this removes the un-necessary step where we fetch all of the tags which requires pulling a lot of un-necessary git history inflating cache size and publish times. The only reason these tags were needing to be fetched is due to an issue in how the `actions/checkout` step works (https://github.com/actions/checkout/issues/882). This reduces the publish times by at least 4 minutes by removing the tags fetching step https://github.com/vercel/next.js/actions/runs/3598569786/jobs/6061449995#step:16:14 As a further optimization this adds concurrency to the `npm publish` calls themselves to hopefully reduce time spent there as well.
This commit is contained in:
parent
2063ff3338
commit
6b863fe294
5 changed files with 166 additions and 129 deletions
5
.github/workflows/build_test_deploy.yml
vendored
5
.github/workflows/build_test_deploy.yml
vendored
|
@ -84,11 +84,10 @@ jobs:
|
||||||
- run: pnpm install
|
- run: pnpm install
|
||||||
- run: pnpm run build
|
- run: pnpm run build
|
||||||
- run: node run-tests.js --timings --write-timings -g 1/1
|
- run: node run-tests.js --timings --write-timings -g 1/1
|
||||||
- run: node ./scripts/fetch-tags.mjs ${{ github.sha }}
|
|
||||||
|
|
||||||
- id: check-release
|
- id: check-release
|
||||||
run: |
|
run: |
|
||||||
if [[ $(git describe --exact-match 2> /dev/null || :) = v* ]];
|
if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]];
|
||||||
then
|
then
|
||||||
echo "::set-output name=IS_RELEASE::true"
|
echo "::set-output name=IS_RELEASE::true"
|
||||||
else
|
else
|
||||||
|
@ -893,7 +892,7 @@ jobs:
|
||||||
|
|
||||||
- run: npm i -g pnpm@${PNPM_VERSION}
|
- run: npm i -g pnpm@${PNPM_VERSION}
|
||||||
- run: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
|
- run: echo "//registry.npmjs.org/:_authToken=$NPM_TOKEN" >> ~/.npmrc
|
||||||
- run: ./scripts/publish-native.js $GITHUB_REF
|
- run: ./scripts/publish-native.js
|
||||||
- run: ./scripts/publish-release.js
|
- run: ./scripts/publish-release.js
|
||||||
|
|
||||||
testDeployE2E:
|
testDeployE2E:
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import { execSync } from 'child_process'
|
const { execSync } = require('child_process')
|
||||||
import execa from 'execa'
|
|
||||||
;(async () => {
|
const checkIsRelease = async () => {
|
||||||
let commitId = process.argv[2] || ''
|
let commitId = process.argv[2] || ''
|
||||||
|
|
||||||
// parse only the last string which should be version if
|
// parse only the last string which should be version if
|
||||||
|
@ -14,21 +14,13 @@ import execa from 'execa'
|
||||||
const versionString = commitMsg.split(' ').pop().trim()
|
const versionString = commitMsg.split(' ').pop().trim()
|
||||||
const publishMsgRegex = /^v\d{1,}\.\d{1,}\.\d{1,}(-\w{1,}\.\d{1,})?$/
|
const publishMsgRegex = /^v\d{1,}\.\d{1,}\.\d{1,}(-\w{1,}\.\d{1,})?$/
|
||||||
|
|
||||||
console.log({ commitId, commitMsg, versionString })
|
|
||||||
|
|
||||||
if (publishMsgRegex.test(versionString)) {
|
if (publishMsgRegex.test(versionString)) {
|
||||||
console.log('publish commit, fetching tags')
|
console.log(versionString)
|
||||||
|
process.exit(0)
|
||||||
const result = await execa(
|
|
||||||
'git',
|
|
||||||
['fetch', '--depth=1', 'origin', '+refs/tags/*:refs/tags/*'],
|
|
||||||
{
|
|
||||||
stdio: ['ignore', 'inherit', 'inherit'],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
process.exit(result.exitCode)
|
|
||||||
} else {
|
} else {
|
||||||
console.log('not publish commit')
|
console.log('not publish commit', { commitId, commitMsg, versionString })
|
||||||
|
process.exit(1)
|
||||||
}
|
}
|
||||||
})()
|
}
|
||||||
|
|
||||||
|
checkIsRelease()
|
|
@ -1,18 +1,20 @@
|
||||||
#!/usr/bin/env node
|
#!/usr/bin/env node
|
||||||
|
|
||||||
const path = require('path')
|
const path = require('path')
|
||||||
const { readFile, readdir, writeFile } = require('fs/promises')
|
const execa = require('execa')
|
||||||
const { copy } = require('fs-extra')
|
const { copy } = require('fs-extra')
|
||||||
const { execSync } = require('child_process')
|
const { Sema } = require('async-sema')
|
||||||
|
const { readFile, readdir, writeFile } = require('fs/promises')
|
||||||
|
|
||||||
const cwd = process.cwd()
|
const cwd = process.cwd()
|
||||||
|
|
||||||
;(async function () {
|
;(async function () {
|
||||||
try {
|
try {
|
||||||
|
const publishSema = new Sema(2)
|
||||||
|
|
||||||
let version = JSON.parse(
|
let version = JSON.parse(
|
||||||
await readFile(path.join(cwd, 'lerna.json'))
|
await readFile(path.join(cwd, 'lerna.json'))
|
||||||
).version
|
).version
|
||||||
let gitref = process.argv.slice(2)[0]
|
|
||||||
|
|
||||||
// Copy binaries to package folders, update version, and publish
|
// Copy binaries to package folders, update version, and publish
|
||||||
let nativePackagesDir = path.join(cwd, 'packages/next-swc/crates/napi/npm')
|
let nativePackagesDir = path.join(cwd, 'packages/next-swc/crates/napi/npm')
|
||||||
|
@ -20,93 +22,116 @@ const cwd = process.cwd()
|
||||||
(name) => !name.startsWith('.')
|
(name) => !name.startsWith('.')
|
||||||
)
|
)
|
||||||
|
|
||||||
for (let platform of platforms) {
|
await Promise.all(
|
||||||
try {
|
platforms.map(async (platform) => {
|
||||||
let binaryName = `next-swc.${platform}.node`
|
await publishSema.acquire()
|
||||||
await copy(
|
|
||||||
path.join(cwd, 'packages/next-swc/native', binaryName),
|
|
||||||
path.join(nativePackagesDir, platform, binaryName)
|
|
||||||
)
|
|
||||||
let pkg = JSON.parse(
|
|
||||||
await readFile(path.join(nativePackagesDir, platform, 'package.json'))
|
|
||||||
)
|
|
||||||
pkg.version = version
|
|
||||||
await writeFile(
|
|
||||||
path.join(nativePackagesDir, platform, 'package.json'),
|
|
||||||
JSON.stringify(pkg, null, 2)
|
|
||||||
)
|
|
||||||
execSync(
|
|
||||||
`npm publish ${path.join(
|
|
||||||
nativePackagesDir,
|
|
||||||
platform
|
|
||||||
)} --access public ${
|
|
||||||
gitref.includes('canary') ? ' --tag canary' : ''
|
|
||||||
}`
|
|
||||||
)
|
|
||||||
} catch (err) {
|
|
||||||
// don't block publishing other versions on single platform error
|
|
||||||
console.error(`Failed to publish`, platform)
|
|
||||||
|
|
||||||
if (
|
try {
|
||||||
err.message &&
|
let binaryName = `next-swc.${platform}.node`
|
||||||
err.message.includes(
|
await copy(
|
||||||
'You cannot publish over the previously published versions'
|
path.join(cwd, 'packages/next-swc/native', binaryName),
|
||||||
|
path.join(nativePackagesDir, platform, binaryName)
|
||||||
)
|
)
|
||||||
) {
|
let pkg = JSON.parse(
|
||||||
console.error('Ignoring already published error', platform)
|
await readFile(
|
||||||
} else {
|
path.join(nativePackagesDir, platform, 'package.json')
|
||||||
// throw err
|
)
|
||||||
|
)
|
||||||
|
pkg.version = version
|
||||||
|
await writeFile(
|
||||||
|
path.join(nativePackagesDir, platform, 'package.json'),
|
||||||
|
JSON.stringify(pkg, null, 2)
|
||||||
|
)
|
||||||
|
await execa(
|
||||||
|
`npm`,
|
||||||
|
[
|
||||||
|
`publish`,
|
||||||
|
`${path.join(nativePackagesDir, platform)}`,
|
||||||
|
`--access`,
|
||||||
|
`public`,
|
||||||
|
...(version.includes('canary') ? ['--tag', 'canary'] : []),
|
||||||
|
],
|
||||||
|
{ stdio: 'inherit' }
|
||||||
|
)
|
||||||
|
} catch (err) {
|
||||||
|
// don't block publishing other versions on single platform error
|
||||||
|
console.error(`Failed to publish`, platform, err)
|
||||||
|
|
||||||
|
if (
|
||||||
|
err.message &&
|
||||||
|
err.message.includes(
|
||||||
|
'You cannot publish over the previously published versions'
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
console.error('Ignoring already published error', platform, err)
|
||||||
|
} else {
|
||||||
|
// throw err
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
publishSema.release()
|
||||||
}
|
}
|
||||||
}
|
// lerna publish in next step sill fail if git status is not clean
|
||||||
// lerna publish in next step sill fail if git status is not clean
|
await execa(
|
||||||
execSync(
|
`git`,
|
||||||
`git update-index --skip-worktree ${path.join(
|
[
|
||||||
nativePackagesDir,
|
'update-index',
|
||||||
platform,
|
'--skip-worktree',
|
||||||
'package.json'
|
`${path.join(nativePackagesDir, platform, 'package.json')}`,
|
||||||
)}`
|
],
|
||||||
)
|
{ stdio: 'inherit' }
|
||||||
}
|
)
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
// Update name/version of wasm packages and publish
|
// Update name/version of wasm packages and publish
|
||||||
let wasmDir = path.join(cwd, 'packages/next-swc/crates/wasm')
|
let wasmDir = path.join(cwd, 'packages/next-swc/crates/wasm')
|
||||||
for (let wasmTarget of ['web', 'nodejs']) {
|
|
||||||
let wasmPkg = JSON.parse(
|
|
||||||
await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`))
|
|
||||||
)
|
|
||||||
wasmPkg.name = `@next/swc-wasm-${wasmTarget}`
|
|
||||||
wasmPkg.version = version
|
|
||||||
|
|
||||||
await writeFile(
|
await Promise.all(
|
||||||
path.join(wasmDir, `pkg-${wasmTarget}/package.json`),
|
['web', 'nodejs'].map(async (wasmTarget) => {
|
||||||
JSON.stringify(wasmPkg, null, 2)
|
await publishSema.acquire()
|
||||||
)
|
|
||||||
|
|
||||||
try {
|
let wasmPkg = JSON.parse(
|
||||||
execSync(
|
await readFile(path.join(wasmDir, `pkg-${wasmTarget}/package.json`))
|
||||||
`npm publish ${path.join(
|
|
||||||
wasmDir,
|
|
||||||
`pkg-${wasmTarget}`
|
|
||||||
)} --access public ${
|
|
||||||
gitref.includes('canary') ? ' --tag canary' : ''
|
|
||||||
}`
|
|
||||||
)
|
)
|
||||||
} catch (err) {
|
wasmPkg.name = `@next/swc-wasm-${wasmTarget}`
|
||||||
// don't block publishing other versions on single platform error
|
wasmPkg.version = version
|
||||||
console.error(`Failed to publish`, wasmTarget)
|
|
||||||
|
|
||||||
if (
|
await writeFile(
|
||||||
err.message &&
|
path.join(wasmDir, `pkg-${wasmTarget}/package.json`),
|
||||||
err.message.includes(
|
JSON.stringify(wasmPkg, null, 2)
|
||||||
'You cannot publish over the previously published versions'
|
)
|
||||||
|
|
||||||
|
try {
|
||||||
|
await execa(
|
||||||
|
`npm`,
|
||||||
|
[
|
||||||
|
'publish',
|
||||||
|
`${path.join(wasmDir, `pkg-${wasmTarget}`)}`,
|
||||||
|
'--access',
|
||||||
|
'public',
|
||||||
|
...(version.includes('canary') ? ['--tag', 'canary'] : []),
|
||||||
|
],
|
||||||
|
{ stdio: 'inherit' }
|
||||||
)
|
)
|
||||||
) {
|
} catch (err) {
|
||||||
console.error('Ignoring already published error', wasmTarget)
|
// don't block publishing other versions on single platform error
|
||||||
} else {
|
console.error(`Failed to publish`, wasmTarget, err)
|
||||||
// throw err
|
|
||||||
|
if (
|
||||||
|
err.message &&
|
||||||
|
err.message.includes(
|
||||||
|
'You cannot publish over the previously published versions'
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
console.error('Ignoring already published error', wasmTarget)
|
||||||
|
} else {
|
||||||
|
// throw err
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
publishSema.release()
|
||||||
}
|
}
|
||||||
}
|
})
|
||||||
}
|
)
|
||||||
|
|
||||||
// Update optional dependencies versions
|
// Update optional dependencies versions
|
||||||
let nextPkg = JSON.parse(
|
let nextPkg = JSON.parse(
|
||||||
|
@ -122,7 +147,13 @@ const cwd = process.cwd()
|
||||||
JSON.stringify(nextPkg, null, 2)
|
JSON.stringify(nextPkg, null, 2)
|
||||||
)
|
)
|
||||||
// lerna publish in next step will fail if git status is not clean
|
// lerna publish in next step will fail if git status is not clean
|
||||||
execSync('git update-index --skip-worktree packages/next/package.json')
|
await execa(
|
||||||
|
'git',
|
||||||
|
['update-index', '--skip-worktree', 'packages/next/package.json'],
|
||||||
|
{
|
||||||
|
stdio: 'inherit',
|
||||||
|
}
|
||||||
|
)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(err)
|
console.error(err)
|
||||||
process.exit(1)
|
process.exit(1)
|
||||||
|
|
|
@ -2,22 +2,20 @@
|
||||||
// @ts-check
|
// @ts-check
|
||||||
|
|
||||||
const path = require('path')
|
const path = require('path')
|
||||||
const { readdir } = require('fs/promises')
|
const execa = require('execa')
|
||||||
|
const { Sema } = require('async-sema')
|
||||||
const { execSync } = require('child_process')
|
const { execSync } = require('child_process')
|
||||||
const { readJson } = require('fs-extra')
|
const { readJson, readdir } = require('fs-extra')
|
||||||
|
|
||||||
const cwd = process.cwd()
|
const cwd = process.cwd()
|
||||||
|
|
||||||
;(async function () {
|
;(async function () {
|
||||||
let isCanary = true
|
let isCanary = true
|
||||||
|
|
||||||
if (!process.env.NPM_TOKEN) {
|
|
||||||
console.log('No NPM_TOKEN, exiting...')
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const tagOutput = execSync('git describe --exact-match').toString()
|
const tagOutput = execSync(
|
||||||
|
`node ${path.join(__dirname, 'check-is-release.js')}`
|
||||||
|
).toString()
|
||||||
console.log(tagOutput)
|
console.log(tagOutput)
|
||||||
|
|
||||||
if (tagOutput.trim().startsWith('v')) {
|
if (tagOutput.trim().startsWith('v')) {
|
||||||
|
@ -34,15 +32,28 @@ const cwd = process.cwd()
|
||||||
}
|
}
|
||||||
console.log(`Publishing ${isCanary ? 'canary' : 'stable'}`)
|
console.log(`Publishing ${isCanary ? 'canary' : 'stable'}`)
|
||||||
|
|
||||||
|
if (!process.env.NPM_TOKEN) {
|
||||||
|
console.log('No NPM_TOKEN, exiting...')
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
const packagesDir = path.join(cwd, 'packages')
|
const packagesDir = path.join(cwd, 'packages')
|
||||||
const packageDirs = await readdir(packagesDir)
|
const packageDirs = await readdir(packagesDir)
|
||||||
|
const publishSema = new Sema(2)
|
||||||
|
|
||||||
const publish = async (pkg, retry = 0) => {
|
const publish = async (pkg, retry = 0) => {
|
||||||
try {
|
try {
|
||||||
execSync(
|
await publishSema.acquire()
|
||||||
`npm publish ${path.join(packagesDir, pkg)} --access public${
|
await execa(
|
||||||
isCanary ? ' --tag canary' : ''
|
`npm`,
|
||||||
}`
|
[
|
||||||
|
'publish',
|
||||||
|
`${path.join(packagesDir, pkg)}`,
|
||||||
|
'--access',
|
||||||
|
'public',
|
||||||
|
...(isCanary ? ['--tag', 'canary'] : []),
|
||||||
|
],
|
||||||
|
{ stdio: 'inherit' }
|
||||||
)
|
)
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(`Failed to publish ${pkg}`, err)
|
console.error(`Failed to publish ${pkg}`, err)
|
||||||
|
@ -66,18 +77,22 @@ const cwd = process.cwd()
|
||||||
await publish(pkg, retry + 1)
|
await publish(pkg, retry + 1)
|
||||||
}
|
}
|
||||||
throw err
|
throw err
|
||||||
|
} finally {
|
||||||
|
publishSema.release()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const packageDir of packageDirs) {
|
await Promise.all(
|
||||||
const pkgJson = await readJson(
|
packageDirs.map(async (packageDir) => {
|
||||||
path.join(packagesDir, packageDir, 'package.json')
|
const pkgJson = await readJson(
|
||||||
)
|
path.join(packagesDir, packageDir, 'package.json')
|
||||||
|
)
|
||||||
|
|
||||||
if (pkgJson.private) {
|
if (pkgJson.private) {
|
||||||
console.log(`Skipping private package ${packageDir}`)
|
console.log(`Skipping private package ${packageDir}`)
|
||||||
continue
|
return
|
||||||
}
|
}
|
||||||
await publish(packageDir)
|
await publish(packageDir)
|
||||||
}
|
})
|
||||||
|
)
|
||||||
})()
|
})()
|
||||||
|
|
|
@ -1,11 +1,12 @@
|
||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
git describe --exact-match
|
if [[ $(node ./scripts/check-is-release.js 2> /dev/null || :) = v* ]];
|
||||||
|
then
|
||||||
if [[ ! $? -eq 0 ]];then
|
echo "Publish occurred, running release stats..."
|
||||||
echo "Nothing to publish, exiting.."
|
else
|
||||||
touch .github/actions/next-stats-action/SKIP_NEXT_STATS.txt
|
echo "Not publish commit, exiting..."
|
||||||
exit 0;
|
touch .github/actions/next-stats-action/SKIP_NEXT_STATS.txt
|
||||||
|
exit 0;
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [[ -z "$NPM_TOKEN" ]];then
|
if [[ -z "$NPM_TOKEN" ]];then
|
||||||
|
@ -13,6 +14,5 @@ if [[ -z "$NPM_TOKEN" ]];then
|
||||||
exit 0;
|
exit 0;
|
||||||
fi
|
fi
|
||||||
|
|
||||||
echo "Publish occurred, running release stats..."
|
|
||||||
echo "Waiting 30 seconds to allow publish to finalize"
|
echo "Waiting 30 seconds to allow publish to finalize"
|
||||||
sleep 30
|
sleep 30
|
||||||
|
|
Loading…
Reference in a new issue