docs(turbopack): move docs to separate (#62069)
### What For better organization, extracting docs into separate location. Closes PACK-2494
This commit is contained in:
OJ Kwon
GitHub
parent
e1787bb719
commit
00633f1774
24 changed files with 2 additions and 195756 deletions
38
.github/workflows/build_and_deploy.yml
vendored
38
.github/workflows/build_and_deploy.yml
vendored
|
|
@ -473,44 +473,6 @@ jobs:
|
|||
VERCEL_API_TOKEN: ${{ secrets.VERCEL_API_TOKEN }}
|
||||
DEPLOY_ENVIRONMENT: production
|
||||
|
||||
deployTurbopackDocs:
|
||||
name: Deploy Turbopack docs
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build]
|
||||
continue-on-error: true
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 25
|
||||
- name: Install Vercel CLI
|
||||
run: npm i -g vercel@latest
|
||||
- name: Install Rust
|
||||
uses: ./.github/actions/setup-rust
|
||||
- name: Setup mdBook
|
||||
uses: peaceiris/actions-mdbook@v1
|
||||
with:
|
||||
mdbook-version: 'latest'
|
||||
- name: Install cargo-binstall
|
||||
uses: cargo-bins/cargo-binstall@v1.6.3
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
|
||||
- run: |
|
||||
cargo binstall mdbook-mermaid --no-confirm
|
||||
|
||||
- name: Deploy preview docs
|
||||
if: ${{ needs.build.outputs.isRelease != 'true' }}
|
||||
run: ./scripts/deploy-turbopack-docs.sh
|
||||
env:
|
||||
VERCEL_API_TOKEN: ${{ secrets.VERCEL_API_TOKEN }}
|
||||
DEPLOY_ENVIRONMENT: preview
|
||||
- name: Deploy production docs
|
||||
if: ${{ github.ref_name == 'canary' || needs.build.outputs.isRelease == 'true' }}
|
||||
run: ./scripts/deploy-turbopack-docs.sh
|
||||
env:
|
||||
VERCEL_API_TOKEN: ${{ secrets.VERCEL_API_TOKEN }}
|
||||
DEPLOY_ENVIRONMENT: production
|
||||
|
||||
releaseStats:
|
||||
name: Release Stats
|
||||
runs-on:
|
||||
|
|
|
|||
2
.github/workflows/build_and_test.yml
vendored
2
.github/workflows/build_and_test.yml
vendored
|
|
@ -148,7 +148,7 @@ jobs:
|
|||
needsRust: 'yes'
|
||||
skipInstallBuild: 'yes'
|
||||
skipNativeBuild: 'yes'
|
||||
afterBuild: cargo binstall mdbook-mermaid --no-confirm && ./scripts/deploy-turbopack-docs.sh
|
||||
afterBuild: ./scripts/deploy-turbopack-docs.sh
|
||||
stepName: 'rust-doc-check'
|
||||
secrets: inherit
|
||||
|
||||
|
|
|
|||
9
.github/workflows/build_reusable.yml
vendored
9
.github/workflows/build_reusable.yml
vendored
|
|
@ -108,15 +108,6 @@ jobs:
|
|||
with:
|
||||
components: rustfmt, clippy
|
||||
|
||||
- name: Setup mdBook
|
||||
uses: peaceiris/actions-mdbook@v1
|
||||
if: ${{ inputs.skipNativeBuild != 'yes' || inputs.needsNextest == 'yes' || inputs.needsRust == 'yes' }}
|
||||
with:
|
||||
mdbook-version: 'latest'
|
||||
|
||||
- name: Install cargo-binstall
|
||||
uses: cargo-bins/cargo-binstall@v1.6.3
|
||||
|
||||
- name: 'Install mold linker'
|
||||
if: ${{ inputs.mold == 'yes' }}
|
||||
run: |
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
mermaid.initialize({ startOnLoad: true })
|
||||
195290
packages/next-swc/docs/assets/mermaid.min.js
vendored
195290
packages/next-swc/docs/assets/mermaid.min.js
vendored
File diff suppressed because one or more lines are too long
|
|
@ -1,16 +0,0 @@
|
|||
[book]
|
||||
authors = []
|
||||
language = "en"
|
||||
multilingual = false
|
||||
src = "src"
|
||||
title = "Turbopack"
|
||||
|
||||
[preprocessor]
|
||||
|
||||
[preprocessor.mermaid]
|
||||
command = "mdbook-mermaid"
|
||||
|
||||
[output]
|
||||
|
||||
[output.html]
|
||||
additional-js = ["assets/mermaid.min.js", "assets/mermaid-init.js"]
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
# Turbopack
|
||||
|
||||
Turbopack is an incremental bundler optimized for JavaScript and TypeScript, written in Rust.
|
||||
|
||||
This document aims to cover documentation for developing/debugging Turbopack. For information on using Turbopack for Next.js applications, see the Next.js [documentation](https://nextjs.org/docs).
|
||||
|
||||
### What to read
|
||||
|
||||
If this is your first time setting up a development environment, recommend that you read [Setup dev environment](setup.md) and [How to run tests](how_to_run_tests.md) first.
|
||||
|
||||
The implementation of Next.js features in Turbopack is covered in the `Next.js integration` section, and `How Turbopack works` covers high level documentation for how turbopack's internal works.
|
||||
|
||||
Lastly, [API references](api.md) contains all of Turbopack-related package's api documentation based on rustdoc.
|
||||
|
|
@ -1,30 +0,0 @@
|
|||
# Summary
|
||||
|
||||
[Introduction](README.md)
|
||||
|
||||
# Getting started
|
||||
|
||||
- [Setup dev environment](setup.md)
|
||||
- [How to run tests](how_to_run_tests.md)
|
||||
- [Tracing Turbopack](tracing_turbopack.md)
|
||||
|
||||
# How Turbopack works
|
||||
|
||||
- [Architecture](architecture.md)
|
||||
- [Turbo tasks](turbo_tasks.md)
|
||||
- [Turbopack](turbopack.md)
|
||||
|
||||
# Next.js integration
|
||||
|
||||
- [Turbopack binding](turbopack_binding.md)
|
||||
- [Next.rs api](next_rs_api.md)
|
||||
- [Contexts](turbopack_next_contexts.md)
|
||||
- [Custom transforms](turbopack_custom_transforms.md)
|
||||
- [Next/dynamic](next_dynamic.md)
|
||||
- [React server components](rsc.md)
|
||||
|
||||
# Appendix
|
||||
|
||||
- [API references](api.md)
|
||||
- [Additional Links](links.md)
|
||||
- [Writing documentation](documentation.md)
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
# API references
|
||||
|
||||
Turbopack uses code comments and rustdoc to document the API. The full package documentation can be found [here](rustdoc/index.html).
|
||||
|
||||
The list below are packages and brief descriptions that you may want to refer to when first approaching the code.
|
||||
|
||||
- [napi](rustdoc/next_swc_napi/index.html): NAPI bindings to make Turbopack and SWC's features available in JS code in Next.js
|
||||
- [next-custom-transforms](rustdoc/next_custom_transforms/index.html): Collection of SWC transform visitor implementations for features provided by Next.js
|
||||
- [next_api](rustdoc/next_api/index.html): Linking the interfaces and data structures of the NAPI bindings with the actual Turbopack's next.js feature implementation interfaces
|
||||
- [next_core](rustdoc/next_core/index.html): Support and implementation of Next.js features in Turbopack
|
||||
|
|
@ -1,183 +0,0 @@
|
|||
# Architecture
|
||||
|
||||
This is a high level overview of Turbopack's architecture. For a deeper understanding in specific topic, check out the following docs:
|
||||
|
||||
- [turbo tasks](turbo_tasks.md)
|
||||
|
||||
## Core Concepts
|
||||
|
||||
Turbopack has three guiding principles:
|
||||
|
||||
### Incremental
|
||||
|
||||
- Avoid operations that require broad/complete graph traversal
|
||||
- Changes should only have local effects
|
||||
- Avoid depending on "global" information (information from the whole app)
|
||||
- Avoid "one to many" dependencies (depending on one piece of information from many sources)
|
||||
|
||||
### Lazy
|
||||
|
||||
- Avoid computing more information than needed for a given operation
|
||||
- Put extra indirection when needed
|
||||
|
||||
### Extensible
|
||||
|
||||
- Use traits
|
||||
- Avoid depending on concrete implementations (avoid casting)
|
||||
- Make behavior configurable by passing options
|
||||
|
||||
## Layers
|
||||
|
||||
Turbopack models the user's code as it travels through Turbopack in multiple ways, each of which can be thought of as a layer in the system below:
|
||||
|
||||
```
|
||||
|
||||
┌───────────────┬──────────────────────────────┐
|
||||
│ Source code │ │
|
||||
├───────────────┘ │
|
||||
│ What the user wrote as│
|
||||
│ application code│
|
||||
└──────────────────────────────────────────────┘
|
||||
|
||||
┌───────────────┬──────────────────────────────┐
|
||||
│ Modules │ │
|
||||
├───────────────┘ │
|
||||
│ The compiler's understanding of the│
|
||||
│ application code│
|
||||
└──────────────────────────────────────────────┘
|
||||
|
||||
┌───────────────┬──────────────────────────────┐
|
||||
│ Output assets │ │
|
||||
├───────────────┘ │
|
||||
│ What the target environment understands as│
|
||||
│ executable application│
|
||||
└──────────────────────────────────────────────┘
|
||||
|
||||
```
|
||||
|
||||
Each layer is implemented as a decorator/wrapper around the previous layer in top-down order. In contrast to tools like webpack, where whole {source,module,chunk} graphs are built in serial, Turbopack builds lazily by successively wrapping an asset in each layer and building only what's needed without blocking on everything in a particular phase.
|
||||
|
||||
### Source code
|
||||
|
||||
- This is the original, unmodified code from the user's application
|
||||
- Does **not** model references (the relationships between files like through `import`)
|
||||
|
||||
### Modules
|
||||
|
||||
- Result of parsing and transpiling source code
|
||||
- Includes references to other modules
|
||||
- If references are followed, can traverse a subgraph of the application's transpiled code and its recursive dependencies
|
||||
|
||||
### Output asset
|
||||
|
||||
- These are chunks
|
||||
- Result of transforming a module given an output format
|
||||
- Includes references to other output assets
|
||||
- Can be emitted to disk or served from the dev server
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph Source code
|
||||
src ---> b.src.js[ ]
|
||||
src ---> util
|
||||
util ---> d.src.css[ ]
|
||||
util ---> c.src.js[ ]
|
||||
end
|
||||
subgraph Module graph
|
||||
a.ts ---> b.js
|
||||
a.ts ---> c.js
|
||||
b.js ---> c.js
|
||||
c.js ---> d.css
|
||||
c.js ---> e.js[ ]
|
||||
b.js ---> f.js[ ]
|
||||
|
||||
b.js -.-> b.src.js
|
||||
d.css -.-> d.src.css
|
||||
end
|
||||
subgraph Output graph
|
||||
main.js --> chunk.js
|
||||
main.js --> chunk2.js[ ]
|
||||
|
||||
main.js -.-> a.ts
|
||||
main.js -.-> b.js
|
||||
main.js -.-> c.js
|
||||
end
|
||||
```
|
||||
|
||||
## Output format
|
||||
|
||||
An output format changes how a module is run in a specific target environment. The output format is chosen based on target environment and the input module.
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
Browser ---> css_chunking[CSS chunking]
|
||||
Browser ---> static_asset[Static asset]
|
||||
Browser ---> js_chunking[JS chunking]
|
||||
|
||||
Node.js ---> Rebasing
|
||||
|
||||
vercel_nodejs[Vercel Node.js] ---> nft.json
|
||||
|
||||
bundler[Bundler or module-native target] ---> Modules
|
||||
```
|
||||
|
||||
| Target | Input | Output format |
|
||||
| -------------- | ------------ | ---------------------------------------- |
|
||||
| Script tags | JS/TS | JS Chunking |
|
||||
| Link tags | CSS | CSS Chunking |
|
||||
| Browser | Static asset | Static Asset (no chunking) |
|
||||
| Node.js | JS | Rebasing (node-file trace build mode) |
|
||||
| Node.js@Vercel | JS | nft.json file (node-file-trace annotate) |
|
||||
|
||||
In the future, Turbopack should support producing modules as an output format, to be used by other bundlers or runtimes that natively support ESM modules.
|
||||
|
||||
## Chunking
|
||||
|
||||
Chunking is the process that decides which modules are placed into which bundles, and the relationship between these bundles.
|
||||
|
||||
```mermaid
|
||||
|
||||
graph LR
|
||||
module[Module] --->|place in a Chunk| chunk_item[Chunk Item]
|
||||
module_[Module] --->|wrap in Chunk| Chunk
|
||||
|
||||
```
|
||||
|
||||
In order to be placed into chunks (bundles), a module must be wrapped in a Chunk Item. Doing so re-generates the module code, e.g. using `__turbopack_require__`. This still strictly wraps the content of a single module, meaning it assumes that any dependencies must be available to be required.
|
||||
|
||||
If a module is placeable in a chunk, it can be wrapped in a Chunk, which will traverse all of its references.
|
||||
|
||||
## Mix and match
|
||||
|
||||
Not only can Turbopack travel between these different layers, but it can do so across and between different environments.
|
||||
|
||||
- It's possible to transitions between Output Formats
|
||||
- For example, starting in ecmascript but referencing static assets with `new URL('./foo.png')`
|
||||
- Embed/Reference Output Assets in Modules
|
||||
- e.g. embedding base64 data url of a chunk inside another chunk
|
||||
- SSR Page references client chunks for hydration
|
||||
- embed urls to client-side chunks into server-generated html
|
||||
- references output assets from a different chunking root from a different environment
|
||||
- Client Component referenced from Server Component
|
||||
- Client component wrapper (Server Component) references client chunks of a client component
|
||||
|
||||
All of this is made possible by having a single graph that models all source, modules, and output assets across all different environments.
|
||||
|
||||
## Crates
|
||||
|
||||
Currently `turbopack` splits into `turbopack-core`, `turbopack-css`
|
||||
, `turbopack-ecmascript` and `turbopack` (facade).
|
||||
|
||||
The `turbopack-*` crates (except for `turbopack-core`) add support for a certain
|
||||
type of asset.
|
||||
|
||||
Each of them export a `ModuleAsset`, an implementation of `Asset`, which is able
|
||||
to extract references from this kind of asset (and in the future generate an
|
||||
optimized (either for dev or prod) version of it).
|
||||
|
||||
We are currently missing:
|
||||
|
||||
- `turbopack-node` (node native modules)
|
||||
- `turbopack-wasm` (WASM in node and on the web)
|
||||
- `turbopack-image` (images on the web)
|
||||
- probably more? (e.g. `turbopack-svg`).
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
# Writing documentation for Turbopack
|
||||
|
||||
Turbopack has two main types of documentation. One is a description of the code and APIs, and the other is a high-level description of the overall behavior. Since there is no [way](https://github.com/rust-lang/rust/issues/66249) to integrate with rustdoc yet, we deploy mdbook and rustdoc together with a script.
|
||||
|
||||
## Write a general overview document
|
||||
|
||||
Files written in markdown format in `packages/next-swc/docs/src` can be read by [mdbook](https://rust-lang.github.io/mdBook/)(Like the page you're looking at now). Our configuration includes [mermaid](https://mermaid.js.org/) support to include diagrams. See [archtecture](archtecture.md) how it looks like.
|
||||
|
||||
## Write a rustdoc comment
|
||||
|
||||
If you write a [documentation comment](https://doc.rust-lang.org/reference/comments.html) in your code, it will be included in the documentation generated by rustdoc. The full index can be found [here](https://turbopack-rust-docs.vercel.sh/rustdoc/index.html), and individual packages can be read via `https://turbopack-rust-docs.vercel.sh/rustdoc/${pkg_name}/index.html`.
|
||||
|
||||
## Build the documentation
|
||||
|
||||
The `scripts/deploy-turbopack-docs.sh` script bundles the mdbook and rustdoc and deploys them. You can use it locally as a build script because it won't run the deployment if it doesn't have a token. The script runs a rustdoc build followed by an mdbook build, and then copies the output of the rustdoc build under `mdbook/rustdoc`.
|
||||
|
||||
To locally build the documentation, ensure these are installed
|
||||
|
||||
- Rust compiler
|
||||
- [mdbook](https://github.com/rust-lang/mdBook)
|
||||
- [mdbook-mermaid](https://github.com/badboy/mdbook-mermaid)
|
||||
|
||||
```
|
||||
./scripts/deploy-turbopack-docs.sh
|
||||
open ./target/mdbook/index.html
|
||||
```
|
||||
|
||||
or
|
||||
|
||||
```
|
||||
// note you can't check rustdoc links when running devserver
|
||||
mdbook serve --dest-dir $(pwd)/target/mdbook ./packages/next-swc/docs
|
||||
```
|
||||
|
|
@ -1 +0,0 @@
|
|||
# How to run tests
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
# Links
|
||||
|
||||
The links below are additional materials such as conference recording. Some of links are not publicly available yet. Ask in team channel if you don't have access.
|
||||
|
||||
- [GitNation: The Core of Turbopack Explained (Live Coding)](https://portal.gitnation.org/contents/the-core-of-turbopack-explained-live-coding)
|
||||
- [GitNation(Rethinking Bundling Strategies)](https://portal.gitnation.org/contents/rethinking-bundling-strategies)
|
||||
- [GitNation(Turbopack. Why? How? When? and the Vision...)](https://portal.gitnation.org/contents/turbopack-why-how-when-and-the-vision)
|
||||
|
||||
- [Internal discussions](https://www.notion.so/vercel/Deep-Dives-d25c7e3b383347dfad542d9053b0c7e6?pvs=4)
|
||||
|
|
@ -1 +0,0 @@
|
|||
# Next/dynamic
|
||||
|
|
@ -1 +0,0 @@
|
|||
# Next.rs api
|
||||
|
|
@ -1 +0,0 @@
|
|||
# React server components
|
||||
|
|
@ -1 +0,0 @@
|
|||
# Setup
|
||||
|
|
@ -1 +0,0 @@
|
|||
# Tracing Turbopack
|
||||
|
|
@ -1,29 +0,0 @@
|
|||
# Turbo tasks
|
||||
|
||||
## idea
|
||||
|
||||
## concept
|
||||
|
||||
## Tasks
|
||||
|
||||
## Cells and Vcs
|
||||
|
||||
## Passing by value (TaskInput)
|
||||
|
||||
## Best practices
|
||||
|
||||
### Singleton
|
||||
|
||||
## turbo-tasks-fs, turbo-tasks-env
|
||||
|
||||
## Advanced
|
||||
|
||||
### Serialization
|
||||
|
||||
### Registry, turbo-tasks-build, register
|
||||
|
||||
### State
|
||||
|
||||
### Garbage collection
|
||||
|
||||
### Determinism
|
||||
|
|
@ -1,56 +0,0 @@
|
|||
# Turbopack
|
||||
|
||||
## Sources
|
||||
|
||||
### FileSource
|
||||
|
||||
### VirtualSource
|
||||
|
||||
## Modules
|
||||
|
||||
### AssetContext
|
||||
|
||||
### ModuleReference
|
||||
|
||||
#### Reference types
|
||||
|
||||
## OuputAssets
|
||||
|
||||
## Crate structure
|
||||
|
||||
## Advanced
|
||||
|
||||
### Resolving
|
||||
|
||||
#### ResolveOptions
|
||||
|
||||
### Layers
|
||||
|
||||
### Chunking
|
||||
|
||||
#### AvailablilityInfo
|
||||
#### Chunk groups
|
||||
|
||||
### Transitions
|
||||
|
||||
### CompileTimeInfo
|
||||
|
||||
### SourceMaps
|
||||
|
||||
### Issues
|
||||
|
||||
## Ecmascript
|
||||
|
||||
### VarGraph
|
||||
|
||||
### Linking
|
||||
|
||||
## CSS
|
||||
|
||||
## JSON
|
||||
|
||||
## Assets
|
||||
|
||||
## Images
|
||||
|
||||
## Non ecma inputs (MDX)
|
||||
|
|
@ -1 +0,0 @@
|
|||
# Turbopack binding
|
||||
|
|
@ -1 +0,0 @@
|
|||
# Custom transforms
|
||||
|
|
@ -1 +0,0 @@
|
|||
# Contexts
|
||||
|
|
@ -1,33 +1,5 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
|
||||
PROD=""
|
||||
|
||||
if [ -z ${DEPLOY_ENVIRONMENT+x} ]; then
|
||||
DEPLOY_ENVIRONMENT=""
|
||||
else
|
||||
if [ "$DEPLOY_ENVIRONMENT" = "production" ]; then
|
||||
PROD="--prod"
|
||||
fi
|
||||
fi
|
||||
|
||||
mdbook build --dest-dir $(pwd)/target/mdbook ./packages/next-swc/docs
|
||||
|
||||
PACKAGES="-p next-swc-napi -p next-api -p next-build -p next-core -p next-custom-transforms"
|
||||
RUSTDOCFLAGS="-Z unstable-options --enable-index-page" cargo doc $PACKAGES --no-deps
|
||||
|
||||
cp -r $(pwd)/target/doc $(pwd)/target/mdbook/rustdoc
|
||||
|
||||
if [ -z ${VERCEL_API_TOKEN+x} ]; then
|
||||
echo "VERCEL_API_TOKEN was not providing, skipping..."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
||||
DOCS_OUTDIR="$(pwd)/target/mdbook"
|
||||
PROJECT="turbopack-rust-docs"
|
||||
echo "Deploying directory $DOCS_OUTDIR as $PROJECT to Vercel..."
|
||||
|
||||
vercel link --cwd $DOCS_OUTDIR --scope vercel --project $PROJECT --token "$VERCEL_API_TOKEN" --yes
|
||||
vercel deploy --cwd $DOCS_OUTDIR --token "$VERCEL_API_TOKEN" $PROD
|
||||
RUSTDOCFLAGS="-Z unstable-options --enable-index-page" cargo doc $PACKAGES --no-deps
|
||||
Loading…
Reference in a new issue