Auto merge of #127786 - ehuss:rustbook-workspace, r=Mark-Simulacrum

Move rustbook to its own workspace.

This moves rustbook (the wrapper around mdbook) to its own Cargo workspace. This is done for two reasons:

- Some users want to avoid having to check out documentation submodules if they are not working on documentation. These submodules are required for submodules that have Cargo dependencies in the tree (such as mdbook preprocessors).
- The [pinned `memchr`](eb72697e41/compiler/rustc_ast/Cargo.toml (L10)) is causing problems with updating. That pin is only necessary for the standard library, but unfortunately it is affecting all other crates.

This will have some drawbacks:

- A slight increase in the vendor directory size. My measurement shows about a 14M increase (0.7%), but somehow the compressed filesize is smaller.
- The dependencies for rustbook now need to be managed separately. I have updated the cron job to try to mitigate this.
- There will be a slight dist build time penalty. I'm not sure what it will be, since it heavily depends on the machine, but I suspect in the 30-45s range.
- Adds more complexity to things like bootstrap and tidy.

There are a few other alternatives considered:

- Publish preprocessors on crates.io. This adds the burden of publishing every change, and ensuring those publishes happen and the sources don't get out of sync, and somehow syncing those updates back to rust-lang/rust during the automatic updates. This is also more work.
- Move the submodules to subtrees. These have the added burden of doing updates in a way that is more difficult than submodules. I believe it also causes problems with GitHub's `#NNNN` tagging and closing issues. This is also more work.

The only thing I haven't tested here is the cron job. However, there's a pretty decent chance this won't pass CI, or that I missed something.
This commit is contained in:
bors 2024-07-22 20:12:52 +00:00
commit cefe1dcef0
10 changed files with 1796 additions and 297 deletions

View File

@ -64,11 +64,17 @@ jobs:
- name: cargo update
# Remove first line that always just says "Updating crates.io index"
run: cargo update 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
- name: cargo update rustbook
run: |
echo -e "\nrustbook dependencies:" >> cargo_update.log
cargo update --manifest-path src/tools/rustbook 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
- name: upload Cargo.lock artifact for use in PR
uses: actions/upload-artifact@v4
with:
name: Cargo-lock
path: Cargo.lock
path: |
Cargo.lock
src/tools/rustbook/Cargo.lock
retention-days: 1
- name: upload cargo-update log artifact for use in PR
uses: actions/upload-artifact@v4
@ -113,7 +119,7 @@ jobs:
git config user.name github-actions
git config user.email github-actions@github.com
git switch --force-create cargo_update
git add ./Cargo.lock
git add ./Cargo.lock ./src/tools/rustbook/Cargo.lock
git commit --no-verify --file=commit.txt
- name: push

View File

@ -284,21 +284,6 @@ dependencies = [
"nom",
]
[[package]]
name = "assert_cmd"
version = "2.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed72493ac66d5804837f480ab3766c72bdfab91a65e565fc54fa9e42db0073a8"
dependencies = [
"anstyle",
"bstr",
"doc-comment",
"predicates",
"predicates-core",
"predicates-tree",
"wait-timeout",
]
[[package]]
name = "autocfg"
version = "1.3.0"
@ -1111,12 +1096,6 @@ version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8"
[[package]]
name = "difflib"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
[[package]]
name = "digest"
version = "0.10.7"
@ -1208,12 +1187,6 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "doc-comment"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10"
[[package]]
name = "either"
version = "1.12.0"
@ -2203,21 +2176,6 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "line-wrap"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd1bc4d24ad230d21fb898d1116b1801d7adfc449d42026475862ab48b11e70e"
[[package]]
name = "linereader"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d921fea6860357575519aca014c6e22470585accdd543b370c404a8a72d0dd1d"
dependencies = [
"memchr",
]
[[package]]
name = "linkchecker"
version = "0.1.0"
@ -2226,12 +2184,6 @@ dependencies = [
"regex",
]
[[package]]
name = "linked-hash-map"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
[[package]]
name = "lint-docs"
version = "0.1.0"
@ -2368,52 +2320,6 @@ dependencies = [
"topological-sort",
]
[[package]]
name = "mdbook-i18n-helpers"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c8f972ab672d366c3dad77ea5aa7bae68db2d25fbeb889849f97469d7b658e4"
dependencies = [
"anyhow",
"chrono",
"mdbook",
"polib",
"pulldown-cmark 0.10.3",
"pulldown-cmark-to-cmark",
"regex",
"semver",
"serde_json",
"syntect",
"textwrap",
]
[[package]]
name = "mdbook-trpl-listing"
version = "0.1.0"
dependencies = [
"assert_cmd",
"clap",
"mdbook",
"pulldown-cmark 0.10.3",
"pulldown-cmark-to-cmark",
"serde_json",
"thiserror",
"toml 0.8.14",
"xmlparser",
]
[[package]]
name = "mdbook-trpl-note"
version = "1.0.0"
dependencies = [
"assert_cmd",
"clap",
"mdbook",
"pulldown-cmark 0.10.3",
"pulldown-cmark-to-cmark",
"serde_json",
]
[[package]]
name = "measureme"
version = "11.0.1"
@ -2755,28 +2661,6 @@ version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "onig"
version = "6.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f"
dependencies = [
"bitflags 1.3.2",
"libc",
"once_cell",
"onig_sys",
]
[[package]]
name = "onig_sys"
version = "69.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7"
dependencies = [
"cc",
"pkg-config",
]
[[package]]
name = "opener"
version = "0.6.1"
@ -3076,29 +2960,6 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
[[package]]
name = "plist"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9d34169e64b3c7a80c8621a48adaf44e0cf62c78a9b25dd9dd35f1881a17cf9"
dependencies = [
"base64",
"indexmap",
"line-wrap",
"quick-xml",
"serde",
"time",
]
[[package]]
name = "polib"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b393b155cf9be86249cba1b56cc81be0e6212c66d94ac0d76d37a1761f3bb1b"
dependencies = [
"linereader",
]
[[package]]
name = "polonius-engine"
version = "0.13.0"
@ -3134,33 +2995,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]]
name = "predicates"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68b87bfd4605926cdfefc1c3b5f8fe560e3feca9d5552cf68c466d3d8236c7e8"
dependencies = [
"anstyle",
"difflib",
"predicates-core",
]
[[package]]
name = "predicates-core"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174"
[[package]]
name = "predicates-tree"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf"
dependencies = [
"predicates-core",
"termtree",
]
[[package]]
name = "prettydiff"
version = "0.6.4"
@ -3230,7 +3064,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76979bea66e7875e7509c4ec5300112b316af87fa7a252ca91c448b32dfe3993"
dependencies = [
"bitflags 2.5.0",
"getopts",
"memchr",
"pulldown-cmark-escape 0.10.1",
"unicase",
@ -3260,30 +3093,12 @@ version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"
[[package]]
name = "pulldown-cmark-to-cmark"
version = "13.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f609795c8d835f79dcfcf768415b9fb57ef1b74891e99f86e73f43a7a257163b"
dependencies = [
"pulldown-cmark 0.10.3",
]
[[package]]
name = "punycode"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9e1dcb320d6839f6edb64f7a4a59d39b30480d4d1765b56873f7c858538a5fe"
[[package]]
name = "quick-xml"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
dependencies = [
"memchr",
]
[[package]]
name = "quine-mc_cluskey"
version = "0.2.4"
@ -3518,18 +3333,6 @@ dependencies = [
"wasmparser 0.118.2",
]
[[package]]
name = "rustbook"
version = "0.1.0"
dependencies = [
"clap",
"env_logger",
"mdbook",
"mdbook-i18n-helpers",
"mdbook-trpl-listing",
"mdbook-trpl-note",
]
[[package]]
name = "rustc-build-sysroot"
version = "0.5.2"
@ -5549,28 +5352,6 @@ dependencies = [
"syn 2.0.67",
]
[[package]]
name = "syntect"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "874dcfa363995604333cf947ae9f751ca3af4522c60886774c4963943b4746b1"
dependencies = [
"bincode",
"bitflags 1.3.2",
"flate2",
"fnv",
"once_cell",
"onig",
"plist",
"regex-syntax 0.8.4",
"serde",
"serde_derive",
"serde_json",
"thiserror",
"walkdir",
"yaml-rust",
]
[[package]]
name = "sysinfo"
version = "0.30.12"
@ -5678,12 +5459,6 @@ dependencies = [
"winapi",
]
[[package]]
name = "termtree"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76"
[[package]]
name = "test"
version = "0.0.0"
@ -5705,12 +5480,6 @@ dependencies = [
"rayon",
]
[[package]]
name = "textwrap"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
[[package]]
name = "thin-vec"
version = "0.2.13"
@ -5874,19 +5643,7 @@ dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"toml_edit 0.19.15",
]
[[package]]
name = "toml"
version = "0.8.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335"
dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"toml_edit 0.22.14",
"toml_edit",
]
[[package]]
@ -5908,20 +5665,7 @@ dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
"winnow 0.5.40",
]
[[package]]
name = "toml_edit"
version = "0.22.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f21c7aaf97f1bd9ca9d4f9e73b0a6c74bd5afef56f2bc931943a6e1c37e04e38"
dependencies = [
"indexmap",
"serde",
"serde_spanned",
"toml_datetime",
"winnow 0.6.13",
"winnow",
]
[[package]]
@ -6331,15 +6075,6 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "wait-timeout"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
dependencies = [
"libc",
]
[[package]]
name = "walkdir"
version = "2.5.0"
@ -6744,15 +6479,6 @@ dependencies = [
"memchr",
]
[[package]]
name = "winnow"
version = "0.6.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1"
dependencies = [
"memchr",
]
[[package]]
name = "wit-component"
version = "0.210.0"
@ -6807,12 +6533,6 @@ dependencies = [
"rustix",
]
[[package]]
name = "xmlparser"
version = "0.13.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4"
[[package]]
name = "xz2"
version = "0.1.7"
@ -6822,15 +6542,6 @@ dependencies = [
"lzma-sys",
]
[[package]]
name = "yaml-rust"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
dependencies = [
"linked-hash-map",
]
[[package]]
name = "yansi-term"
version = "0.1.2"

View File

@ -16,7 +16,6 @@ members = [
"src/tools/linkchecker",
"src/tools/lint-docs",
"src/tools/miropt-test-tools",
"src/tools/rustbook",
"src/tools/unstable-book-gen",
"src/tools/tidy",
"src/tools/tier-check",

View File

@ -1040,6 +1040,8 @@ impl Step for PlainSourceTarball {
.arg(builder.src.join("./src/tools/opt-dist/Cargo.toml"))
.arg("--sync")
.arg(builder.src.join("./src/tools/rustc-perf/Cargo.toml"))
.arg("--sync")
.arg(builder.src.join("./src/tools/rustbook/Cargo.toml"))
// Will read the libstd Cargo.toml
// which uses the unstable `public-dependency` feature.
.env("RUSTC_BOOTSTRAP", "1")

View File

@ -1172,6 +1172,12 @@ impl Step for RustcBook {
/// in the "md-doc" directory in the build output directory. Then
/// "rustbook" is used to convert it to HTML.
fn run(self, builder: &Builder<'_>) {
// These submodules are required to be checked out to build rustbook
// because they have Cargo dependencies that are needed.
#[allow(clippy::single_element_loop)] // This will change soon.
for path in ["src/doc/book"] {
builder.update_submodule(Path::new(path));
}
let out_base = builder.md_doc_out(self.target).join("rustc");
t!(fs::create_dir_all(&out_base));
let out_listing = out_base.join("src/lints");

View File

@ -34,8 +34,10 @@ impl Step for Vendor {
cmd.arg("--versioned-dirs");
}
// cargo submodule must be present for `x vendor` to work.
builder.build.update_submodule(Path::new("src/tools/cargo"));
// These submodules must be present for `x vendor` to work.
for path in ["src/tools/cargo", "src/doc/book"] {
builder.build.update_submodule(Path::new(path));
}
// Sync these paths by default.
for p in [
@ -44,6 +46,7 @@ impl Step for Vendor {
"compiler/rustc_codegen_cranelift/Cargo.toml",
"compiler/rustc_codegen_gcc/Cargo.toml",
"src/bootstrap/Cargo.toml",
"src/tools/rustbook/Cargo.toml",
] {
cmd.arg("--sync").arg(builder.src.join(p));
}

View File

@ -439,7 +439,7 @@ impl Build {
// Make sure we update these before gathering metadata so we don't get an error about missing
// Cargo.toml files.
let rust_submodules = ["src/doc/book", "library/backtrace", "library/stdarch"];
let rust_submodules = ["library/backtrace", "library/stdarch"];
for s in rust_submodules {
build.update_submodule(Path::new(s));
}

File diff suppressed because it is too large Load Diff

View File

@ -1,3 +1,5 @@
[workspace]
[package]
name = "rustbook"
version = "0.1.0"

View File

@ -70,6 +70,7 @@ pub(crate) const WORKSPACES: &[(&str, ExceptionList, Option<(&[&str], &[&str])>)
//("src/tools/miri/test-cargo-miri", &[], None), // FIXME uncomment once all deps are vendored
//("src/tools/miri/test_dependencies", &[], None), // FIXME uncomment once all deps are vendored
("src/tools/rust-analyzer", EXCEPTIONS_RUST_ANALYZER, None),
("src/tools/rustbook", EXCEPTIONS_RUSTBOOK, None),
("src/tools/rustc-perf", EXCEPTIONS_RUSTC_PERF, None),
("src/tools/x", &[], None),
// tidy-alphabetical-end
@ -167,6 +168,13 @@ const EXCEPTIONS_RUSTC_PERF: ExceptionList = &[
// tidy-alphabetical-end
];
const EXCEPTIONS_RUSTBOOK: ExceptionList = &[
// tidy-alphabetical-start
("mdbook", "MPL-2.0"),
("ryu", "Apache-2.0 OR BSL-1.0"),
// tidy-alphabetical-end
];
const EXCEPTIONS_CRANELIFT: ExceptionList = &[
// tidy-alphabetical-start
("cranelift-bforest", "Apache-2.0 WITH LLVM-exception"),