Auto merge of #123192 - RalfJung:bootstrap-test-miri, r=onur-ozkan

Refactor the way bootstrap invokes `cargo miri`

Instead of basically doing `cargo run --manifest-path=<cargo-miri's manifest> -- miri`, let's invoke the `cargo-miri` binary directly. That means less indirections, and also makes it easier to e.g. run the libcore test suite in Miri. (But there are still other issues with that.)

Also also adjusted Miri's stage numbering so that it is consistent with rustc/rustdoc.

This also makes `./x.py test miri` honor `--no-doc`.

And this fixes https://github.com/rust-lang/rust/issues/123177 by moving where we handle parallel_compiler.
This commit is contained in:
bors 2024-04-01 07:19:57 +00:00
commit 871df0d13a
10 changed files with 172 additions and 133 deletions

View File

@ -150,7 +150,7 @@ fn main() {
{
cmd.arg("-Ztls-model=initial-exec");
}
} else if std::env::var("MIRI").is_err() {
} else {
// Find any host flags that were passed by bootstrap.
// The flags are stored in a RUSTC_HOST_FLAGS variable, separated by spaces.
if let Ok(flags) = std::env::var("RUSTC_HOST_FLAGS") {

View File

@ -1130,12 +1130,6 @@ pub fn rustc_cargo_env(
cargo.env("CFG_DEFAULT_LINKER", s);
}
if builder.config.rustc_parallel {
// keep in sync with `bootstrap/lib.rs:Build::rustc_features`
// `cfg` option for rustc, `features` option for cargo, for conditional compilation
cargo.rustflag("--cfg=parallel_compiler");
cargo.rustdocflag("--cfg=parallel_compiler");
}
if builder.config.rust_verify_llvm_ir {
cargo.env("RUSTC_VERIFY_LLVM_IR", "1");
}

View File

@ -121,8 +121,6 @@ impl Step for ReplaceVersionPlaceholder {
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Miri {
stage: u32,
host: TargetSelection,
target: TargetSelection,
}
@ -135,29 +133,35 @@ impl Step for Miri {
}
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Miri {
stage: run.builder.top_stage,
host: run.build_triple(),
target: run.target,
});
run.builder.ensure(Miri { target: run.target });
}
fn run(self, builder: &Builder<'_>) {
let stage = self.stage;
let host = self.host;
let host = builder.build.build;
let target = self.target;
let compiler = builder.compiler(stage, host);
let stage = builder.top_stage;
if stage == 0 {
eprintln!("miri cannot be run at stage 0");
std::process::exit(1);
}
let miri =
builder.ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() });
let miri_sysroot = test::Miri::build_miri_sysroot(builder, compiler, &miri, target);
// This compiler runs on the host, we'll just use it for the target.
let target_compiler = builder.compiler(stage, host);
// Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
// we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage
// compilers, which isn't what we want. Rustdoc should be linked in the same way as the
// rustc compiler it's paired with, so it must be built with the previous stage compiler.
let host_compiler = builder.compiler(stage - 1, host);
// Get a target sysroot for Miri.
let miri_sysroot = test::Miri::build_miri_sysroot(builder, target_compiler, target);
// # Run miri.
// Running it via `cargo run` as that figures out the right dylib path.
// add_rustc_lib_path does not add the path that contains librustc_driver-<...>.so.
let mut miri = tool::prepare_tool_cargo(
builder,
compiler,
host_compiler,
Mode::ToolRustc,
host,
"run",

View File

@ -493,8 +493,6 @@ impl Step for RustDemangler {
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct Miri {
stage: u32,
host: TargetSelection,
target: TargetSelection,
}
@ -503,41 +501,26 @@ impl Miri {
pub fn build_miri_sysroot(
builder: &Builder<'_>,
compiler: Compiler,
miri: &Path,
target: TargetSelection,
) -> String {
let miri_sysroot = builder.out.join(compiler.host.triple).join("miri-sysroot");
let mut cargo = tool::prepare_tool_cargo(
let mut cargo = builder::Cargo::new(
builder,
compiler,
Mode::ToolRustc,
compiler.host,
"run",
"src/tools/miri/cargo-miri",
SourceType::InTree,
&[],
Mode::Std,
SourceType::Submodule,
target,
"miri-setup",
);
cargo.add_rustc_lib_path(builder);
cargo.arg("--").arg("miri").arg("setup");
cargo.arg("--target").arg(target.rustc_target_arg());
// Tell `cargo miri setup` where to find the sources.
cargo.env("MIRI_LIB_SRC", builder.src.join("library"));
// Tell it where to find Miri.
cargo.env("MIRI", miri);
// Tell it where to put the sysroot.
cargo.env("MIRI_SYSROOT", &miri_sysroot);
// Debug things.
cargo.env("RUST_BACKTRACE", "1");
let mut cargo = Command::from(cargo);
let _guard = builder.msg(
Kind::Build,
compiler.stage + 1,
"miri sysroot",
compiler.host,
compiler.host,
);
let _guard =
builder.msg(Kind::Build, compiler.stage, "miri sysroot", compiler.host, target);
builder.run(&mut cargo);
// # Determine where Miri put its sysroot.
@ -574,41 +557,51 @@ impl Step for Miri {
}
fn make_run(run: RunConfig<'_>) {
run.builder.ensure(Miri {
stage: run.builder.top_stage,
host: run.build_triple(),
target: run.target,
});
run.builder.ensure(Miri { target: run.target });
}
/// Runs `cargo test` for miri.
fn run(self, builder: &Builder<'_>) {
let stage = self.stage;
let host = self.host;
let host = builder.build.build;
let target = self.target;
let compiler = builder.compiler(stage, host);
// We need the stdlib for the *next* stage, as it was built with this compiler that also built Miri.
// Except if we are at stage 2, the bootstrap loop is complete and we can stick with our current stage.
let compiler_std = builder.compiler(if stage < 2 { stage + 1 } else { stage }, host);
let stage = builder.top_stage;
if stage == 0 {
eprintln!("miri cannot be tested at stage 0");
std::process::exit(1);
}
let miri =
builder.ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() });
let _cargo_miri = builder.ensure(tool::CargoMiri {
compiler,
target: self.host,
// This compiler runs on the host, we'll just use it for the target.
let target_compiler = builder.compiler(stage, host);
// Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise
// we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage
// compilers, which isn't what we want. Rustdoc should be linked in the same way as the
// rustc compiler it's paired with, so it must be built with the previous stage compiler.
let host_compiler = builder.compiler(stage - 1, host);
// Build our tools.
let miri = builder.ensure(tool::Miri {
compiler: host_compiler,
target: host,
extra_features: Vec::new(),
});
// The stdlib we need might be at a different stage. And just asking for the
// sysroot does not seem to populate it, so we do that first.
builder.ensure(compile::Std::new(compiler_std, host));
let sysroot = builder.sysroot(compiler_std);
// We also need a Miri sysroot.
let miri_sysroot = Miri::build_miri_sysroot(builder, compiler, &miri, target);
// the ui tests also assume cargo-miri has been built
builder.ensure(tool::CargoMiri {
compiler: host_compiler,
target: host,
extra_features: Vec::new(),
});
// We also need sysroots, for Miri and for the host (the latter for build scripts).
// This is for the tests so everything is done with the target compiler.
let miri_sysroot = Miri::build_miri_sysroot(builder, target_compiler, target);
builder.ensure(compile::Std::new(target_compiler, host));
let sysroot = builder.sysroot(target_compiler);
// # Run `cargo test`.
// This is with the Miri crate, so it uses the host compiler.
let mut cargo = tool::prepare_tool_cargo(
builder,
compiler,
host_compiler,
Mode::ToolRustc,
host,
"test",
@ -616,26 +609,23 @@ impl Step for Miri {
SourceType::InTree,
&[],
);
let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "miri", host, target);
cargo.add_rustc_lib_path(builder);
// We can NOT use `run_cargo_test` since Miri's integration tests do not use the usual test
// harness and therefore do not understand the flags added by `add_flags_and_try_run_test`.
let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", host_compiler, host, builder);
// miri tests need to know about the stage sysroot
cargo.env("MIRI_SYSROOT", &miri_sysroot);
cargo.env("MIRI_HOST_SYSROOT", &sysroot);
cargo.env("MIRI", &miri);
if builder.config.locked_deps {
// enforce lockfiles
cargo.env("CARGO_EXTRA_FLAGS", "--locked");
}
// Set the target.
cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg());
// This can NOT be `run_cargo_test` since the Miri test runner
// does not understand the flags added by `add_flags_and_try_run_test`.
let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder);
{
let _guard = builder.msg_sysroot_tool(Kind::Test, stage, "miri", host, target);
let _time = helpers::timeit(builder);
builder.run(&mut cargo);
}
@ -650,8 +640,14 @@ impl Step for Miri {
// Optimizations can change error locations and remove UB so don't run `fail` tests.
cargo.args(["tests/pass", "tests/panic"]);
let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder);
{
let _guard = builder.msg_sysroot_tool(
Kind::Test,
stage,
"miri (mir-opt-level 4)",
host,
target,
);
let _time = helpers::timeit(builder);
builder.run(&mut cargo);
}
@ -660,42 +656,40 @@ impl Step for Miri {
// # Run `cargo miri test`.
// This is just a smoke test (Miri's own CI invokes this in a bunch of different ways and ensures
// that we get the desired output), but that is sufficient to make sure that the libtest harness
// itself executes properly under Miri.
// itself executes properly under Miri, and that all the logic in `cargo-miri` does not explode.
// This is running the build `cargo-miri` for the given target, so we need the target compiler.
let mut cargo = tool::prepare_tool_cargo(
builder,
compiler,
Mode::ToolRustc,
host,
"run",
"src/tools/miri/cargo-miri",
target_compiler,
Mode::ToolStd, // it's unclear what to use here, we're not building anything just doing a smoke test!
target,
"miri-test",
"src/tools/miri/test-cargo-miri",
SourceType::Submodule,
&[],
);
cargo.add_rustc_lib_path(builder);
cargo.arg("--").arg("miri").arg("test");
if builder.config.locked_deps {
cargo.arg("--locked");
// We're not using `prepare_cargo_test` so we have to do this ourselves.
// (We're not using that as the test-cargo-miri crate is not known to bootstrap.)
match builder.doc_tests {
DocTests::Yes => {}
DocTests::No => {
cargo.args(["--lib", "--bins", "--examples", "--tests", "--benches"]);
}
DocTests::Only => {
cargo.arg("--doc");
}
}
cargo
.arg("--manifest-path")
.arg(builder.src.join("src/tools/miri/test-cargo-miri/Cargo.toml"));
cargo.arg("--target").arg(target.rustc_target_arg());
cargo.arg("--").args(builder.config.test_args());
// `prepare_tool_cargo` sets RUSTDOC to the bootstrap wrapper and RUSTDOC_REAL to a dummy path as this is a "run", not a "test".
// Also, we want the rustdoc from the "next" stage for the same reason that we build a std from the next stage.
// So let's just set that here, and bypass bootstrap's RUSTDOC (just like cargo-miri already ignores bootstrap's RUSTC_WRAPPER).
cargo.env("RUSTDOC", builder.rustdoc(compiler_std));
// Tell `cargo miri` where to find things.
// Tell `cargo miri` where to find the sysroots.
cargo.env("MIRI_SYSROOT", &miri_sysroot);
cargo.env("MIRI_HOST_SYSROOT", sysroot);
cargo.env("MIRI", &miri);
// Debug things.
cargo.env("RUST_BACKTRACE", "1");
// Finally, pass test-args and run everything.
cargo.arg("--").args(builder.config.test_args());
let mut cargo = Command::from(cargo);
{
let _guard = builder.msg_sysroot_tool(Kind::Test, stage, "cargo-miri", host, target);
let _time = helpers::timeit(builder);
builder.run(&mut cargo);
}

View File

@ -469,7 +469,7 @@ impl Step for Rustdoc {
features.push("jemalloc".to_string());
}
let mut cargo = prepare_tool_cargo(
let cargo = prepare_tool_cargo(
builder,
build_compiler,
Mode::ToolRustc,
@ -480,10 +480,6 @@ impl Step for Rustdoc {
features.as_slice(),
);
if builder.config.rustc_parallel {
cargo.rustflag("--cfg=parallel_compiler");
}
let _guard = builder.msg_tool(
Mode::ToolRustc,
"rustdoc",
@ -732,7 +728,7 @@ impl Step for LlvmBitcodeLinker {
builder.ensure(compile::Std::new(self.compiler, self.compiler.host));
builder.ensure(compile::Rustc::new(self.compiler, self.target));
let mut cargo = prepare_tool_cargo(
let cargo = prepare_tool_cargo(
builder,
self.compiler,
Mode::ToolRustc,
@ -743,10 +739,6 @@ impl Step for LlvmBitcodeLinker {
&self.extra_features,
);
if builder.config.rustc_parallel {
cargo.rustflag("--cfg=parallel_compiler");
}
builder.run(&mut cargo.into());
let tool_out = builder

View File

@ -1009,10 +1009,10 @@ impl<'a> Builder<'a> {
StepDescription::run(v, self, paths);
}
/// Obtain a compiler at a given stage and for a given host. Explicitly does
/// not take `Compiler` since all `Compiler` instances are meant to be
/// obtained through this function, since it ensures that they are valid
/// (i.e., built and assembled).
/// Obtain a compiler at a given stage and for a given host (i.e., this is the target that the
/// compiler will run on, *not* the target it will build code for). Explicitly does not take
/// `Compiler` since all `Compiler` instances are meant to be obtained through this function,
/// since it ensures that they are valid (i.e., built and assembled).
pub fn compiler(&self, stage: u32, host: TargetSelection) -> Compiler {
self.ensure(compile::Assemble { target_compiler: Compiler { stage, host } })
}
@ -1231,6 +1231,36 @@ impl<'a> Builder<'a> {
cmd
}
pub fn cargo_miri_cmd(&self, run_compiler: Compiler) -> Command {
assert!(run_compiler.stage > 0, "miri can not be invoked at stage 0");
let build_compiler = self.compiler(run_compiler.stage - 1, self.build.build);
let miri = self.ensure(tool::Miri {
compiler: build_compiler,
target: self.build.build,
extra_features: Vec::new(),
});
let cargo_miri = self.ensure(tool::CargoMiri {
compiler: build_compiler,
target: self.build.build,
extra_features: Vec::new(),
});
// Invoke cargo-miri, make sure we can find miri and cargo.
let mut cmd = Command::new(cargo_miri);
cmd.env("MIRI", &miri);
cmd.env("CARGO", &self.initial_cargo);
// Need to add the `run_compiler` libs. Those are the libs produces *by* `build_compiler`,
// so they match the Miri we just built. However this means they are actually living one
// stage up, i.e. we are running `stage0-tools-bin/miri` with the libraries in `stage1/lib`.
// This is an unfortunate off-by-1 caused (possibly) by the fact that Miri doesn't have an
// "assemble" step like rustc does that would cross the stage boundary. We can't use
// `add_rustc_lib_path` as that's a NOP on Windows but we do need these libraries added to
// the PATH due to the stage mismatch.
// Also see https://github.com/rust-lang/rust/pull/123192#issuecomment-2028901503.
add_dylib_path(self.rustc_lib_paths(run_compiler), &mut cmd);
cmd
}
pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command {
let mut cmd = Command::new(self.bootstrap_out.join("rustdoc"));
cmd.env("RUSTC_STAGE", compiler.stage.to_string())
@ -1272,20 +1302,26 @@ impl<'a> Builder<'a> {
compiler: Compiler,
mode: Mode,
target: TargetSelection,
cmd: &str,
cmd: &str, // FIXME make this properly typed
) -> Command {
let mut cargo = if cmd == "clippy" {
self.cargo_clippy_cmd(compiler)
let mut cargo;
if cmd == "clippy" {
cargo = self.cargo_clippy_cmd(compiler);
cargo.arg(cmd);
} else if let Some(subcmd) = cmd.strip_prefix("miri-") {
cargo = self.cargo_miri_cmd(compiler);
cargo.arg("miri").arg(subcmd);
} else {
Command::new(&self.initial_cargo)
};
cargo = Command::new(&self.initial_cargo);
cargo.arg(cmd);
}
// Run cargo from the source root so it can find .cargo/config.
// This matters when using vendoring and the working directory is outside the repository.
cargo.current_dir(&self.src);
let out_dir = self.stage_out(compiler, mode);
cargo.env("CARGO_TARGET_DIR", &out_dir).arg(cmd);
cargo.env("CARGO_TARGET_DIR", &out_dir);
// Found with `rg "init_env_logger\("`. If anyone uses `init_env_logger`
// from out of tree it shouldn't matter, since x.py is only used for
@ -1315,7 +1351,8 @@ impl<'a> Builder<'a> {
if self.config.rust_optimize.is_release() {
// FIXME: cargo bench/install do not accept `--release`
if cmd != "bench" && cmd != "install" {
// and miri doesn't want it
if cmd != "bench" && cmd != "install" && !cmd.starts_with("miri-") {
cargo.arg("--release");
}
}
@ -1331,14 +1368,15 @@ impl<'a> Builder<'a> {
/// Cargo. This cargo will be configured to use `compiler` as the actual
/// rustc compiler, its output will be scoped by `mode`'s output directory,
/// it will pass the `--target` flag for the specified `target`, and will be
/// executing the Cargo command `cmd`.
/// executing the Cargo command `cmd`. `cmd` can be `miri-cmd` for commands
/// to be run with Miri.
fn cargo(
&self,
compiler: Compiler,
mode: Mode,
source_type: SourceType,
target: TargetSelection,
cmd: &str,
cmd: &str, // FIXME make this properly typed
) -> Cargo {
let mut cargo = self.bare_cargo(compiler, mode, target, cmd);
let out_dir = self.stage_out(compiler, mode);
@ -1649,7 +1687,8 @@ impl<'a> Builder<'a> {
.env("RUSTDOC", self.bootstrap_out.join("rustdoc"))
.env(
"RUSTDOC_REAL",
if cmd == "doc" || cmd == "rustdoc" || (cmd == "test" && want_rustdoc) {
// Make sure to handle both `test` and `miri-test` commands.
if cmd == "doc" || cmd == "rustdoc" || (cmd.ends_with("test") && want_rustdoc) {
self.rustdoc(compiler)
} else {
PathBuf::from("/path/to/nowhere/rustdoc/not/required")
@ -2045,6 +2084,15 @@ impl<'a> Builder<'a> {
rustflags.arg("-Zinline-mir");
}
if self.config.rustc_parallel
&& matches!(mode, Mode::ToolRustc | Mode::Rustc | Mode::Codegen)
{
// keep in sync with `bootstrap/lib.rs:Build::rustc_features`
// `cfg` option for rustc, `features` option for cargo, for conditional compilation
rustflags.arg("--cfg=parallel_compiler");
rustdocflags.arg("--cfg=parallel_compiler");
}
// set rustc args passed from command line
let rustc_args =
self.config.cmd.rustc_args().iter().map(|s| s.to_string()).collect::<Vec<_>>();
@ -2309,7 +2357,7 @@ impl Cargo {
mode: Mode,
source_type: SourceType,
target: TargetSelection,
cmd: &str,
cmd: &str, // FIXME make this properly typed
) -> Cargo {
let mut cargo = builder.cargo(compiler, mode, source_type, target, cmd);
cargo.configure_linker(builder);
@ -2323,7 +2371,7 @@ impl Cargo {
mode: Mode,
source_type: SourceType,
target: TargetSelection,
cmd: &str,
cmd: &str, // FIXME make this properly typed
) -> Cargo {
builder.cargo(compiler, mode, source_type, target, cmd)
}

View File

@ -250,6 +250,8 @@ pub enum Mode {
/// directory. This is for miscellaneous sets of tools that are built
/// using the bootstrap stage0 compiler in its entirety (target libraries
/// and all). Typically these tools compile with stable Rust.
///
/// Only works for stage 0.
ToolBootstrap,
/// Build a tool which uses the locally built std, placing output in the

View File

@ -2,6 +2,7 @@
# ignore-tidy-linelength
set -eu
set -x # so one can see where we are in the script
X_PY="$1"

View File

@ -89,8 +89,12 @@ pub fn phase_cargo_miri(mut args: impl Iterator<Item = String>) {
let verbose = num_arg_flag("-v");
// Determine the involved architectures.
let rustc_version = VersionMeta::for_command(miri_for_host())
.expect("failed to determine underlying rustc version of Miri");
let rustc_version = VersionMeta::for_command(miri_for_host()).unwrap_or_else(|err| {
panic!(
"failed to determine underlying rustc version of Miri ({:?}):\n{err:?}",
miri_for_host()
)
});
let host = &rustc_version.host;
let target = get_arg_flag_value("--target");
let target = target.as_ref().unwrap_or(host);
@ -222,7 +226,7 @@ pub fn phase_cargo_miri(mut args: impl Iterator<Item = String>) {
}
// Run cargo.
debug_cmd("[cargo-miri miri]", verbose, &cmd);
debug_cmd("[cargo-miri cargo]", verbose, &cmd);
exec(cmd)
}

View File

@ -90,13 +90,13 @@ pub fn setup(
let cargo_cmd = {
let mut command = cargo();
// Use Miri as rustc to build a libstd compatible with us (and use the right flags).
// We set ourselves (`cargo-miri`) instead of Miri directly to be able to patch the flags
// for `libpanic_abort` (usually this is done by bootstrap but we have to do it ourselves).
// The `MIRI_CALLED_FROM_SETUP` will mean we dispatch to `phase_setup_rustc`.
// However, when we are running in bootstrap, we cannot just overwrite `RUSTC`,
// because we still need bootstrap to distinguish between host and target crates.
// In that case we overwrite `RUSTC_REAL` instead which determines the rustc used
// for target crates.
// We set ourselves (`cargo-miri`) instead of Miri directly to be able to patch the flags
// for `libpanic_abort` (usually this is done by bootstrap but we have to do it ourselves).
// The `MIRI_CALLED_FROM_SETUP` will mean we dispatch to `phase_setup_rustc`.
let cargo_miri_path = std::env::current_exe().expect("current executable path invalid");
if env::var_os("RUSTC_STAGE").is_some() {
assert!(env::var_os("RUSTC").is_some());