Three tasks have been implemented here.

Add a new `download-ci-llvm = if-unchange` option and enable if by
default for `profile = codegen`.

Include all build artifacts by traversing the llvm-project build output,
Keep the downloadable llvm the same state as if you have just run a full
source build.

After selecting the codegen profile during ./x.py setup, the submodule
will be automatically downloaded.
This commit is contained in:
LuuuX 2023-10-18 09:55:18 +08:00
parent 4e0fb98a5c
commit 1b8dee19e8
7 changed files with 120 additions and 32 deletions

View File

@ -30,7 +30,7 @@
#
# If `change-id` does not match the version that is currently running,
# `x.py` will prompt you to update it and check the related PR for more details.
change-id = 117435
change-id = 116881
# =============================================================================
# Tweaking how LLVM is compiled
@ -42,11 +42,15 @@ change-id = 117435
# Unless you're developing for a target where Rust CI doesn't build a compiler
# toolchain or changing LLVM locally, you probably want to leave this enabled.
#
# All tier 1 targets are currently supported; set this to `"if-available"` if
# you are not sure whether you're on a tier 1 target.
# Set this to `"if-available"` if you are not sure whether you're on a tier 1
# target. All tier 1 targets are currently supported;
#
# We also currently only support this when building LLVM for the build triple.
#
# Set this to `"if-unchanged"` to only download if the llvm-project have not
# been modified. (If there are no changes or if built from tarball source,
# the logic is the same as "if-available")
#
# Note that many of the LLVM options are not currently supported for
# downloading. Currently only the "assertions" option can be toggled.
#download-ci-llvm = if rust.channel == "dev" { "if-available" } else { false }

View File

@ -10,7 +10,7 @@ assertions = true
# enable warnings during the llvm compilation
enable-warnings = true
# build llvm from source
download-ci-llvm = false
download-ci-llvm = "if-unchanged"
[rust]
# This enables `RUSTC_LOG=debug`, avoiding confusing situations

View File

@ -1,4 +1,4 @@
Change this file to make users of the `download-ci-llvm` configuration download
a new version of LLVM from CI, even if the LLVM submodule hasnt changed.
Last change is for: https://github.com/rust-lang/rust/pull/113996
Last change is for: https://github.com/rust-lang/rust/pull/116881

View File

@ -2219,23 +2219,19 @@ impl Step for RustDev {
builder.ensure(crate::core::build_steps::llvm::Lld { target });
let src_bindir = builder.llvm_out(target).join("bin");
// If updating this list, you likely want to change
// If updating this, you likely want to change
// src/bootstrap/download-ci-llvm-stamp as well, otherwise local users
// will not pick up the extra file until LLVM gets bumped.
for bin in &[
"llvm-config",
"llvm-ar",
"llvm-objdump",
"llvm-profdata",
"llvm-bcanalyzer",
"llvm-cov",
"llvm-dwp",
"llvm-nm",
"llvm-dwarfdump",
"llvm-dis",
"llvm-tblgen",
] {
tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755);
// We should include all the build artifacts obtained from a source build,
// so that you can use the downloadable LLVM as if youve just run a full source build.
if src_bindir.exists() {
for entry in walkdir::WalkDir::new(&src_bindir) {
let entry = t!(entry);
if entry.file_type().is_file() && !entry.path_is_symlink() {
let name = entry.file_name().to_str().unwrap();
tarball.add_file(src_bindir.join(name), "bin", 0o755);
}
}
}
// We don't build LLD on some platforms, so only add it if it exists

View File

@ -147,6 +147,15 @@ impl Step for Profile {
}
fn run(self, builder: &Builder<'_>) {
// During ./x.py setup once you select the codegen profile.
// The submodule will be downloaded. It does not work in the
// tarball case since they don't include Git and submodules
// are already included.
if !builder.rust_info().is_from_tarball() {
if self == Profile::Codegen {
builder.update_submodule(&Path::new("src/llvm-project"));
}
}
setup(&builder.build.config, self)
}
}

View File

@ -19,6 +19,7 @@ use std::process::Command;
use std::str::FromStr;
use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX;
use crate::core::build_steps::llvm;
use crate::core::config::flags::{Color, Flags, Warnings};
use crate::utils::cache::{Interned, INTERNER};
use crate::utils::channel::{self, GitInfo};
@ -1530,17 +1531,7 @@ impl Config {
config.llvm_build_config = llvm.build_config.clone().unwrap_or(Default::default());
let asserts = llvm_assertions.unwrap_or(false);
config.llvm_from_ci = match llvm.download_ci_llvm {
Some(StringOrBool::String(s)) => {
assert_eq!(s, "if-available", "unknown option `{s}` for download-ci-llvm");
crate::core::build_steps::llvm::is_ci_llvm_available(&config, asserts)
}
Some(StringOrBool::Bool(b)) => b,
None => {
config.channel == "dev"
&& crate::core::build_steps::llvm::is_ci_llvm_available(&config, asserts)
}
};
config.llvm_from_ci = config.parse_download_ci_llvm(llvm.download_ci_llvm, asserts);
if config.llvm_from_ci {
// None of the LLVM options, except assertions, are supported
@ -2104,6 +2095,94 @@ impl Config {
Some(commit.to_string())
}
fn parse_download_ci_llvm(
&self,
download_ci_llvm: Option<StringOrBool>,
asserts: bool,
) -> bool {
match download_ci_llvm {
None => self.channel == "dev" && llvm::is_ci_llvm_available(&self, asserts),
Some(StringOrBool::Bool(b)) => b,
Some(StringOrBool::String(s)) if s == "if-available" => {
llvm::is_ci_llvm_available(&self, asserts)
}
Some(StringOrBool::String(s)) if s == "if-unchanged" => {
// Git is needed to track modifications here, but tarball source is not available.
// If not modified here or built through tarball source, we maintain consistency
// with '"if available"'.
if !self.rust_info.is_from_tarball()
&& self
.last_modified_commit(&["src/llvm-project"], "download-ci-llvm", true)
.is_none()
{
// there are some untracked changes in the the given paths.
false
} else {
llvm::is_ci_llvm_available(&self, asserts)
}
}
Some(StringOrBool::String(other)) => {
panic!("unrecognized option for download-ci-llvm: {:?}", other)
}
}
}
/// Returns the last commit in which any of `modified_paths` were changed,
/// or `None` if there are untracked changes in the working directory and `if_unchanged` is true.
pub fn last_modified_commit(
&self,
modified_paths: &[&str],
option_name: &str,
if_unchanged: bool,
) -> Option<String> {
// Handle running from a directory other than the top level
let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"]));
let top_level = top_level.trim_end();
// Look for a version to compare to based on the current commit.
// Only commits merged by bors will have CI artifacts.
let merge_base = output(
self.git()
.arg("rev-list")
.arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email))
.args(&["-n1", "--first-parent", "HEAD"]),
);
let commit = merge_base.trim_end();
if commit.is_empty() {
println!("error: could not find commit hash for downloading components from CI");
println!("help: maybe your repository history is too shallow?");
println!("help: consider disabling `{option_name}`");
println!("help: or fetch enough history to include one upstream commit");
crate::exit!(1);
}
// Warn if there were changes to the compiler or standard library since the ancestor commit.
let mut git = self.git();
git.args(&["diff-index", "--quiet", &commit, "--"]);
for path in modified_paths {
git.arg(format!("{top_level}/{path}"));
}
let has_changes = !t!(git.status()).success();
if has_changes {
if if_unchanged {
if self.verbose > 0 {
println!(
"warning: saw changes to one of {modified_paths:?} since {commit}; \
ignoring `{option_name}`"
);
}
return None;
}
println!(
"warning: `{option_name}` is enabled, but there are changes to one of {modified_paths:?}"
);
}
Some(commit.to_string())
}
}
fn set<T>(field: &mut T, val: Option<T>) {

View File

@ -77,7 +77,7 @@ const LLD_FILE_NAMES: &[&str] = &["ld.lld", "ld64.lld", "lld-link", "wasm-ld"];
///
/// If you make any major changes (such as adding new values or changing default values), please
/// ensure that the associated PR ID is added to the end of this list.
pub const CONFIG_CHANGE_HISTORY: &[usize] = &[115898, 116998, 117435];
pub const CONFIG_CHANGE_HISTORY: &[usize] = &[115898, 116998, 117435, 116881];
/// Extra --check-cfg to add when building
/// (Mode restriction, config name, config values (if any))