dnrops
/
rust
mirror of https://github.com/rust-lang/rust.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rollup merge of #97828 - ferrocene:pa-config-artifacts, r=jyn514 

Allow configuring where artifacts are downloaded from

Bootstrap has support for downloading prebuilt LLVM and rustc artifacts to speed up local builds, but that currently works only for users working on `rust-lang/rust`. Forks of the repository (for example Ferrocene) might have different URLs to download artifacts from, or might use a different email address on merge commits, breaking both LLVM and rustc artifact downloads.

This PR refactors bootstrap to load the download URLs and other constants from `src/stage0.json`, allowing downstream forks to tweak those values. It also future-proofs the download code to easily allow forks to add their own custom protocols (like `s3://`).

This PR is best reviewed commit-by-commit.
This commit is contained in:
Yuki Okushi 2022-06-18 10:03:23 +09:00 committed by GitHub
parent fc8027f188 d3b15329f9
commit d09a568b94
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 132 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/bootstrap/bootstrap.py
View File

@ -1043,7 +1043,7 @@ def bootstrap(help_triggered):
build.checksums_sha256 = data["checksums_sha256"]
build.stage0_compiler = Stage0Toolchain(data["compiler"])
build.set_dist_environment(data["dist_server"])
build.set_dist_environment(data["config"]["dist_server"])
build.build = args.build or build.build_triple()

21
src/bootstrap/builder.rs
View File

@ -870,20 +870,23 @@ impl<'a> Builder<'a> {
self.try_run(patchelf.arg(fname));
}
pub(crate) fn download_component(
&self,
base: &str,
url: &str,
dest_path: &Path,
help_on_error: &str,
) {
pub(crate) fn download_component(&self, url: &str, dest_path: &Path, help_on_error: &str) {
// Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
self.download_with_retries(&tempfile, &format!("{}/{}", base, url), help_on_error);
// While bootstrap itself only supports http and https downloads, downstream forks might
// need to download components from other protocols. The match allows them adding more
// protocols without worrying about merge conficts if we change the HTTP implementation.
match url.split_once("://").map(|(proto, _)| proto) {
Some("http") | Some("https") => {
self.download_http_with_retries(&tempfile, url, help_on_error)
}
Some(other) => panic!("unsupported protocol {other} in {url}"),
None => panic!("no protocol in {url}"),
}
t!(std::fs::rename(&tempfile, dest_path));
}
fn download_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) {
println!("downloading {}", url);
// Try curl. If that fails and we are on windows, fallback to PowerShell.
let mut curl = Command::new("curl");

63
src/bootstrap/config.rs
View File

@ -20,7 +20,6 @@ use crate::channel::GitInfo;
pub use crate::flags::Subcommand;
use crate::flags::{Color, Flags};
use crate::util::{exe, output, program_out_of_date, t};
use crate::RustfmtMetadata;
use once_cell::sync::OnceCell;
use serde::{Deserialize, Deserializer};
@ -73,6 +72,7 @@ pub struct Config {
pub test_compare_mode: bool,
pub color: Color,
pub patch_binaries_for_nix: bool,
pub stage0_metadata: Stage0Metadata,
pub on_fail: Option<String>,
pub stage: u32,
@ -212,6 +212,28 @@ pub struct Config {
pub out: PathBuf,
}
#[derive(Default, Deserialize)]
#[cfg_attr(test, derive(Clone))]
pub struct Stage0Metadata {
pub config: Stage0Config,
pub checksums_sha256: HashMap<String, String>,
pub rustfmt: Option<RustfmtMetadata>,
}
#[derive(Default, Deserialize)]
#[cfg_attr(test, derive(Clone))]
pub struct Stage0Config {
pub dist_server: String,
pub artifacts_server: String,
pub artifacts_with_llvm_assertions_server: String,
pub git_merge_commit_email: String,
}
#[derive(Default, Deserialize)]
#[cfg_attr(test, derive(Clone))]
pub struct RustfmtMetadata {
pub date: String,
pub version: String,
}
#[derive(Clone, Debug)]
pub enum RustfmtState {
SystemToolchain(PathBuf),
@ -776,6 +798,9 @@ impl Config {
config.llvm_profile_use = flags.llvm_profile_use;
config.llvm_profile_generate = flags.llvm_profile_generate;
let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json")));
config.stage0_metadata = t!(serde_json::from_slice::<Stage0Metadata>(&stage0_json));
#[cfg(test)]
let get_toml = |_| TomlConfig::default();
#[cfg(not(test))]
@ -1103,8 +1128,11 @@ impl Config {
config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
config.download_rustc_commit =
download_ci_rustc_commit(rust.download_rustc, config.verbose > 0);
config.download_rustc_commit = download_ci_rustc_commit(
&config.stage0_metadata,
rust.download_rustc,
config.verbose > 0,
);
} else {
config.rust_profile_use = flags.rust_profile_use;
config.rust_profile_generate = flags.rust_profile_generate;
@ -1424,7 +1452,11 @@ fn threads_from_config(v: u32) -> u32 {
}
/// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool) -> Option<String> {
fn download_ci_rustc_commit(
stage0_metadata: &Stage0Metadata,
download_rustc: Option<StringOrBool>,
verbose: bool,
) -> Option<String> {
// If `download-rustc` is not set, default to rebuilding.
let if_unchanged = match download_rustc {
None | Some(StringOrBool::Bool(false)) => return None,
@ -1443,13 +1475,12 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool)
// Look for a version to compare to based on the current commit.
// Only commits merged by bors will have CI artifacts.
let merge_base = output(Command::new("git").args(&[
"rev-list",
"--author=bors@rust-lang.org",
"-n1",
"--first-parent",
"HEAD",
]));
let merge_base = output(
Command::new("git")
.arg("rev-list")
.arg(format!("--author={}", stage0_metadata.config.git_merge_commit_email))
.args(&["-n1", "--first-parent", "HEAD"]),
);
let commit = merge_base.trim_end();
if commit.is_empty() {
println!("error: could not find commit hash for downloading rustc");
@ -1484,7 +1515,7 @@ fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool)
}
fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
let RustfmtMetadata { date, version } = builder.stage0_metadata.rustfmt.as_ref()?;
let RustfmtMetadata { date, version } = builder.config.stage0_metadata.rustfmt.as_ref()?;
let channel = format!("{version}-{date}");
let host = builder.config.build;
@ -1568,13 +1599,13 @@ fn download_component(
let tarball = cache_dir.join(&filename);
let (base_url, url, should_verify) = match mode {
DownloadSource::CI => (
"https://ci-artifacts.rust-lang.org/rustc-builds".to_string(),
builder.config.stage0_metadata.config.artifacts_server.clone(),
format!("{key}/{filename}"),
false,
),
DownloadSource::Dist => {
let dist_server = env::var("RUSTUP_DIST_SERVER")
.unwrap_or(builder.stage0_metadata.dist_server.to_string());
.unwrap_or(builder.config.stage0_metadata.config.dist_server.to_string());
// NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json
(dist_server, format!("dist/{key}/{filename}"), true)
}
@ -1590,7 +1621,7 @@ fn download_component(
target at this time, see https://doc.rust-lang.org/nightly\
/rustc/platform-support.html for more information."
);
let sha256 = builder.stage0_metadata.checksums_sha256.get(&url).expect(&error);
let sha256 = builder.config.stage0_metadata.checksums_sha256.get(&url).expect(&error);
if tarball.exists() {
if builder.verify(&tarball, sha256) {
builder.unpack(&tarball, &bin_root, prefix);
@ -1610,7 +1641,7 @@ fn download_component(
None
};
builder.download_component(&base_url, &url, &tarball, "");
builder.download_component(&format!("{base_url}/{url}"), &tarball, "");
if let Some(sha256) = checksum {
if !builder.verify(&tarball, sha256) {
panic!("failed to verify {}", tarball.display());

19
src/bootstrap/lib.rs
View File

@ -118,7 +118,6 @@ use std::os::windows::fs::symlink_file;
use filetime::FileTime;
use once_cell::sync::OnceCell;
use serde::Deserialize;
use crate::builder::Kind;
use crate::config::{LlvmLibunwind, TargetSelection};
@ -294,8 +293,6 @@ pub struct Build {
hosts: Vec<TargetSelection>,
targets: Vec<TargetSelection>,
// Stage 0 (downloaded) compiler, lld and cargo or their local rust equivalents
stage0_metadata: Stage0Metadata,
initial_rustc: PathBuf,
initial_cargo: PathBuf,
initial_lld: PathBuf,
@ -322,18 +319,6 @@ pub struct Build {
metrics: metrics::BuildMetrics,
}
#[derive(Deserialize)]
struct Stage0Metadata {
dist_server: String,
checksums_sha256: HashMap<String, String>,
rustfmt: Option<RustfmtMetadata>,
}
#[derive(Deserialize)]
struct RustfmtMetadata {
date: String,
version: String,
}
#[derive(Debug)]
struct Crate {
name: Interned<String>,
@ -482,11 +467,7 @@ impl Build {
bootstrap_out
};
let stage0_json = t!(std::fs::read_to_string(&src.join("src").join("stage0.json")));
let stage0_metadata = t!(serde_json::from_str::<Stage0Metadata>(&stage0_json));
let mut build = Build {
stage0_metadata,
initial_rustc: config.initial_rustc.clone(),
initial_cargo: config.initial_cargo.clone(),
initial_lld,

15
src/bootstrap/native.rs
View File

@ -121,7 +121,7 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
let mut rev_list = Command::new("git");
rev_list.args(&[
PathBuf::from("rev-list"),
"--author=bors@rust-lang.org".into(),
format!("--author={}", builder.config.stage0_metadata.config.git_merge_commit_email).into(),
"-n1".into(),
"--first-parent".into(),
"HEAD".into(),
@ -170,11 +170,10 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
if !rustc_cache.exists() {
t!(fs::create_dir_all(&rustc_cache));
}
let base = "https://ci-artifacts.rust-lang.org";
let url = if llvm_assertions {
format!("rustc-builds-alt/{}", llvm_sha)
let base = if llvm_assertions {
&builder.config.stage0_metadata.config.artifacts_with_llvm_assertions_server
} else {
format!("rustc-builds/{}", llvm_sha)
&builder.config.stage0_metadata.config.artifacts_server
};
let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple);
let tarball = rustc_cache.join(&filename);
@ -187,7 +186,11 @@ help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in
[llvm]
download-ci-llvm = false
";
builder.download_component(base, &format!("{}/{}", url, filename), &tarball, help_on_error);
builder.download_component(
&format!("{base}/{llvm_sha}/{filename}"),
&tarball,
help_on_error,
);
}
let llvm_root = builder.config.ci_llvm_root();
builder.unpack(&tarball, &llvm_root, "rust-dev");

16
src/stage0.json
View File

@ -1,6 +1,20 @@
{
"__comment": "Generated by `./x.py run src/tools/bump-stage0`. Run that command again to update the bootstrap compiler.",
"config": {
"dist_server": "https://static.rust-lang.org",
"artifacts_server": "https://ci-artifacts.rust-lang.org/rustc-builds",
"artifacts_with_llvm_assertions_server": "https://ci-artifacts.rust-lang.org/rustc-builds-alt",
"git_merge_commit_email": "bors@rust-lang.org"
},
"__comments": [
"The configuration above this comment is editable, and can be changed",
"by forks of the repository if they have alternate values.",
"",
"The section below is generated by `./x.py run src/tools/bump-stage0`,",
"run that command again to update the bootstrap compiler.",
"",
"All changes below this comment will be overridden the next time the",
"tool is executed."
],
"compiler": {
"date": "2022-05-20",
"version": "beta"

67
src/tools/bump-stage0/src/main.rs
View File

@ -4,11 +4,14 @@ use indexmap::IndexMap;
use std::collections::HashMap;
use std::convert::TryInto;
const DIST_SERVER: &str = "https://static.rust-lang.org";
const PATH: &str = "src/stage0.json";
const COMPILER_COMPONENTS: &[&str] = &["rustc", "rust-std", "cargo"];
const RUSTFMT_COMPONENTS: &[&str] = &["rustfmt-preview"];
struct Tool {
config: Config,
comments: Vec<String>,
channel: Channel,
version: [u16; 3],
checksums: IndexMap<String, String>,
@ -32,18 +35,23 @@ impl Tool {
.try_into()
.map_err(|_| anyhow::anyhow!("failed to parse version"))?;
Ok(Self { channel, version, checksums: IndexMap::new() })
let existing: Stage0 = serde_json::from_slice(&std::fs::read(PATH)?)?;
Ok(Self {
channel,
version,
config: existing.config,
comments: existing.comments,
checksums: IndexMap::new(),
})
}
fn update_json(mut self) -> Result<(), Error> {
std::fs::write(
"src/stage0.json",
PATH,
format!(
"{}\n",
serde_json::to_string_pretty(&Stage0 {
comment: "Generated by `./x.py run src/tools/bump-stage0`. \
Run that command again to update the bootstrap compiler.",
dist_server: DIST_SERVER.into(),
compiler: self.detect_compiler()?,
rustfmt: self.detect_rustfmt()?,
checksums_sha256: {
@ -51,7 +59,9 @@ impl Tool {
// are added while filling the other struct fields just above this block.
self.checksums.sort_keys();
self.checksums
}
},
config: self.config,
comments: self.comments,
})?
),
)?;
@ -74,7 +84,7 @@ impl Tool {
Channel::Nightly => "beta".to_string(),
};
let manifest = fetch_manifest(&channel)?;
let manifest = fetch_manifest(&self.config, &channel)?;
self.collect_checksums(&manifest, COMPILER_COMPONENTS)?;
Ok(Stage0Toolchain {
date: manifest.date,
@ -100,13 +110,13 @@ impl Tool {
return Ok(None);
}
let manifest = fetch_manifest("nightly")?;
let manifest = fetch_manifest(&self.config, "nightly")?;
self.collect_checksums(&manifest, RUSTFMT_COMPONENTS)?;
Ok(Some(Stage0Toolchain { date: manifest.date, version: "nightly".into() }))
}
fn collect_checksums(&mut self, manifest: &Manifest, components: &[&str]) -> Result<(), Error> {
let prefix = format!("{}/", DIST_SERVER);
let prefix = format!("{}/", self.config.dist_server);
for component in components {
let pkg = manifest
.pkg
@ -136,10 +146,10 @@ fn main() -> Result<(), Error> {
Ok(())
}
fn fetch_manifest(channel: &str) -> Result<Manifest, Error> {
fn fetch_manifest(config: &Config, channel: &str) -> Result<Manifest, Error> {
Ok(toml::from_slice(&http_get(&format!(
"{}/dist/channel-rust-{}.toml",
DIST_SERVER, channel
config.dist_server, channel
))?)?)
}
@ -166,35 +176,52 @@ enum Channel {
Nightly,
}
#[derive(Debug, serde::Serialize)]
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct Stage0 {
#[serde(rename = "__comment")]
comment: &'static str,
dist_server: String,
config: Config,
// Comments are explicitly below the config, do not move them above.
//
// Downstream forks of the compiler codebase can change the configuration values defined above,
// but doing so would risk merge conflicts whenever they import new changes that include a
// bootstrap compiler bump.
//
// To lessen the pain, a big block of comments is placed between the configuration and the
// auto-generated parts of the file, preventing git diffs of the config to include parts of the
// auto-generated content and vice versa. This should prevent merge conflicts.
#[serde(rename = "__comments")]
comments: Vec<String>,
compiler: Stage0Toolchain,
rustfmt: Option<Stage0Toolchain>,
checksums_sha256: IndexMap<String, String>,
}
#[derive(Debug, serde::Serialize)]
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct Config {
dist_server: String,
artifacts_server: String,
artifacts_with_llvm_assertions_server: String,
git_merge_commit_email: String,
}
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct Stage0Toolchain {
date: String,
version: String,
}
#[derive(Debug, serde::Deserialize)]
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct Manifest {
date: String,
pkg: HashMap<String, ManifestPackage>,
}
#[derive(Debug, serde::Deserialize)]
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct ManifestPackage {
version: String,
target: HashMap<String, ManifestTargetPackage>,
}
#[derive(Debug, serde::Deserialize)]
#[derive(Debug, serde::Serialize, serde::Deserialize)]
struct ManifestTargetPackage {
url: Option<String>,
hash: Option<String>,