mirror of https://github.com/rust-lang/rust.git
Auto merge of #96687 - jyn514:download-rustc, r=Mark-Simulacrum
Move download-rustc from python to rustbuild - Remove download-rustc handling from bootstrap.py - Allow a custom `pattern` in `builder.unpack()` - Only download rustc once another part of bootstrap depends on it. This is somewhat necessary since the download functions rely on having a full `Builder`, which isn't available until after config parsing finishes. Helps with https://github.com/rust-lang/rust/issues/94829.
This commit is contained in:
commit
303d916867
|
@ -444,9 +444,8 @@ class RustBuild(object):
|
||||||
self.verbose = False
|
self.verbose = False
|
||||||
self.git_version = None
|
self.git_version = None
|
||||||
self.nix_deps_dir = None
|
self.nix_deps_dir = None
|
||||||
self.rustc_commit = None
|
|
||||||
|
|
||||||
def download_toolchain(self, stage0=True, rustc_channel=None):
|
def download_toolchain(self):
|
||||||
"""Fetch the build system for Rust, written in Rust
|
"""Fetch the build system for Rust, written in Rust
|
||||||
|
|
||||||
This method will build a cache directory, then it will fetch the
|
This method will build a cache directory, then it will fetch the
|
||||||
|
@ -456,37 +455,27 @@ class RustBuild(object):
|
||||||
Each downloaded tarball is extracted, after that, the script
|
Each downloaded tarball is extracted, after that, the script
|
||||||
will move all the content to the right place.
|
will move all the content to the right place.
|
||||||
"""
|
"""
|
||||||
if rustc_channel is None:
|
rustc_channel = self.stage0_compiler.version
|
||||||
rustc_channel = self.stage0_compiler.version
|
bin_root = self.bin_root()
|
||||||
bin_root = self.bin_root(stage0)
|
|
||||||
|
|
||||||
key = self.stage0_compiler.date
|
key = self.stage0_compiler.date
|
||||||
if not stage0:
|
if self.rustc().startswith(bin_root) and \
|
||||||
key += str(self.rustc_commit)
|
(not os.path.exists(self.rustc()) or
|
||||||
if self.rustc(stage0).startswith(bin_root) and \
|
self.program_out_of_date(self.rustc_stamp(), key)):
|
||||||
(not os.path.exists(self.rustc(stage0)) or
|
|
||||||
self.program_out_of_date(self.rustc_stamp(stage0), key)):
|
|
||||||
if os.path.exists(bin_root):
|
if os.path.exists(bin_root):
|
||||||
shutil.rmtree(bin_root)
|
shutil.rmtree(bin_root)
|
||||||
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
|
tarball_suffix = '.tar.xz' if support_xz() else '.tar.gz'
|
||||||
filename = "rust-std-{}-{}{}".format(
|
filename = "rust-std-{}-{}{}".format(
|
||||||
rustc_channel, self.build, tarball_suffix)
|
rustc_channel, self.build, tarball_suffix)
|
||||||
pattern = "rust-std-{}".format(self.build)
|
pattern = "rust-std-{}".format(self.build)
|
||||||
self._download_component_helper(filename, pattern, tarball_suffix, stage0)
|
self._download_component_helper(filename, pattern, tarball_suffix)
|
||||||
filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
|
filename = "rustc-{}-{}{}".format(rustc_channel, self.build,
|
||||||
tarball_suffix)
|
tarball_suffix)
|
||||||
self._download_component_helper(filename, "rustc", tarball_suffix, stage0)
|
self._download_component_helper(filename, "rustc", tarball_suffix)
|
||||||
# download-rustc doesn't need its own cargo, it can just use beta's.
|
filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
|
||||||
if stage0:
|
tarball_suffix)
|
||||||
filename = "cargo-{}-{}{}".format(rustc_channel, self.build,
|
self._download_component_helper(filename, "cargo", tarball_suffix)
|
||||||
tarball_suffix)
|
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
|
||||||
self._download_component_helper(filename, "cargo", tarball_suffix)
|
|
||||||
self.fix_bin_or_dylib("{}/bin/cargo".format(bin_root))
|
|
||||||
else:
|
|
||||||
filename = "rustc-dev-{}-{}{}".format(rustc_channel, self.build, tarball_suffix)
|
|
||||||
self._download_component_helper(
|
|
||||||
filename, "rustc-dev", tarball_suffix, stage0
|
|
||||||
)
|
|
||||||
|
|
||||||
self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
|
self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
|
||||||
self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
|
self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
|
||||||
|
@ -494,7 +483,7 @@ class RustBuild(object):
|
||||||
for lib in os.listdir(lib_dir):
|
for lib in os.listdir(lib_dir):
|
||||||
if lib.endswith(".so"):
|
if lib.endswith(".so"):
|
||||||
self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
|
self.fix_bin_or_dylib(os.path.join(lib_dir, lib))
|
||||||
with output(self.rustc_stamp(stage0)) as rust_stamp:
|
with output(self.rustc_stamp()) as rust_stamp:
|
||||||
rust_stamp.write(key)
|
rust_stamp.write(key)
|
||||||
|
|
||||||
if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
|
if self.rustfmt() and self.rustfmt().startswith(bin_root) and (
|
||||||
|
@ -518,24 +507,17 @@ class RustBuild(object):
|
||||||
rustfmt_stamp.write(self.stage0_rustfmt.channel())
|
rustfmt_stamp.write(self.stage0_rustfmt.channel())
|
||||||
|
|
||||||
def _download_component_helper(
|
def _download_component_helper(
|
||||||
self, filename, pattern, tarball_suffix, stage0=True, key=None
|
self, filename, pattern, tarball_suffix, key=None
|
||||||
):
|
):
|
||||||
if key is None:
|
if key is None:
|
||||||
if stage0:
|
key = self.stage0_compiler.date
|
||||||
key = self.stage0_compiler.date
|
|
||||||
else:
|
|
||||||
key = self.rustc_commit
|
|
||||||
cache_dst = os.path.join(self.build_dir, "cache")
|
cache_dst = os.path.join(self.build_dir, "cache")
|
||||||
rustc_cache = os.path.join(cache_dst, key)
|
rustc_cache = os.path.join(cache_dst, key)
|
||||||
if not os.path.exists(rustc_cache):
|
if not os.path.exists(rustc_cache):
|
||||||
os.makedirs(rustc_cache)
|
os.makedirs(rustc_cache)
|
||||||
|
|
||||||
if stage0:
|
base = self._download_url
|
||||||
base = self._download_url
|
url = "dist/{}".format(key)
|
||||||
url = "dist/{}".format(key)
|
|
||||||
else:
|
|
||||||
base = "https://ci-artifacts.rust-lang.org"
|
|
||||||
url = "rustc-builds/{}".format(self.rustc_commit)
|
|
||||||
tarball = os.path.join(rustc_cache, filename)
|
tarball = os.path.join(rustc_cache, filename)
|
||||||
if not os.path.exists(tarball):
|
if not os.path.exists(tarball):
|
||||||
get(
|
get(
|
||||||
|
@ -544,9 +526,9 @@ class RustBuild(object):
|
||||||
tarball,
|
tarball,
|
||||||
self.checksums_sha256,
|
self.checksums_sha256,
|
||||||
verbose=self.verbose,
|
verbose=self.verbose,
|
||||||
do_verify=stage0,
|
do_verify=True,
|
||||||
)
|
)
|
||||||
unpack(tarball, tarball_suffix, self.bin_root(stage0), match=pattern, verbose=self.verbose)
|
unpack(tarball, tarball_suffix, self.bin_root(), match=pattern, verbose=self.verbose)
|
||||||
|
|
||||||
def fix_bin_or_dylib(self, fname):
|
def fix_bin_or_dylib(self, fname):
|
||||||
"""Modifies the interpreter section of 'fname' to fix the dynamic linker,
|
"""Modifies the interpreter section of 'fname' to fix the dynamic linker,
|
||||||
|
@ -644,62 +626,15 @@ class RustBuild(object):
|
||||||
print("warning: failed to call patchelf:", reason)
|
print("warning: failed to call patchelf:", reason)
|
||||||
return
|
return
|
||||||
|
|
||||||
# If `download-rustc` is set, download the most recent commit with CI artifacts
|
def rustc_stamp(self):
|
||||||
def maybe_download_ci_toolchain(self):
|
|
||||||
# If `download-rustc` is not set, default to rebuilding.
|
|
||||||
download_rustc = self.get_toml("download-rustc", section="rust")
|
|
||||||
if download_rustc is None or download_rustc == "false":
|
|
||||||
return None
|
|
||||||
assert download_rustc == "true" or download_rustc == "if-unchanged", download_rustc
|
|
||||||
|
|
||||||
# Handle running from a directory other than the top level
|
|
||||||
rev_parse = ["git", "rev-parse", "--show-toplevel"]
|
|
||||||
top_level = subprocess.check_output(rev_parse, universal_newlines=True).strip()
|
|
||||||
compiler = "{}/compiler/".format(top_level)
|
|
||||||
library = "{}/library/".format(top_level)
|
|
||||||
|
|
||||||
# Look for a version to compare to based on the current commit.
|
|
||||||
# Only commits merged by bors will have CI artifacts.
|
|
||||||
merge_base = [
|
|
||||||
"git", "rev-list", "--author=bors@rust-lang.org", "-n1",
|
|
||||||
"--first-parent", "HEAD"
|
|
||||||
]
|
|
||||||
commit = subprocess.check_output(merge_base, universal_newlines=True).strip()
|
|
||||||
if not commit:
|
|
||||||
print("error: could not find commit hash for downloading rustc")
|
|
||||||
print("help: maybe your repository history is too shallow?")
|
|
||||||
print("help: consider disabling `download-rustc`")
|
|
||||||
print("help: or fetch enough history to include one upstream commit")
|
|
||||||
exit(1)
|
|
||||||
|
|
||||||
# Warn if there were changes to the compiler or standard library since the ancestor commit.
|
|
||||||
status = subprocess.call(["git", "diff-index", "--quiet", commit, "--", compiler, library])
|
|
||||||
if status != 0:
|
|
||||||
if download_rustc == "if-unchanged":
|
|
||||||
if self.verbose:
|
|
||||||
print("warning: saw changes to compiler/ or library/ since {}; " \
|
|
||||||
"ignoring `download-rustc`".format(commit))
|
|
||||||
return None
|
|
||||||
print("warning: `download-rustc` is enabled, but there are changes to " \
|
|
||||||
"compiler/ or library/")
|
|
||||||
|
|
||||||
if self.verbose:
|
|
||||||
print("using downloaded stage2 artifacts from CI (commit {})".format(commit))
|
|
||||||
self.rustc_commit = commit
|
|
||||||
# FIXME: support downloading artifacts from the beta channel
|
|
||||||
self.download_toolchain(False, "nightly")
|
|
||||||
|
|
||||||
def rustc_stamp(self, stage0):
|
|
||||||
"""Return the path for .rustc-stamp at the given stage
|
"""Return the path for .rustc-stamp at the given stage
|
||||||
|
|
||||||
>>> rb = RustBuild()
|
>>> rb = RustBuild()
|
||||||
>>> rb.build_dir = "build"
|
>>> rb.build_dir = "build"
|
||||||
>>> rb.rustc_stamp(True) == os.path.join("build", "stage0", ".rustc-stamp")
|
>>> rb.rustc_stamp() == os.path.join("build", "stage0", ".rustc-stamp")
|
||||||
True
|
|
||||||
>>> rb.rustc_stamp(False) == os.path.join("build", "ci-rustc", ".rustc-stamp")
|
|
||||||
True
|
True
|
||||||
"""
|
"""
|
||||||
return os.path.join(self.bin_root(stage0), '.rustc-stamp')
|
return os.path.join(self.bin_root(), '.rustc-stamp')
|
||||||
|
|
||||||
def rustfmt_stamp(self):
|
def rustfmt_stamp(self):
|
||||||
"""Return the path for .rustfmt-stamp
|
"""Return the path for .rustfmt-stamp
|
||||||
|
@ -709,7 +644,7 @@ class RustBuild(object):
|
||||||
>>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
|
>>> rb.rustfmt_stamp() == os.path.join("build", "stage0", ".rustfmt-stamp")
|
||||||
True
|
True
|
||||||
"""
|
"""
|
||||||
return os.path.join(self.bin_root(True), '.rustfmt-stamp')
|
return os.path.join(self.bin_root(), '.rustfmt-stamp')
|
||||||
|
|
||||||
def program_out_of_date(self, stamp_path, key):
|
def program_out_of_date(self, stamp_path, key):
|
||||||
"""Check if the given program stamp is out of date"""
|
"""Check if the given program stamp is out of date"""
|
||||||
|
@ -718,26 +653,21 @@ class RustBuild(object):
|
||||||
with open(stamp_path, 'r') as stamp:
|
with open(stamp_path, 'r') as stamp:
|
||||||
return key != stamp.read()
|
return key != stamp.read()
|
||||||
|
|
||||||
def bin_root(self, stage0):
|
def bin_root(self):
|
||||||
"""Return the binary root directory for the given stage
|
"""Return the binary root directory for the given stage
|
||||||
|
|
||||||
>>> rb = RustBuild()
|
>>> rb = RustBuild()
|
||||||
>>> rb.build_dir = "build"
|
>>> rb.build_dir = "build"
|
||||||
>>> rb.bin_root(True) == os.path.join("build", "stage0")
|
>>> rb.bin_root() == os.path.join("build", "stage0")
|
||||||
True
|
|
||||||
>>> rb.bin_root(False) == os.path.join("build", "ci-rustc")
|
|
||||||
True
|
True
|
||||||
|
|
||||||
When the 'build' property is given should be a nested directory:
|
When the 'build' property is given should be a nested directory:
|
||||||
|
|
||||||
>>> rb.build = "devel"
|
>>> rb.build = "devel"
|
||||||
>>> rb.bin_root(True) == os.path.join("build", "devel", "stage0")
|
>>> rb.bin_root() == os.path.join("build", "devel", "stage0")
|
||||||
True
|
True
|
||||||
"""
|
"""
|
||||||
if stage0:
|
subdir = "stage0"
|
||||||
subdir = "stage0"
|
|
||||||
else:
|
|
||||||
subdir = "ci-rustc"
|
|
||||||
return os.path.join(self.build_dir, self.build, subdir)
|
return os.path.join(self.build_dir, self.build, subdir)
|
||||||
|
|
||||||
def get_toml(self, key, section=None):
|
def get_toml(self, key, section=None):
|
||||||
|
@ -785,9 +715,9 @@ class RustBuild(object):
|
||||||
"""Return config path for cargo"""
|
"""Return config path for cargo"""
|
||||||
return self.program_config('cargo')
|
return self.program_config('cargo')
|
||||||
|
|
||||||
def rustc(self, stage0):
|
def rustc(self):
|
||||||
"""Return config path for rustc"""
|
"""Return config path for rustc"""
|
||||||
return self.program_config('rustc', stage0)
|
return self.program_config('rustc')
|
||||||
|
|
||||||
def rustfmt(self):
|
def rustfmt(self):
|
||||||
"""Return config path for rustfmt"""
|
"""Return config path for rustfmt"""
|
||||||
|
@ -795,7 +725,7 @@ class RustBuild(object):
|
||||||
return None
|
return None
|
||||||
return self.program_config('rustfmt')
|
return self.program_config('rustfmt')
|
||||||
|
|
||||||
def program_config(self, program, stage0=True):
|
def program_config(self, program):
|
||||||
"""Return config path for the given program at the given stage
|
"""Return config path for the given program at the given stage
|
||||||
|
|
||||||
>>> rb = RustBuild()
|
>>> rb = RustBuild()
|
||||||
|
@ -803,19 +733,15 @@ class RustBuild(object):
|
||||||
>>> rb.program_config('rustc')
|
>>> rb.program_config('rustc')
|
||||||
'rustc'
|
'rustc'
|
||||||
>>> rb.config_toml = ''
|
>>> rb.config_toml = ''
|
||||||
>>> cargo_path = rb.program_config('cargo', True)
|
>>> cargo_path = rb.program_config('cargo')
|
||||||
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(True),
|
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(),
|
||||||
... "bin", "cargo")
|
|
||||||
True
|
|
||||||
>>> cargo_path = rb.program_config('cargo', False)
|
|
||||||
>>> cargo_path.rstrip(".exe") == os.path.join(rb.bin_root(False),
|
|
||||||
... "bin", "cargo")
|
... "bin", "cargo")
|
||||||
True
|
True
|
||||||
"""
|
"""
|
||||||
config = self.get_toml(program)
|
config = self.get_toml(program)
|
||||||
if config:
|
if config:
|
||||||
return os.path.expanduser(config)
|
return os.path.expanduser(config)
|
||||||
return os.path.join(self.bin_root(stage0), "bin", "{}{}".format(
|
return os.path.join(self.bin_root(), "bin", "{}{}".format(
|
||||||
program, self.exe_suffix()))
|
program, self.exe_suffix()))
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
|
@ -871,14 +797,14 @@ class RustBuild(object):
|
||||||
if "CARGO_BUILD_TARGET" in env:
|
if "CARGO_BUILD_TARGET" in env:
|
||||||
del env["CARGO_BUILD_TARGET"]
|
del env["CARGO_BUILD_TARGET"]
|
||||||
env["CARGO_TARGET_DIR"] = build_dir
|
env["CARGO_TARGET_DIR"] = build_dir
|
||||||
env["RUSTC"] = self.rustc(True)
|
env["RUSTC"] = self.rustc()
|
||||||
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
|
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
|
||||||
(os.pathsep + env["LD_LIBRARY_PATH"]) \
|
(os.pathsep + env["LD_LIBRARY_PATH"]) \
|
||||||
if "LD_LIBRARY_PATH" in env else ""
|
if "LD_LIBRARY_PATH" in env else ""
|
||||||
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
|
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
|
||||||
(os.pathsep + env["DYLD_LIBRARY_PATH"]) \
|
(os.pathsep + env["DYLD_LIBRARY_PATH"]) \
|
||||||
if "DYLD_LIBRARY_PATH" in env else ""
|
if "DYLD_LIBRARY_PATH" in env else ""
|
||||||
env["LIBRARY_PATH"] = os.path.join(self.bin_root(True), "lib") + \
|
env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \
|
||||||
(os.pathsep + env["LIBRARY_PATH"]) \
|
(os.pathsep + env["LIBRARY_PATH"]) \
|
||||||
if "LIBRARY_PATH" in env else ""
|
if "LIBRARY_PATH" in env else ""
|
||||||
|
|
||||||
|
@ -900,7 +826,7 @@ class RustBuild(object):
|
||||||
if self.get_toml("deny-warnings", "rust") != "false":
|
if self.get_toml("deny-warnings", "rust") != "false":
|
||||||
env["RUSTFLAGS"] += " -Dwarnings"
|
env["RUSTFLAGS"] += " -Dwarnings"
|
||||||
|
|
||||||
env["PATH"] = os.path.join(self.bin_root(True), "bin") + \
|
env["PATH"] = os.path.join(self.bin_root(), "bin") + \
|
||||||
os.pathsep + env["PATH"]
|
os.pathsep + env["PATH"]
|
||||||
if not os.path.isfile(self.cargo()):
|
if not os.path.isfile(self.cargo()):
|
||||||
raise Exception("no cargo executable found at `{}`".format(
|
raise Exception("no cargo executable found at `{}`".format(
|
||||||
|
@ -1171,8 +1097,6 @@ def bootstrap(help_triggered):
|
||||||
|
|
||||||
# Fetch/build the bootstrap
|
# Fetch/build the bootstrap
|
||||||
build.download_toolchain()
|
build.download_toolchain()
|
||||||
# Download the master compiler if `download-rustc` is set
|
|
||||||
build.maybe_download_ci_toolchain()
|
|
||||||
sys.stdout.flush()
|
sys.stdout.flush()
|
||||||
build.ensure_vendored()
|
build.ensure_vendored()
|
||||||
build.build_bootstrap()
|
build.build_bootstrap()
|
||||||
|
@ -1184,8 +1108,6 @@ def bootstrap(help_triggered):
|
||||||
env = os.environ.copy()
|
env = os.environ.copy()
|
||||||
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
|
env["BOOTSTRAP_PARENT_ID"] = str(os.getpid())
|
||||||
env["BOOTSTRAP_PYTHON"] = sys.executable
|
env["BOOTSTRAP_PYTHON"] = sys.executable
|
||||||
if build.rustc_commit is not None:
|
|
||||||
env["BOOTSTRAP_DOWNLOAD_RUSTC"] = '1'
|
|
||||||
run(args, env=env, verbose=build.verbose, is_bootstrap=True)
|
run(args, env=env, verbose=build.verbose, is_bootstrap=True)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -2,13 +2,14 @@ use std::any::{type_name, Any};
|
||||||
use std::cell::{Cell, RefCell};
|
use std::cell::{Cell, RefCell};
|
||||||
use std::collections::BTreeSet;
|
use std::collections::BTreeSet;
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::ffi::OsStr;
|
use std::ffi::{OsStr, OsString};
|
||||||
use std::fmt::{Debug, Write};
|
use std::fmt::{Debug, Write};
|
||||||
use std::fs;
|
use std::fs::{self, File};
|
||||||
use std::hash::Hash;
|
use std::hash::Hash;
|
||||||
|
use std::io::{BufRead, BufReader, ErrorKind};
|
||||||
use std::ops::Deref;
|
use std::ops::Deref;
|
||||||
use std::path::{Component, Path, PathBuf};
|
use std::path::{Component, Path, PathBuf};
|
||||||
use std::process::Command;
|
use std::process::{Command, Stdio};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use crate::cache::{Cache, Interned, INTERNER};
|
use crate::cache::{Cache, Interned, INTERNER};
|
||||||
|
@ -29,7 +30,8 @@ use crate::{Build, CLang, DocTests, GitRepo, Mode};
|
||||||
|
|
||||||
pub use crate::Compiler;
|
pub use crate::Compiler;
|
||||||
// FIXME: replace with std::lazy after it gets stabilized and reaches beta
|
// FIXME: replace with std::lazy after it gets stabilized and reaches beta
|
||||||
use once_cell::sync::Lazy;
|
use once_cell::sync::{Lazy, OnceCell};
|
||||||
|
use xz2::bufread::XzDecoder;
|
||||||
|
|
||||||
pub struct Builder<'a> {
|
pub struct Builder<'a> {
|
||||||
pub build: &'a Build,
|
pub build: &'a Build,
|
||||||
|
@ -758,6 +760,207 @@ impl<'a> Builder<'a> {
|
||||||
StepDescription::run(v, self, paths);
|
StepDescription::run(v, self, paths);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Modifies the interpreter section of 'fname' to fix the dynamic linker,
|
||||||
|
/// or the RPATH section, to fix the dynamic library search path
|
||||||
|
///
|
||||||
|
/// This is only required on NixOS and uses the PatchELF utility to
|
||||||
|
/// change the interpreter/RPATH of ELF executables.
|
||||||
|
///
|
||||||
|
/// Please see https://nixos.org/patchelf.html for more information
|
||||||
|
pub(crate) fn fix_bin_or_dylib(&self, fname: &Path) {
|
||||||
|
// FIXME: cache NixOS detection?
|
||||||
|
match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
|
||||||
|
Err(_) => return,
|
||||||
|
Ok(output) if !output.status.success() => return,
|
||||||
|
Ok(output) => {
|
||||||
|
let mut s = output.stdout;
|
||||||
|
if s.last() == Some(&b'\n') {
|
||||||
|
s.pop();
|
||||||
|
}
|
||||||
|
if s != b"Linux" {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// If the user has asked binaries to be patched for Nix, then
|
||||||
|
// don't check for NixOS or `/lib`, just continue to the patching.
|
||||||
|
// NOTE: this intentionally comes after the Linux check:
|
||||||
|
// - patchelf only works with ELF files, so no need to run it on Mac or Windows
|
||||||
|
// - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc.
|
||||||
|
if !self.config.patch_binaries_for_nix {
|
||||||
|
// Use `/etc/os-release` instead of `/etc/NIXOS`.
|
||||||
|
// The latter one does not exist on NixOS when using tmpfs as root.
|
||||||
|
const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""];
|
||||||
|
let os_release = match File::open("/etc/os-release") {
|
||||||
|
Err(e) if e.kind() == ErrorKind::NotFound => return,
|
||||||
|
Err(e) => panic!("failed to access /etc/os-release: {}", e),
|
||||||
|
Ok(f) => f,
|
||||||
|
};
|
||||||
|
if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if Path::new("/lib").exists() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// At this point we're pretty sure the user is running NixOS or using Nix
|
||||||
|
println!("info: you seem to be using Nix. Attempting to patch {}", fname.display());
|
||||||
|
|
||||||
|
// Only build `.nix-deps` once.
|
||||||
|
static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
|
||||||
|
let mut nix_build_succeeded = true;
|
||||||
|
let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
|
||||||
|
// Run `nix-build` to "build" each dependency (which will likely reuse
|
||||||
|
// the existing `/nix/store` copy, or at most download a pre-built copy).
|
||||||
|
//
|
||||||
|
// Importantly, we create a gc-root called `.nix-deps` in the `build/`
|
||||||
|
// directory, but still reference the actual `/nix/store` path in the rpath
|
||||||
|
// as it makes it significantly more robust against changes to the location of
|
||||||
|
// the `.nix-deps` location.
|
||||||
|
//
|
||||||
|
// bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
|
||||||
|
// zlib: Needed as a system dependency of `libLLVM-*.so`.
|
||||||
|
// patchelf: Needed for patching ELF binaries (see doc comment above).
|
||||||
|
let nix_deps_dir = self.out.join(".nix-deps");
|
||||||
|
const NIX_EXPR: &str = "
|
||||||
|
with (import <nixpkgs> {});
|
||||||
|
symlinkJoin {
|
||||||
|
name = \"rust-stage0-dependencies\";
|
||||||
|
paths = [
|
||||||
|
zlib
|
||||||
|
patchelf
|
||||||
|
stdenv.cc.bintools
|
||||||
|
];
|
||||||
|
}
|
||||||
|
";
|
||||||
|
nix_build_succeeded = self.try_run(Command::new("nix-build").args(&[
|
||||||
|
Path::new("-E"),
|
||||||
|
Path::new(NIX_EXPR),
|
||||||
|
Path::new("-o"),
|
||||||
|
&nix_deps_dir,
|
||||||
|
]));
|
||||||
|
nix_deps_dir
|
||||||
|
});
|
||||||
|
if !nix_build_succeeded {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf"));
|
||||||
|
let rpath_entries = {
|
||||||
|
// ORIGIN is a relative default, all binary and dynamic libraries we ship
|
||||||
|
// appear to have this (even when `../lib` is redundant).
|
||||||
|
// NOTE: there are only two paths here, delimited by a `:`
|
||||||
|
let mut entries = OsString::from("$ORIGIN/../lib:");
|
||||||
|
entries.push(t!(fs::canonicalize(nix_deps_dir)));
|
||||||
|
entries.push("/lib");
|
||||||
|
entries
|
||||||
|
};
|
||||||
|
patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]);
|
||||||
|
if !fname.extension().map_or(false, |ext| ext == "so") {
|
||||||
|
// Finally, set the corret .interp for binaries
|
||||||
|
let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
|
||||||
|
// FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ...
|
||||||
|
let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path))));
|
||||||
|
patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]);
|
||||||
|
}
|
||||||
|
|
||||||
|
self.try_run(patchelf.arg(fname));
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn download_component(&self, base: &str, url: &str, dest_path: &Path) {
|
||||||
|
// Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
|
||||||
|
let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
|
||||||
|
// FIXME: support `do_verify` (only really needed for nightly rustfmt)
|
||||||
|
self.download_with_retries(&tempfile, &format!("{}/{}", base, url));
|
||||||
|
t!(std::fs::rename(&tempfile, dest_path));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn download_with_retries(&self, tempfile: &Path, url: &str) {
|
||||||
|
println!("downloading {}", url);
|
||||||
|
// Try curl. If that fails and we are on windows, fallback to PowerShell.
|
||||||
|
let mut curl = Command::new("curl");
|
||||||
|
curl.args(&[
|
||||||
|
"-#",
|
||||||
|
"-y",
|
||||||
|
"30",
|
||||||
|
"-Y",
|
||||||
|
"10", // timeout if speed is < 10 bytes/sec for > 30 seconds
|
||||||
|
"--connect-timeout",
|
||||||
|
"30", // timeout if cannot connect within 30 seconds
|
||||||
|
"--retry",
|
||||||
|
"3",
|
||||||
|
"-Sf",
|
||||||
|
"-o",
|
||||||
|
]);
|
||||||
|
curl.arg(tempfile);
|
||||||
|
curl.arg(url);
|
||||||
|
if !self.check_run(&mut curl) {
|
||||||
|
if self.build.build.contains("windows-msvc") {
|
||||||
|
println!("Fallback to PowerShell");
|
||||||
|
for _ in 0..3 {
|
||||||
|
if self.try_run(Command::new("PowerShell.exe").args(&[
|
||||||
|
"/nologo",
|
||||||
|
"-Command",
|
||||||
|
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
|
||||||
|
&format!(
|
||||||
|
"(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
|
||||||
|
url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"),
|
||||||
|
),
|
||||||
|
])) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
println!("\nspurious failure, trying again");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
std::process::exit(1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) {
|
||||||
|
println!("extracting {} to {}", tarball.display(), dst.display());
|
||||||
|
if !dst.exists() {
|
||||||
|
t!(fs::create_dir_all(dst));
|
||||||
|
}
|
||||||
|
|
||||||
|
// `tarball` ends with `.tar.xz`; strip that suffix
|
||||||
|
// example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
|
||||||
|
let uncompressed_filename =
|
||||||
|
Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
|
||||||
|
let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
|
||||||
|
|
||||||
|
// decompress the file
|
||||||
|
let data = t!(File::open(tarball));
|
||||||
|
let decompressor = XzDecoder::new(BufReader::new(data));
|
||||||
|
|
||||||
|
let mut tar = tar::Archive::new(decompressor);
|
||||||
|
for member in t!(tar.entries()) {
|
||||||
|
let mut member = t!(member);
|
||||||
|
let original_path = t!(member.path()).into_owned();
|
||||||
|
// skip the top-level directory
|
||||||
|
if original_path == directory_prefix {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let mut short_path = t!(original_path.strip_prefix(directory_prefix));
|
||||||
|
if !short_path.starts_with(pattern) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
short_path = t!(short_path.strip_prefix(pattern));
|
||||||
|
let dst_path = dst.join(short_path);
|
||||||
|
self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display()));
|
||||||
|
if !t!(member.unpack_in(dst)) {
|
||||||
|
panic!("path traversal attack ??");
|
||||||
|
}
|
||||||
|
let src_path = dst.join(original_path);
|
||||||
|
if src_path.is_dir() && dst_path.exists() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
t!(fs::rename(src_path, dst_path));
|
||||||
|
}
|
||||||
|
t!(fs::remove_dir_all(dst.join(directory_prefix)));
|
||||||
|
}
|
||||||
|
|
||||||
/// Obtain a compiler at a given stage and for a given host. Explicitly does
|
/// Obtain a compiler at a given stage and for a given host. Explicitly does
|
||||||
/// not take `Compiler` since all `Compiler` instances are meant to be
|
/// not take `Compiler` since all `Compiler` instances are meant to be
|
||||||
/// obtained through this function, since it ensures that they are valid
|
/// obtained through this function, since it ensures that they are valid
|
||||||
|
@ -819,7 +1022,7 @@ impl<'a> Builder<'a> {
|
||||||
.join("lib");
|
.join("lib");
|
||||||
// Avoid deleting the rustlib/ directory we just copied
|
// Avoid deleting the rustlib/ directory we just copied
|
||||||
// (in `impl Step for Sysroot`).
|
// (in `impl Step for Sysroot`).
|
||||||
if !builder.config.download_rustc {
|
if !builder.download_rustc() {
|
||||||
let _ = fs::remove_dir_all(&sysroot);
|
let _ = fs::remove_dir_all(&sysroot);
|
||||||
t!(fs::create_dir_all(&sysroot));
|
t!(fs::create_dir_all(&sysroot));
|
||||||
}
|
}
|
||||||
|
@ -976,6 +1179,10 @@ impl<'a> Builder<'a> {
|
||||||
Config::llvm_link_shared(self)
|
Config::llvm_link_shared(self)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn download_rustc(&self) -> bool {
|
||||||
|
Config::download_rustc(self)
|
||||||
|
}
|
||||||
|
|
||||||
/// Prepares an invocation of `cargo` to be run.
|
/// Prepares an invocation of `cargo` to be run.
|
||||||
///
|
///
|
||||||
/// This will create a `Command` that represents a pending execution of
|
/// This will create a `Command` that represents a pending execution of
|
||||||
|
|
|
@ -42,8 +42,10 @@ impl Step for Std {
|
||||||
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
|
fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
|
||||||
// When downloading stage1, the standard library has already been copied to the sysroot, so
|
// When downloading stage1, the standard library has already been copied to the sysroot, so
|
||||||
// there's no need to rebuild it.
|
// there's no need to rebuild it.
|
||||||
let download_rustc = run.builder.config.download_rustc;
|
let builder = run.builder;
|
||||||
run.all_krates("test").path("library").default_condition(!download_rustc)
|
run.all_krates("test")
|
||||||
|
.path("library")
|
||||||
|
.lazy_default_condition(Box::new(|| !builder.download_rustc()))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn make_run(run: RunConfig<'_>) {
|
fn make_run(run: RunConfig<'_>) {
|
||||||
|
@ -66,7 +68,7 @@ impl Step for Std {
|
||||||
// Don't recompile them.
|
// Don't recompile them.
|
||||||
// NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
|
// NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
|
||||||
// so its artifacts can't be reused.
|
// so its artifacts can't be reused.
|
||||||
if builder.config.download_rustc && compiler.stage != 0 {
|
if builder.download_rustc() && compiler.stage != 0 {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -551,7 +553,7 @@ impl Step for Rustc {
|
||||||
|
|
||||||
// NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
|
// NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler,
|
||||||
// so its artifacts can't be reused.
|
// so its artifacts can't be reused.
|
||||||
if builder.config.download_rustc && compiler.stage != 0 {
|
if builder.download_rustc() && compiler.stage != 0 {
|
||||||
// Copy the existing artifacts instead of rebuilding them.
|
// Copy the existing artifacts instead of rebuilding them.
|
||||||
// NOTE: this path is only taken for tools linking to rustc-dev.
|
// NOTE: this path is only taken for tools linking to rustc-dev.
|
||||||
builder.ensure(Sysroot { compiler });
|
builder.ensure(Sysroot { compiler });
|
||||||
|
@ -995,7 +997,7 @@ impl Step for Sysroot {
|
||||||
t!(fs::create_dir_all(&sysroot));
|
t!(fs::create_dir_all(&sysroot));
|
||||||
|
|
||||||
// If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
|
// If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
|
||||||
if builder.config.download_rustc && compiler.stage != 0 {
|
if builder.download_rustc() && compiler.stage != 0 {
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
builder.config.build, compiler.host,
|
builder.config.build, compiler.host,
|
||||||
"Cross-compiling is not yet supported with `download-rustc`",
|
"Cross-compiling is not yet supported with `download-rustc`",
|
||||||
|
@ -1090,7 +1092,7 @@ impl Step for Assemble {
|
||||||
let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build);
|
let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build);
|
||||||
|
|
||||||
// If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
|
// If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0.
|
||||||
if builder.config.download_rustc {
|
if builder.download_rustc() {
|
||||||
builder.ensure(Sysroot { compiler: target_compiler });
|
builder.ensure(Sysroot { compiler: target_compiler });
|
||||||
return target_compiler;
|
return target_compiler;
|
||||||
}
|
}
|
||||||
|
|
|
@ -7,9 +7,11 @@ use std::cell::Cell;
|
||||||
use std::cmp;
|
use std::cmp;
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{HashMap, HashSet};
|
||||||
use std::env;
|
use std::env;
|
||||||
|
use std::ffi::OsStr;
|
||||||
use std::fmt;
|
use std::fmt;
|
||||||
use std::fs;
|
use std::fs;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::process::{exit, Command};
|
||||||
use std::str::FromStr;
|
use std::str::FromStr;
|
||||||
|
|
||||||
use crate::builder::{Builder, TaskPath};
|
use crate::builder::{Builder, TaskPath};
|
||||||
|
@ -17,7 +19,8 @@ use crate::cache::{Interned, INTERNER};
|
||||||
use crate::channel::GitInfo;
|
use crate::channel::GitInfo;
|
||||||
pub use crate::flags::Subcommand;
|
pub use crate::flags::Subcommand;
|
||||||
use crate::flags::{Color, Flags};
|
use crate::flags::{Color, Flags};
|
||||||
use crate::util::{exe, t};
|
use crate::util::{exe, output, program_out_of_date, t};
|
||||||
|
use once_cell::sync::OnceCell;
|
||||||
use serde::{Deserialize, Deserializer};
|
use serde::{Deserialize, Deserializer};
|
||||||
|
|
||||||
macro_rules! check_ci_llvm {
|
macro_rules! check_ci_llvm {
|
||||||
|
@ -81,7 +84,11 @@ pub struct Config {
|
||||||
pub cmd: Subcommand,
|
pub cmd: Subcommand,
|
||||||
pub incremental: bool,
|
pub incremental: bool,
|
||||||
pub dry_run: bool,
|
pub dry_run: bool,
|
||||||
pub download_rustc: bool,
|
/// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should.
|
||||||
|
#[cfg(not(test))]
|
||||||
|
download_rustc_commit: Option<String>,
|
||||||
|
#[cfg(test)]
|
||||||
|
pub download_rustc_commit: Option<String>,
|
||||||
|
|
||||||
pub deny_warnings: bool,
|
pub deny_warnings: bool,
|
||||||
pub backtrace_on_ice: bool,
|
pub backtrace_on_ice: bool,
|
||||||
|
@ -1080,7 +1087,8 @@ impl Config {
|
||||||
config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
|
config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config);
|
||||||
config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
|
config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use);
|
||||||
config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
|
config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate);
|
||||||
config.download_rustc = env::var("BOOTSTRAP_DOWNLOAD_RUSTC").as_deref() == Ok("1");
|
config.download_rustc_commit =
|
||||||
|
download_ci_rustc_commit(rust.download_rustc, config.verbose > 0);
|
||||||
} else {
|
} else {
|
||||||
config.rust_profile_use = flags.rust_profile_use;
|
config.rust_profile_use = flags.rust_profile_use;
|
||||||
config.rust_profile_generate = flags.rust_profile_generate;
|
config.rust_profile_generate = flags.rust_profile_generate;
|
||||||
|
@ -1192,7 +1200,7 @@ impl Config {
|
||||||
let default = config.channel == "dev";
|
let default = config.channel == "dev";
|
||||||
config.ignore_git = ignore_git.unwrap_or(default);
|
config.ignore_git = ignore_git.unwrap_or(default);
|
||||||
|
|
||||||
let download_rustc = config.download_rustc;
|
let download_rustc = config.download_rustc_commit.is_some();
|
||||||
// See https://github.com/rust-lang/compiler-team/issues/326
|
// See https://github.com/rust-lang/compiler-team/issues/326
|
||||||
config.stage = match config.cmd {
|
config.stage = match config.cmd {
|
||||||
Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0),
|
Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0),
|
||||||
|
@ -1309,6 +1317,23 @@ impl Config {
|
||||||
llvm_link_shared
|
llvm_link_shared
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source.
|
||||||
|
pub(crate) fn download_rustc(builder: &Builder<'_>) -> bool {
|
||||||
|
static DOWNLOAD_RUSTC: OnceCell<bool> = OnceCell::new();
|
||||||
|
if builder.config.dry_run && DOWNLOAD_RUSTC.get().is_none() {
|
||||||
|
// avoid trying to actually download the commit
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
*DOWNLOAD_RUSTC.get_or_init(|| match &builder.config.download_rustc_commit {
|
||||||
|
None => false,
|
||||||
|
Some(commit) => {
|
||||||
|
download_ci_rustc(builder, commit);
|
||||||
|
true
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn verbose(&self) -> bool {
|
pub fn verbose(&self) -> bool {
|
||||||
self.verbose > 0
|
self.verbose > 0
|
||||||
}
|
}
|
||||||
|
@ -1358,3 +1383,116 @@ fn threads_from_config(v: u32) -> u32 {
|
||||||
n => n,
|
n => n,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
|
||||||
|
fn download_ci_rustc_commit(download_rustc: Option<StringOrBool>, verbose: bool) -> Option<String> {
|
||||||
|
// If `download-rustc` is not set, default to rebuilding.
|
||||||
|
let if_unchanged = match download_rustc {
|
||||||
|
None | Some(StringOrBool::Bool(false)) => return None,
|
||||||
|
Some(StringOrBool::Bool(true)) => false,
|
||||||
|
Some(StringOrBool::String(s)) if s == "if-unchanged" => true,
|
||||||
|
Some(StringOrBool::String(other)) => {
|
||||||
|
panic!("unrecognized option for download-rustc: {}", other)
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// Handle running from a directory other than the top level
|
||||||
|
let top_level = output(Command::new("git").args(&["rev-parse", "--show-toplevel"]));
|
||||||
|
let top_level = top_level.trim_end();
|
||||||
|
let compiler = format!("{top_level}/compiler/");
|
||||||
|
let library = format!("{top_level}/library/");
|
||||||
|
|
||||||
|
// Look for a version to compare to based on the current commit.
|
||||||
|
// Only commits merged by bors will have CI artifacts.
|
||||||
|
let merge_base = output(Command::new("git").args(&[
|
||||||
|
"rev-list",
|
||||||
|
"--author=bors@rust-lang.org",
|
||||||
|
"-n1",
|
||||||
|
"--first-parent",
|
||||||
|
"HEAD",
|
||||||
|
]));
|
||||||
|
let commit = merge_base.trim_end();
|
||||||
|
if commit.is_empty() {
|
||||||
|
println!("error: could not find commit hash for downloading rustc");
|
||||||
|
println!("help: maybe your repository history is too shallow?");
|
||||||
|
println!("help: consider disabling `download-rustc`");
|
||||||
|
println!("help: or fetch enough history to include one upstream commit");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Warn if there were changes to the compiler or standard library since the ancestor commit.
|
||||||
|
let has_changes = !t!(Command::new("git")
|
||||||
|
.args(&["diff-index", "--quiet", &commit, "--", &compiler, &library])
|
||||||
|
.status())
|
||||||
|
.success();
|
||||||
|
if has_changes {
|
||||||
|
if if_unchanged {
|
||||||
|
if verbose {
|
||||||
|
println!(
|
||||||
|
"warning: saw changes to compiler/ or library/ since {commit}; \
|
||||||
|
ignoring `download-rustc`"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
println!(
|
||||||
|
"warning: `download-rustc` is enabled, but there are changes to \
|
||||||
|
compiler/ or library/"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
Some(commit.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn download_ci_rustc(builder: &Builder<'_>, commit: &str) {
|
||||||
|
builder.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})"));
|
||||||
|
// FIXME: support downloading artifacts from the beta channel
|
||||||
|
const CHANNEL: &str = "nightly";
|
||||||
|
let host = builder.config.build.triple;
|
||||||
|
let bin_root = builder.out.join(host).join("ci-rustc");
|
||||||
|
let rustc_stamp = bin_root.join(".rustc-stamp");
|
||||||
|
|
||||||
|
if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit) {
|
||||||
|
if bin_root.exists() {
|
||||||
|
t!(fs::remove_dir_all(&bin_root));
|
||||||
|
}
|
||||||
|
let filename = format!("rust-std-{CHANNEL}-{host}.tar.xz");
|
||||||
|
let pattern = format!("rust-std-{host}");
|
||||||
|
download_component(builder, filename, &pattern, commit);
|
||||||
|
let filename = format!("rustc-{CHANNEL}-{host}.tar.xz");
|
||||||
|
download_component(builder, filename, "rustc", commit);
|
||||||
|
// download-rustc doesn't need its own cargo, it can just use beta's.
|
||||||
|
let filename = format!("rustc-dev-{CHANNEL}-{host}.tar.xz");
|
||||||
|
download_component(builder, filename, "rustc-dev", commit);
|
||||||
|
|
||||||
|
builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustc"));
|
||||||
|
builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc"));
|
||||||
|
let lib_dir = bin_root.join("lib");
|
||||||
|
for lib in t!(fs::read_dir(lib_dir)) {
|
||||||
|
let lib = t!(lib);
|
||||||
|
if lib.path().extension() == Some(OsStr::new("so")) {
|
||||||
|
builder.fix_bin_or_dylib(&lib.path());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
t!(fs::write(rustc_stamp, commit));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Download a single component of a CI-built toolchain (not necessarily a published nightly).
|
||||||
|
// NOTE: intentionally takes an owned string to avoid downloading multiple times by accident
|
||||||
|
fn download_component(builder: &Builder<'_>, filename: String, prefix: &str, commit: &str) {
|
||||||
|
let cache_dst = builder.out.join("cache");
|
||||||
|
let rustc_cache = cache_dst.join(commit);
|
||||||
|
if !rustc_cache.exists() {
|
||||||
|
t!(fs::create_dir_all(&rustc_cache));
|
||||||
|
}
|
||||||
|
|
||||||
|
let base = "https://ci-artifacts.rust-lang.org";
|
||||||
|
let url = format!("rustc-builds/{commit}");
|
||||||
|
let tarball = rustc_cache.join(&filename);
|
||||||
|
if !tarball.exists() {
|
||||||
|
builder.download_component(base, &format!("{url}/{filename}"), &tarball);
|
||||||
|
}
|
||||||
|
let bin_root = builder.out.join(builder.config.build.triple).join("ci-rustc");
|
||||||
|
builder.unpack(&tarball, &bin_root, prefix)
|
||||||
|
}
|
||||||
|
|
|
@ -12,16 +12,13 @@ use std::env;
|
||||||
use std::env::consts::EXE_EXTENSION;
|
use std::env::consts::EXE_EXTENSION;
|
||||||
use std::ffi::{OsStr, OsString};
|
use std::ffi::{OsStr, OsString};
|
||||||
use std::fs::{self, File};
|
use std::fs::{self, File};
|
||||||
use std::io::{self, BufRead, BufReader, ErrorKind};
|
use std::io;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::process::{Command, Stdio};
|
use std::process::Command;
|
||||||
|
|
||||||
use once_cell::sync::OnceCell;
|
|
||||||
use xz2::bufread::XzDecoder;
|
|
||||||
|
|
||||||
use crate::builder::{Builder, RunConfig, ShouldRun, Step};
|
use crate::builder::{Builder, RunConfig, ShouldRun, Step};
|
||||||
use crate::config::TargetSelection;
|
use crate::config::TargetSelection;
|
||||||
use crate::util::{self, exe, output, t, up_to_date};
|
use crate::util::{self, exe, output, program_out_of_date, t, up_to_date};
|
||||||
use crate::{CLang, GitRepo};
|
use crate::{CLang, GitRepo};
|
||||||
|
|
||||||
pub struct Meta {
|
pub struct Meta {
|
||||||
|
@ -151,13 +148,13 @@ pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) {
|
||||||
if program_out_of_date(&llvm_stamp, &key) && !config.dry_run {
|
if program_out_of_date(&llvm_stamp, &key) && !config.dry_run {
|
||||||
download_ci_llvm(builder, &llvm_sha);
|
download_ci_llvm(builder, &llvm_sha);
|
||||||
for binary in ["llvm-config", "FileCheck"] {
|
for binary in ["llvm-config", "FileCheck"] {
|
||||||
fix_bin_or_dylib(builder, &llvm_root.join("bin").join(binary));
|
builder.fix_bin_or_dylib(&llvm_root.join("bin").join(binary));
|
||||||
}
|
}
|
||||||
let llvm_lib = llvm_root.join("lib");
|
let llvm_lib = llvm_root.join("lib");
|
||||||
for entry in t!(fs::read_dir(&llvm_lib)) {
|
for entry in t!(fs::read_dir(&llvm_lib)) {
|
||||||
let lib = t!(entry).path();
|
let lib = t!(entry).path();
|
||||||
if lib.extension().map_or(false, |ext| ext == "so") {
|
if lib.extension().map_or(false, |ext| ext == "so") {
|
||||||
fix_bin_or_dylib(builder, &lib);
|
builder.fix_bin_or_dylib(&lib);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
t!(fs::write(llvm_stamp, key));
|
t!(fs::write(llvm_stamp, key));
|
||||||
|
@ -182,218 +179,10 @@ fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) {
|
||||||
let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple);
|
let filename = format!("rust-dev-nightly-{}.tar.xz", builder.build.build.triple);
|
||||||
let tarball = rustc_cache.join(&filename);
|
let tarball = rustc_cache.join(&filename);
|
||||||
if !tarball.exists() {
|
if !tarball.exists() {
|
||||||
download_component(builder, base, &format!("{}/{}", url, filename), &tarball);
|
builder.download_component(base, &format!("{}/{}", url, filename), &tarball);
|
||||||
}
|
}
|
||||||
let llvm_root = builder.config.ci_llvm_root();
|
let llvm_root = builder.config.ci_llvm_root();
|
||||||
unpack(builder, &tarball, &llvm_root);
|
builder.unpack(&tarball, &llvm_root, "rust-dev");
|
||||||
}
|
|
||||||
|
|
||||||
/// Modifies the interpreter section of 'fname' to fix the dynamic linker,
|
|
||||||
/// or the RPATH section, to fix the dynamic library search path
|
|
||||||
///
|
|
||||||
/// This is only required on NixOS and uses the PatchELF utility to
|
|
||||||
/// change the interpreter/RPATH of ELF executables.
|
|
||||||
///
|
|
||||||
/// Please see https://nixos.org/patchelf.html for more information
|
|
||||||
fn fix_bin_or_dylib(builder: &Builder<'_>, fname: &Path) {
|
|
||||||
// FIXME: cache NixOS detection?
|
|
||||||
match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() {
|
|
||||||
Err(_) => return,
|
|
||||||
Ok(output) if !output.status.success() => return,
|
|
||||||
Ok(output) => {
|
|
||||||
let mut s = output.stdout;
|
|
||||||
if s.last() == Some(&b'\n') {
|
|
||||||
s.pop();
|
|
||||||
}
|
|
||||||
if s != b"Linux" {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// If the user has asked binaries to be patched for Nix, then
|
|
||||||
// don't check for NixOS or `/lib`, just continue to the patching.
|
|
||||||
// FIXME: shouldn't this take precedence over the `uname` check above?
|
|
||||||
if !builder.config.patch_binaries_for_nix {
|
|
||||||
// Use `/etc/os-release` instead of `/etc/NIXOS`.
|
|
||||||
// The latter one does not exist on NixOS when using tmpfs as root.
|
|
||||||
const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""];
|
|
||||||
let os_release = match File::open("/etc/os-release") {
|
|
||||||
Err(e) if e.kind() == ErrorKind::NotFound => return,
|
|
||||||
Err(e) => panic!("failed to access /etc/os-release: {}", e),
|
|
||||||
Ok(f) => f,
|
|
||||||
};
|
|
||||||
if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if Path::new("/lib").exists() {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// At this point we're pretty sure the user is running NixOS or using Nix
|
|
||||||
println!("info: you seem to be using Nix. Attempting to patch {}", fname.display());
|
|
||||||
|
|
||||||
// Only build `.nix-deps` once.
|
|
||||||
static NIX_DEPS_DIR: OnceCell<PathBuf> = OnceCell::new();
|
|
||||||
let mut nix_build_succeeded = true;
|
|
||||||
let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| {
|
|
||||||
// Run `nix-build` to "build" each dependency (which will likely reuse
|
|
||||||
// the existing `/nix/store` copy, or at most download a pre-built copy).
|
|
||||||
//
|
|
||||||
// Importantly, we create a gc-root called `.nix-deps` in the `build/`
|
|
||||||
// directory, but still reference the actual `/nix/store` path in the rpath
|
|
||||||
// as it makes it significantly more robust against changes to the location of
|
|
||||||
// the `.nix-deps` location.
|
|
||||||
//
|
|
||||||
// bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`).
|
|
||||||
// zlib: Needed as a system dependency of `libLLVM-*.so`.
|
|
||||||
// patchelf: Needed for patching ELF binaries (see doc comment above).
|
|
||||||
let nix_deps_dir = builder.out.join(".nix-deps");
|
|
||||||
const NIX_EXPR: &str = "
|
|
||||||
with (import <nixpkgs> {});
|
|
||||||
symlinkJoin {
|
|
||||||
name = \"rust-stage0-dependencies\";
|
|
||||||
paths = [
|
|
||||||
zlib
|
|
||||||
patchelf
|
|
||||||
stdenv.cc.bintools
|
|
||||||
];
|
|
||||||
}
|
|
||||||
";
|
|
||||||
nix_build_succeeded = builder.try_run(Command::new("nix-build").args(&[
|
|
||||||
Path::new("-E"),
|
|
||||||
Path::new(NIX_EXPR),
|
|
||||||
Path::new("-o"),
|
|
||||||
&nix_deps_dir,
|
|
||||||
]));
|
|
||||||
nix_deps_dir
|
|
||||||
});
|
|
||||||
if !nix_build_succeeded {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf"));
|
|
||||||
let rpath_entries = {
|
|
||||||
// ORIGIN is a relative default, all binary and dynamic libraries we ship
|
|
||||||
// appear to have this (even when `../lib` is redundant).
|
|
||||||
// NOTE: there are only two paths here, delimited by a `:`
|
|
||||||
let mut entries = OsString::from("$ORIGIN/../lib:");
|
|
||||||
entries.push(t!(fs::canonicalize(nix_deps_dir)));
|
|
||||||
entries.push("/lib");
|
|
||||||
entries
|
|
||||||
};
|
|
||||||
patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]);
|
|
||||||
if !fname.extension().map_or(false, |ext| ext == "so") {
|
|
||||||
// Finally, set the corret .interp for binaries
|
|
||||||
let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker");
|
|
||||||
// FIXME: can we support utf8 here? `args` doesn't accept Vec<u8>, only OsString ...
|
|
||||||
let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path))));
|
|
||||||
patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]);
|
|
||||||
}
|
|
||||||
|
|
||||||
builder.try_run(patchelf.arg(fname));
|
|
||||||
}
|
|
||||||
|
|
||||||
fn download_component(builder: &Builder<'_>, base: &str, url: &str, dest_path: &Path) {
|
|
||||||
// Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
|
|
||||||
let tempfile = builder.tempdir().join(dest_path.file_name().unwrap());
|
|
||||||
// FIXME: support `do_verify` (only really needed for nightly rustfmt)
|
|
||||||
// FIXME: support non-utf8 paths?
|
|
||||||
download_with_retries(builder, tempfile.to_str().unwrap(), &format!("{}/{}", base, url));
|
|
||||||
t!(std::fs::rename(&tempfile, dest_path));
|
|
||||||
}
|
|
||||||
|
|
||||||
fn download_with_retries(builder: &Builder<'_>, tempfile: &str, url: &str) {
|
|
||||||
println!("downloading {}", url);
|
|
||||||
// Try curl. If that fails and we are on windows, fallback to PowerShell.
|
|
||||||
if !builder.check_run(Command::new("curl").args(&[
|
|
||||||
"-#",
|
|
||||||
"-y",
|
|
||||||
"30",
|
|
||||||
"-Y",
|
|
||||||
"10", // timeout if speed is < 10 bytes/sec for > 30 seconds
|
|
||||||
"--connect-timeout",
|
|
||||||
"30", // timeout if cannot connect within 30 seconds
|
|
||||||
"--retry",
|
|
||||||
"3",
|
|
||||||
"-Sf",
|
|
||||||
"-o",
|
|
||||||
tempfile,
|
|
||||||
url,
|
|
||||||
])) {
|
|
||||||
if builder.build.build.contains("windows-msvc") {
|
|
||||||
println!("Fallback to PowerShell");
|
|
||||||
for _ in 0..3 {
|
|
||||||
if builder.try_run(Command::new("PowerShell.exe").args(&[
|
|
||||||
"/nologo",
|
|
||||||
"-Command",
|
|
||||||
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;",
|
|
||||||
&format!(
|
|
||||||
"(New-Object System.Net.WebClient).DownloadFile('{}', '{}')",
|
|
||||||
url, tempfile
|
|
||||||
),
|
|
||||||
])) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
println!("\nspurious failure, trying again");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
std::process::exit(1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn unpack(builder: &Builder<'_>, tarball: &Path, dst: &Path) {
|
|
||||||
println!("extracting {} to {}", tarball.display(), dst.display());
|
|
||||||
if !dst.exists() {
|
|
||||||
t!(fs::create_dir_all(dst));
|
|
||||||
}
|
|
||||||
|
|
||||||
// FIXME: will need to be a parameter once `download-rustc` is moved to rustbuild
|
|
||||||
const MATCH: &str = "rust-dev";
|
|
||||||
|
|
||||||
// `tarball` ends with `.tar.xz`; strip that suffix
|
|
||||||
// example: `rust-dev-nightly-x86_64-unknown-linux-gnu`
|
|
||||||
let uncompressed_filename =
|
|
||||||
Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap();
|
|
||||||
let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap());
|
|
||||||
|
|
||||||
// decompress the file
|
|
||||||
let data = t!(File::open(tarball));
|
|
||||||
let decompressor = XzDecoder::new(BufReader::new(data));
|
|
||||||
|
|
||||||
let mut tar = tar::Archive::new(decompressor);
|
|
||||||
for member in t!(tar.entries()) {
|
|
||||||
let mut member = t!(member);
|
|
||||||
let original_path = t!(member.path()).into_owned();
|
|
||||||
// skip the top-level directory
|
|
||||||
if original_path == directory_prefix {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
let mut short_path = t!(original_path.strip_prefix(directory_prefix));
|
|
||||||
if !short_path.starts_with(MATCH) {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
short_path = t!(short_path.strip_prefix(MATCH));
|
|
||||||
let dst_path = dst.join(short_path);
|
|
||||||
builder.verbose(&format!("extracting {} to {}", original_path.display(), dst.display()));
|
|
||||||
if !t!(member.unpack_in(dst)) {
|
|
||||||
panic!("path traversal attack ??");
|
|
||||||
}
|
|
||||||
let src_path = dst.join(original_path);
|
|
||||||
if src_path.is_dir() && dst_path.exists() {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
t!(fs::rename(src_path, dst_path));
|
|
||||||
}
|
|
||||||
t!(fs::remove_dir_all(dst.join(directory_prefix)));
|
|
||||||
}
|
|
||||||
|
|
||||||
fn program_out_of_date(stamp: &Path, key: &str) -> bool {
|
|
||||||
if !stamp.exists() {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
t!(fs::read_to_string(stamp)) != key
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
|
#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
|
||||||
|
|
|
@ -2058,7 +2058,7 @@ impl Step for CrateRustdoc {
|
||||||
let test_kind = self.test_kind;
|
let test_kind = self.test_kind;
|
||||||
let target = self.host;
|
let target = self.host;
|
||||||
|
|
||||||
let compiler = if builder.config.download_rustc {
|
let compiler = if builder.download_rustc() {
|
||||||
builder.compiler(builder.top_stage, target)
|
builder.compiler(builder.top_stage, target)
|
||||||
} else {
|
} else {
|
||||||
// Use the previous stage compiler to reuse the artifacts that are
|
// Use the previous stage compiler to reuse the artifacts that are
|
||||||
|
@ -2127,7 +2127,7 @@ impl Step for CrateRustdoc {
|
||||||
// with.
|
// with.
|
||||||
//
|
//
|
||||||
// Note that this set the host libdir for `download_rustc`, which uses a normal rust distribution.
|
// Note that this set the host libdir for `download_rustc`, which uses a normal rust distribution.
|
||||||
let libdir = if builder.config.download_rustc {
|
let libdir = if builder.download_rustc() {
|
||||||
builder.rustc_libdir(compiler)
|
builder.rustc_libdir(compiler)
|
||||||
} else {
|
} else {
|
||||||
builder.sysroot_libdir(compiler, target).to_path_buf()
|
builder.sysroot_libdir(compiler, target).to_path_buf()
|
||||||
|
|
|
@ -516,7 +516,7 @@ impl Step for Rustdoc {
|
||||||
builder.ensure(compile::Rustc { compiler: build_compiler, target: target_compiler.host });
|
builder.ensure(compile::Rustc { compiler: build_compiler, target: target_compiler.host });
|
||||||
// NOTE: this implies that `download-rustc` is pretty useless when compiling with the stage0
|
// NOTE: this implies that `download-rustc` is pretty useless when compiling with the stage0
|
||||||
// compiler, since you do just as much work.
|
// compiler, since you do just as much work.
|
||||||
if !builder.config.dry_run && builder.config.download_rustc && build_compiler.stage == 0 {
|
if !builder.config.dry_run && builder.download_rustc() && build_compiler.stage == 0 {
|
||||||
println!(
|
println!(
|
||||||
"warning: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead"
|
"warning: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead"
|
||||||
);
|
);
|
||||||
|
|
|
@ -115,6 +115,14 @@ impl Drop for TimeIt {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Used for download caching
|
||||||
|
pub(crate) fn program_out_of_date(stamp: &Path, key: &str) -> bool {
|
||||||
|
if !stamp.exists() {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
t!(fs::read_to_string(stamp)) != key
|
||||||
|
}
|
||||||
|
|
||||||
/// Symlinks two directories, using junctions on Windows and normal symlinks on
|
/// Symlinks two directories, using junctions on Windows and normal symlinks on
|
||||||
/// Unix.
|
/// Unix.
|
||||||
pub fn symlink_dir(config: &Config, src: &Path, dest: &Path) -> io::Result<()> {
|
pub fn symlink_dir(config: &Config, src: &Path, dest: &Path) -> io::Result<()> {
|
||||||
|
|
Loading…
Reference in New Issue