From 754f488d46fa33387c3dca3b08d3f3ec2fe02d3f Mon Sep 17 00:00:00 2001 From: Scott McMurray Date: Sat, 26 Aug 2023 17:42:59 -0700 Subject: [PATCH] Use `preserve_mostcc` for `extern "rust-cold"` As experimentation in 115242 has shown looks better than `coldcc`. And *don't* use a different convention for cold on Windows, because that actually ends up making things worse. cc tracking issue 97544 --- .../rustc_codegen_cranelift/src/abi/mod.rs | 2 +- compiler/rustc_codegen_llvm/src/abi.rs | 4 +- compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 5 +++ compiler/rustc_target/src/abi/call/mod.rs | 11 ++--- compiler/rustc_target/src/json.rs | 4 +- compiler/rustc_target/src/spec/abi.rs | 43 ++++++++++++++----- compiler/rustc_target/src/spec/mod.rs | 7 +++ compiler/rustc_ty_utils/src/abi.rs | 5 ++- tests/codegen/cold-call-declare-and-call.rs | 13 +++++- 9 files changed, 71 insertions(+), 23 deletions(-) diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index b7f56a2986c..5d775b9b532 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -39,7 +39,7 @@ fn clif_sig_from_fn_abi<'tcx>( pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: CallConv) -> CallConv { match c { Conv::Rust | Conv::C => default_call_conv, - Conv::RustCold => CallConv::Cold, + Conv::Cold | Conv::PreserveMost | Conv::PreserveAll => CallConv::Cold, Conv::X86_64SysV => CallConv::SystemV, Conv::X86_64Win64 => CallConv::WindowsFastcall, diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs index c6a7dc95d77..863cb7068f8 100644 --- a/compiler/rustc_codegen_llvm/src/abi.rs +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -571,7 +571,9 @@ impl From for llvm::CallConv { Conv::C | Conv::Rust | Conv::CCmseNonSecureCall | Conv::RiscvInterrupt { .. } => { llvm::CCallConv } - Conv::RustCold => llvm::ColdCallConv, + Conv::Cold => llvm::ColdCallConv, + Conv::PreserveMost => llvm::PreserveMost, + Conv::PreserveAll => llvm::PreserveAll, Conv::AmdGpuKernel => llvm::AmdGpuKernel, Conv::AvrInterrupt => llvm::AvrInterrupt, Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 84157d1e25c..01cbf7d3b11 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -83,12 +83,17 @@ pub enum LLVMModFlagBehavior { // Consts for the LLVM CallConv type, pre-cast to usize. /// LLVM CallingConv::ID. Should we wrap this? +/// +/// See #[derive(Copy, Clone, PartialEq, Debug)] #[repr(C)] pub enum CallConv { CCallConv = 0, FastCallConv = 8, ColdCallConv = 9, + PreserveMost = 14, + PreserveAll = 15, + Tail = 18, X86StdcallCallConv = 64, X86FastcallCallConv = 65, ArmAapcsCallConv = 67, diff --git a/compiler/rustc_target/src/abi/call/mod.rs b/compiler/rustc_target/src/abi/call/mod.rs index 8fab13d5d5d..8d573def9bb 100644 --- a/compiler/rustc_target/src/abi/call/mod.rs +++ b/compiler/rustc_target/src/abi/call/mod.rs @@ -579,10 +579,9 @@ pub enum Conv { C, Rust, - /// For things unlikely to be called, where smaller caller codegen is - /// preferred over raw speed. - /// Stronger than just `#[cold]` because `fn` pointers might be incompatible. - RustCold, + Cold, + PreserveMost, + PreserveAll, // Target-specific calling conventions. ArmAapcs, @@ -605,9 +604,7 @@ pub enum Conv { AvrInterrupt, AvrNonBlockingInterrupt, - RiscvInterrupt { - kind: RiscvInterruptKind, - }, + RiscvInterrupt { kind: RiscvInterruptKind }, } #[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, HashStable_Generic)] diff --git a/compiler/rustc_target/src/json.rs b/compiler/rustc_target/src/json.rs index af455b6432f..c6135149081 100644 --- a/compiler/rustc_target/src/json.rs +++ b/compiler/rustc_target/src/json.rs @@ -96,7 +96,9 @@ impl ToJson for crate::abi::call::Conv { let s = match self { Self::C => "C", Self::Rust => "Rust", - Self::RustCold => "RustCold", + Self::Cold => "Cold", + Self::PreserveMost => "PreserveMost", + Self::PreserveAll => "PreserveAll", Self::ArmAapcs => "ArmAapcs", Self::CCmseNonSecureCall => "CCmseNonSecureCall", Self::Msp430Intr => "Msp430Intr", diff --git a/compiler/rustc_target/src/spec/abi.rs b/compiler/rustc_target/src/spec/abi.rs index 550cdf6bda6..956a5cb5c2f 100644 --- a/compiler/rustc_target/src/spec/abi.rs +++ b/compiler/rustc_target/src/spec/abi.rs @@ -14,15 +14,33 @@ pub enum Abi { // hashing tests. These are used in many places, so giving them stable values reduces test // churn. The specific values are meaningless. Rust, - C { unwind: bool }, - Cdecl { unwind: bool }, - Stdcall { unwind: bool }, - Fastcall { unwind: bool }, - Vectorcall { unwind: bool }, - Thiscall { unwind: bool }, - Aapcs { unwind: bool }, - Win64 { unwind: bool }, - SysV64 { unwind: bool }, + C { + unwind: bool, + }, + Cdecl { + unwind: bool, + }, + Stdcall { + unwind: bool, + }, + Fastcall { + unwind: bool, + }, + Vectorcall { + unwind: bool, + }, + Thiscall { + unwind: bool, + }, + Aapcs { + unwind: bool, + }, + Win64 { + unwind: bool, + }, + SysV64 { + unwind: bool, + }, PtxKernel, Msp430Interrupt, X86Interrupt, @@ -32,11 +50,16 @@ pub enum Abi { AvrNonBlockingInterrupt, CCmseNonSecureCall, Wasm, - System { unwind: bool }, + System { + unwind: bool, + }, RustIntrinsic, RustCall, PlatformIntrinsic, Unadjusted, + /// For things unlikely to be called, where reducing register pressure in + /// `extern "Rust"` callers is worth paying extra cost in the callee. + /// Stronger than just `#[cold]` because `fn` pointers might be incompatible. RustCold, RiscvInterruptM, RiscvInterruptS, diff --git a/compiler/rustc_target/src/spec/mod.rs b/compiler/rustc_target/src/spec/mod.rs index f8d40c4142d..b3cf76d2257 100644 --- a/compiler/rustc_target/src/spec/mod.rs +++ b/compiler/rustc_target/src/spec/mod.rs @@ -2276,6 +2276,13 @@ impl Target { Abi::Vectorcall { .. } if ["x86", "x86_64"].contains(&&self.arch[..]) => abi, Abi::Fastcall { unwind } | Abi::Vectorcall { unwind } => Abi::C { unwind }, + // The Windows x64 calling convention we use for `extern "Rust"` + // + // expects the callee to save `xmm6` through `xmm15`, but `PreserveMost` + // (that we use by default for `extern "rust-cold"`) doesn't save any of those. + // So to avoid bloating callers, just use the Rust convention here. + Abi::RustCold if self.is_like_windows && self.arch == "x86_64" => Abi::Rust, + abi => abi, } } diff --git a/compiler/rustc_ty_utils/src/abi.rs b/compiler/rustc_ty_utils/src/abi.rs index 4d0b847533b..a03da41652c 100644 --- a/compiler/rustc_ty_utils/src/abi.rs +++ b/compiler/rustc_ty_utils/src/abi.rs @@ -172,7 +172,10 @@ fn conv_from_spec_abi(tcx: TyCtxt<'_>, abi: SpecAbi) -> Conv { use rustc_target::spec::abi::Abi::*; match tcx.sess.target.adjust_abi(abi) { RustIntrinsic | PlatformIntrinsic | Rust | RustCall => Conv::Rust, - RustCold => Conv::RustCold, + + // This is intentionally not using `Conv::Cold`, as that has to preserve + // even SIMD registers, which is generally not a good trade-off. + RustCold => Conv::PreserveMost, // It's the ABI's job to select this, not ours. System { .. } => bug!("system abi should be selected elsewhere"), diff --git a/tests/codegen/cold-call-declare-and-call.rs b/tests/codegen/cold-call-declare-and-call.rs index 71d49478bfc..572dc407f51 100644 --- a/tests/codegen/cold-call-declare-and-call.rs +++ b/tests/codegen/cold-call-declare-and-call.rs @@ -1,12 +1,21 @@ +// revisions: NORMAL WINDOWS // compile-flags: -C no-prepopulate-passes +//[NORMAL] ignore-windows +//[WINDOWS] only-windows +//[WINDOWS] only-x86_64 #![crate_type = "lib"] #![feature(rust_cold_cc)] // wasm marks the definition as `dso_local`, so allow that as optional. -// CHECK: define{{( dso_local)?}} coldcc void @this_should_never_happen(i16 -// CHECK: call coldcc void @this_should_never_happen(i16 +// NORMAL: define{{( dso_local)?}} preserve_mostcc void @this_should_never_happen(i16 +// NORMAL: call preserve_mostcc void @this_should_never_happen(i16 + +// See the comment in `Target::adjust_abi` for why this differs + +// WINDOWS: define void @this_should_never_happen(i16 +// WINDOWS: call void @this_should_never_happen(i16 #[no_mangle] pub extern "rust-cold" fn this_should_never_happen(x: u16) {}