dnrops
/
rust
mirror of https://github.com/rust-lang/rust.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Auto merge of #119911 - NCGThompson:is-statically-known, r=oli-obk 

Replacement of #114390: Add new intrinsic `is_var_statically_known` and optimize pow for powers of two

This adds a new intrinsic `is_val_statically_known` that lowers to [``@llvm.is.constant.*`](https://llvm.org/docs/LangRef.html#llvm-is-constant-intrinsic).` It also applies the intrinsic in the int_pow methods to recognize and optimize the idiom `2isize.pow(x)`. See #114390 for more discussion.

While I have extended the scope of the power of two optimization from #114390, I haven't added any new uses for the intrinsic. That can be done in later pull requests.

Note: When testing or using the library, be sure to use `--stage 1` or higher. Otherwise, the intrinsic will be a noop and the doctests will be skipped. If you are trying out edits, you may be interested in [`--keep-stage 0`](https://rustc-dev-guide.rust-lang.org/building/suggested.html#faster-builds-with---keep-stage).

Fixes #47234
Resolves #114390
`@Centri3`
This commit is contained in:
bors 2024-01-25 05:16:53 +00:00
parent 68411c9554 9dccd5dce1
commit 039d887928
17 changed files with 634 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View File

@ -443,6 +443,12 @@ fn codegen_regular_intrinsic_call<'tcx>(
ret.write_cvalue(fx, a);
}
sym::is_val_statically_known => {
intrinsic_args!(fx, args => (_a); intrinsic);
let res = fx.bcx.ins().iconst(types::I8, 0);
ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
}
sym::breakpoint => {
intrinsic_args!(fx, args => (); intrinsic);

7
compiler/rustc_codegen_gcc/src/context.rs
View File

@ -196,15 +196,16 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> {
let mut functions = FxHashMap::default();
let builtins = [
"__builtin_unreachable", "abort", "__builtin_expect", "__builtin_add_overflow", "__builtin_mul_overflow",
"__builtin_saddll_overflow", /*"__builtin_sadd_overflow",*/ "__builtin_smulll_overflow", /*"__builtin_smul_overflow",*/
"__builtin_unreachable", "abort", "__builtin_expect", /*"__builtin_expect_with_probability",*/
"__builtin_constant_p", "__builtin_add_overflow", "__builtin_mul_overflow", "__builtin_saddll_overflow",
/*"__builtin_sadd_overflow",*/ "__builtin_smulll_overflow", /*"__builtin_smul_overflow",*/
"__builtin_ssubll_overflow", /*"__builtin_ssub_overflow",*/ "__builtin_sub_overflow", "__builtin_uaddll_overflow",
"__builtin_uadd_overflow", "__builtin_umulll_overflow", "__builtin_umul_overflow", "__builtin_usubll_overflow",
"__builtin_usub_overflow", "sqrtf", "sqrt", "__builtin_powif", "__builtin_powi", "sinf", "sin", "cosf", "cos",
"powf", "pow", "expf", "exp", "exp2f", "exp2", "logf", "log", "log10f", "log10", "log2f", "log2", "fmaf",
"fma", "fabsf", "fabs", "fminf", "fmin", "fmaxf", "fmax", "copysignf", "copysign", "floorf", "floor", "ceilf",
"ceil", "truncf", "trunc", "rintf", "rint", "nearbyintf", "nearbyint", "roundf", "round",
"__builtin_expect_with_probability",
];
for builtin in builtins.iter() {

6
compiler/rustc_codegen_gcc/src/intrinsic/mod.rs
View File

@ -123,6 +123,12 @@ impl<'a, 'gcc, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
sym::unlikely => {
self.expect(args[0].immediate(), false)
}
sym::is_val_statically_known => {
let a = args[0].immediate();
let builtin = self.context.get_builtin_function("__builtin_constant_p");
let res = self.context.new_call(None, builtin, &[a]);
self.icmp(IntPredicate::IntEQ, res, self.const_i32(0))
}
kw::Try => {
try_intrinsic(
self,

14
compiler/rustc_codegen_llvm/src/context.rs
View File

@ -916,6 +916,20 @@ impl<'ll> CodegenCx<'ll, '_> {
ifn!("llvm.lifetime.start.p0i8", fn(t_i64, ptr) -> void);
ifn!("llvm.lifetime.end.p0i8", fn(t_i64, ptr) -> void);
// FIXME: This is an infinitesimally small portion of the types you can
// pass to this intrinsic, if we can ever lazily register intrinsics we
// should register these when they're used, that way any type can be
// passed.
ifn!("llvm.is.constant.i1", fn(i1) -> i1);
ifn!("llvm.is.constant.i8", fn(t_i8) -> i1);
ifn!("llvm.is.constant.i16", fn(t_i16) -> i1);
ifn!("llvm.is.constant.i32", fn(t_i32) -> i1);
ifn!("llvm.is.constant.i64", fn(t_i64) -> i1);
ifn!("llvm.is.constant.i128", fn(t_i128) -> i1);
ifn!("llvm.is.constant.isize", fn(t_isize) -> i1);
ifn!("llvm.is.constant.f32", fn(t_f32) -> i1);
ifn!("llvm.is.constant.f64", fn(t_f64) -> i1);
ifn!("llvm.expect.i1", fn(i1, i1) -> i1);
ifn!("llvm.eh.typeid.for", fn(ptr) -> t_i32);
ifn!("llvm.localescape", fn(...) -> void);

4
compiler/rustc_codegen_llvm/src/intrinsic.rs
View File

@ -119,6 +119,10 @@ impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> {
sym::likely => {
self.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)])
}
sym::is_val_statically_known => self.call_intrinsic(
&format!("llvm.is.constant.{:?}", args[0].layout.immediate_llvm_type(self.cx)),
&[args[0].immediate()],
),
sym::unlikely => self
.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]),
kw::Try => {

5
compiler/rustc_const_eval/src/const_eval/machine.rs
View File

@ -531,6 +531,11 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir,
)?;
}
}
// The intrinsic represents whether the value is known to the optimizer (LLVM).
// We're not doing any optimizations here, so there is no optimizer that could know the value.
// (We know the value here in the machine of course, but this is the runtime of that code,
// not the optimization stage.)
sym::is_val_statically_known => ecx.write_scalar(Scalar::from_bool(false), dest)?,
_ => {
throw_unsup_format!(
"intrinsic `{intrinsic_name}` is not supported at compile-time"

2
compiler/rustc_hir_analysis/src/check/intrinsic.rs
View File

@ -453,6 +453,8 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
sym::black_box => (1, vec![param(0)], param(0)),
sym::is_val_statically_known => (1, vec![param(0)], tcx.types.bool),
sym::const_eval_select => (4, vec![param(0), param(1), param(2)], param(3)),
sym::vtable_size | sym::vtable_align => {

1
compiler/rustc_span/src/symbol.rs
View File

@ -910,6 +910,7 @@ symbols! {
io_stderr,
io_stdout,
irrefutable_let_patterns,
is_val_statically_known,
isa_attribute,
isize,
issue,

60
library/core/src/intrinsics.rs
View File

@ -2517,6 +2517,66 @@ extern "rust-intrinsic" {
where
G: FnOnce<ARG, Output = RET>,
F: FnOnce<ARG, Output = RET>;
/// Returns whether the argument's value is statically known at
/// compile-time.
///
/// This is useful when there is a way of writing the code that will
/// be *faster* when some variables have known values, but *slower*
/// in the general case: an `if is_val_statically_known(var)` can be used
/// to select between these two variants. The `if` will be optimized away
/// and only the desired branch remains.
///
/// Formally speaking, this function non-deterministically returns `true`
/// or `false`, and the caller has to ensure sound behavior for both cases.
/// In other words, the following code has *Undefined Behavior*:
///
/// ```
/// #![feature(is_val_statically_known)]
/// #![feature(core_intrinsics)]
/// # #![allow(internal_features)]
/// use std::hint::unreachable_unchecked;
/// use std::intrinsics::is_val_statically_known;
///
/// unsafe {
/// if !is_val_statically_known(0) { unreachable_unchecked(); }
/// }
/// ```
///
/// This also means that the following code's behavior is unspecified; it
/// may panic, or it may not:
///
/// ```no_run
/// #![feature(is_val_statically_known)]
/// #![feature(core_intrinsics)]
/// # #![allow(internal_features)]
/// use std::intrinsics::is_val_statically_known;
///
/// unsafe {
/// assert_eq!(is_val_statically_known(0), is_val_statically_known(0));
/// }
/// ```
///
/// Unsafe code may not rely on `is_val_statically_known` returning any
/// particular value, ever. However, the compiler will generally make it
/// return `true` only if the value of the argument is actually known.
///
/// When calling this in a `const fn`, both paths must be semantically
/// equivalent, that is, the result of the `true` branch and the `false`
/// branch must return the same value and have the same side-effects *no
/// matter what*.
#[rustc_const_unstable(feature = "is_val_statically_known", issue = "none")]
#[rustc_nounwind]
#[cfg(not(bootstrap))]
pub fn is_val_statically_known<T: Copy>(arg: T) -> bool;
}
// FIXME: Seems using `unstable` here completely ignores `rustc_allow_const_fn_unstable`
// and thus compiling stage0 core doesn't work.
#[rustc_const_stable(feature = "is_val_statically_known", since = "0.0.0")]
#[cfg(bootstrap)]
pub const unsafe fn is_val_statically_known<T: Copy>(_arg: T) -> bool {
false
}
// Some functions are defined here because they accidentally got made

1
library/core/src/lib.rs
View File

@ -200,6 +200,7 @@
//
// Language features:
// tidy-alphabetical-start
#![cfg_attr(not(bootstrap), feature(is_val_statically_known))]
#![feature(abi_unadjusted)]
#![feature(adt_const_params)]
#![feature(allow_internal_unsafe)]

284
library/core/src/num/int_macros.rs
View File

@ -1374,26 +1374,59 @@ macro_rules! int_impl {
#[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")]
#[must_use = "this returns the result of the operation, \
without modifying the original"]
#[rustc_allow_const_fn_unstable(is_val_statically_known, const_int_unchecked_arith)]
#[inline]
pub const fn checked_pow(self, mut exp: u32) -> Option<Self> {
if exp == 0 {
return Some(1);
}
let mut base = self;
let mut acc: Self = 1;
while exp > 1 {
if (exp & 1) == 1 {
acc = try_opt!(acc.checked_mul(base));
// SAFETY: This path has the same behavior as the other.
if unsafe { intrinsics::is_val_statically_known(self) }
&& self.unsigned_abs().is_power_of_two()
{
if self == 1 { // Avoid divide by zero
return Some(1);
}
exp /= 2;
base = try_opt!(base.checked_mul(base));
if self == -1 { // Avoid divide by zero
return Some(if exp & 1 != 0 { -1 } else { 1 });
}
// SAFETY: We just checked this is a power of two. and above zero.
let power_used = unsafe { intrinsics::cttz_nonzero(self.wrapping_abs()) as u32 };
if exp > Self::BITS / power_used { return None; } // Division of constants is free
// SAFETY: exp <= Self::BITS / power_used
let res = unsafe { intrinsics::unchecked_shl(
1 as Self,
intrinsics::unchecked_mul(power_used, exp) as Self
)};
// LLVM doesn't always optimize out the checks
// at the ir level.
let sign = self.is_negative() && exp & 1 != 0;
if !sign && res == Self::MIN {
None
} else if sign {
Some(res.wrapping_neg())
} else {
Some(res)
}
} else {
if exp == 0 {
return Some(1);
}
let mut base = self;
let mut acc: Self = 1;
while exp > 1 {
if (exp & 1) == 1 {
acc = try_opt!(acc.checked_mul(base));
}
exp /= 2;
base = try_opt!(base.checked_mul(base));
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.checked_mul(base)
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.checked_mul(base)
}
/// Strict exponentiation. Computes `self.pow(exp)`, panicking if
@ -2058,27 +2091,58 @@ macro_rules! int_impl {
#[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")]
#[must_use = "this returns the result of the operation, \
without modifying the original"]
#[rustc_allow_const_fn_unstable(is_val_statically_known, const_int_unchecked_arith)]
#[inline]
pub const fn wrapping_pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc: Self = 1;
while exp > 1 {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
// SAFETY: This path has the same behavior as the other.
if unsafe { intrinsics::is_val_statically_known(self) }
&& self.unsigned_abs().is_power_of_two()
{
if self == 1 { // Avoid divide by zero
return 1;
}
exp /= 2;
base = base.wrapping_mul(base);
}
if self == -1 { // Avoid divide by zero
return if exp & 1 != 0 { -1 } else { 1 };
}
// SAFETY: We just checked this is a power of two. and above zero.
let power_used = unsafe { intrinsics::cttz_nonzero(self.wrapping_abs()) as u32 };
if exp > Self::BITS / power_used { return 0; } // Division of constants is free
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.wrapping_mul(base)
// SAFETY: exp <= Self::BITS / power_used
let res = unsafe { intrinsics::unchecked_shl(
1 as Self,
intrinsics::unchecked_mul(power_used, exp) as Self
)};
// LLVM doesn't always optimize out the checks
// at the ir level.
let sign = self.is_negative() && exp & 1 != 0;
if sign {
res.wrapping_neg()
} else {
res
}
} else {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc: Self = 1;
while exp > 1 {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
}
exp /= 2;
base = base.wrapping_mul(base);
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.wrapping_mul(base)
}
}
/// Calculates `self` + `rhs`
@ -2561,36 +2625,68 @@ macro_rules! int_impl {
#[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")]
#[must_use = "this returns the result of the operation, \
without modifying the original"]
#[rustc_allow_const_fn_unstable(is_val_statically_known, const_int_unchecked_arith)]
#[inline]
pub const fn overflowing_pow(self, mut exp: u32) -> (Self, bool) {
if exp == 0 {
return (1,false);
}
let mut base = self;
let mut acc: Self = 1;
let mut overflown = false;
// Scratch space for storing results of overflowing_mul.
let mut r;
// SAFETY: This path has the same behavior as the other.
if unsafe { intrinsics::is_val_statically_known(self) }
&& self.unsigned_abs().is_power_of_two()
{
if self == 1 { // Avoid divide by zero
return (1, false);
}
if self == -1 { // Avoid divide by zero
return (if exp & 1 != 0 { -1 } else { 1 }, false);
}
// SAFETY: We just checked this is a power of two. and above zero.
let power_used = unsafe { intrinsics::cttz_nonzero(self.wrapping_abs()) as u32 };
if exp > Self::BITS / power_used { return (0, true); } // Division of constants is free
while exp > 1 {
if (exp & 1) == 1 {
r = acc.overflowing_mul(base);
acc = r.0;
// SAFETY: exp <= Self::BITS / power_used
let res = unsafe { intrinsics::unchecked_shl(
1 as Self,
intrinsics::unchecked_mul(power_used, exp) as Self
)};
// LLVM doesn't always optimize out the checks
// at the ir level.
let sign = self.is_negative() && exp & 1 != 0;
let overflow = res == Self::MIN;
if sign {
(res.wrapping_neg(), overflow)
} else {
(res, overflow)
}
} else {
if exp == 0 {
return (1,false);
}
let mut base = self;
let mut acc: Self = 1;
let mut overflown = false;
// Scratch space for storing results of overflowing_mul.
let mut r;
while exp > 1 {
if (exp & 1) == 1 {
r = acc.overflowing_mul(base);
acc = r.0;
overflown |= r.1;
}
exp /= 2;
r = base.overflowing_mul(base);
base = r.0;
overflown |= r.1;
}
exp /= 2;
r = base.overflowing_mul(base);
base = r.0;
overflown |= r.1;
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
r = acc.overflowing_mul(base);
r.1 |= overflown;
r
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
r = acc.overflowing_mul(base);
r.1 |= overflown;
r
}
}
/// Raises self to the power of `exp`, using exponentiation by squaring.
@ -2608,28 +2704,68 @@ macro_rules! int_impl {
#[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")]
#[must_use = "this returns the result of the operation, \
without modifying the original"]
#[rustc_allow_const_fn_unstable(is_val_statically_known, const_int_unchecked_arith)]
#[inline]
#[rustc_inherit_overflow_checks]
#[track_caller] // Hides the hackish overflow check for powers of two.
pub const fn pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc = 1;
while exp > 1 {
if (exp & 1) == 1 {
acc = acc * base;
// SAFETY: This path has the same behavior as the other.
if unsafe { intrinsics::is_val_statically_known(self) }
&& self.unsigned_abs().is_power_of_two()
{
if self == 1 { // Avoid divide by zero
return 1;
}
if self == -1 { // Avoid divide by zero
return if exp & 1 != 0 { -1 } else { 1 };
}
// SAFETY: We just checked this is a power of two. and above zero.
let power_used = unsafe { intrinsics::cttz_nonzero(self.wrapping_abs()) as u32 };
if exp > Self::BITS / power_used { // Division of constants is free
#[allow(arithmetic_overflow)]
return Self::MAX * Self::MAX * 0;
}
exp /= 2;
base = base * base;
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc * base
// SAFETY: exp <= Self::BITS / power_used
let res = unsafe { intrinsics::unchecked_shl(
1 as Self,
intrinsics::unchecked_mul(power_used, exp) as Self
)};
// LLVM doesn't always optimize out the checks
// at the ir level.
let sign = self.is_negative() && exp & 1 != 0;
#[allow(arithmetic_overflow)]
if !sign && res == Self::MIN {
// So it panics.
_ = Self::MAX * Self::MAX;
}
if sign {
res.wrapping_neg()
} else {
res
}
} else {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc = 1;
while exp > 1 {
if (exp & 1) == 1 {
acc = acc * base;
}
exp /= 2;
base = base * base;
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc * base
}
}
/// Returns the square root of the number, rounded down.

253
library/core/src/num/uint_macros.rs
View File

@ -1364,28 +1364,49 @@ macro_rules! uint_impl {
#[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")]
#[must_use = "this returns the result of the operation, \
without modifying the original"]
#[rustc_allow_const_fn_unstable(is_val_statically_known, const_int_unchecked_arith)]
#[inline]
pub const fn checked_pow(self, mut exp: u32) -> Option<Self> {
if exp == 0 {
return Some(1);
}
let mut base = self;
let mut acc: Self = 1;
while exp > 1 {
if (exp & 1) == 1 {
acc = try_opt!(acc.checked_mul(base));
// SAFETY: This path has the same behavior as the other.
if unsafe { intrinsics::is_val_statically_known(self) }
&& self.is_power_of_two()
{
if self == 1 { // Avoid divide by zero
return Some(1);
}
exp /= 2;
base = try_opt!(base.checked_mul(base));
// SAFETY: We just checked this is a power of two. and above zero.
let power_used = unsafe { intrinsics::cttz_nonzero(self) as u32 };
if exp > Self::BITS / power_used { return None; } // Division of constants is free
// SAFETY: exp <= Self::BITS / power_used
unsafe { Some(intrinsics::unchecked_shl(
1 as Self,
intrinsics::unchecked_mul(power_used, exp) as Self
)) }
// LLVM doesn't always optimize out the checks
// at the ir level.
} else {
if exp == 0 {
return Some(1);
}
let mut base = self;
let mut acc: Self = 1;
while exp > 1 {
if (exp & 1) == 1 {
acc = try_opt!(acc.checked_mul(base));
}
exp /= 2;
base = try_opt!(base.checked_mul(base));
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.checked_mul(base)
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.checked_mul(base)
}
/// Strict exponentiation. Computes `self.pow(exp)`, panicking if
@ -1887,27 +1908,48 @@ macro_rules! uint_impl {
#[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")]
#[must_use = "this returns the result of the operation, \
without modifying the original"]
#[rustc_allow_const_fn_unstable(is_val_statically_known, const_int_unchecked_arith)]
#[inline]
pub const fn wrapping_pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc: Self = 1;
while exp > 1 {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
// SAFETY: This path has the same behavior as the other.
if unsafe { intrinsics::is_val_statically_known(self) }
&& self.is_power_of_two()
{
if self == 1 { // Avoid divide by zero
return 1;
}
exp /= 2;
base = base.wrapping_mul(base);
}
// SAFETY: We just checked this is a power of two. and above zero.
let power_used = unsafe { intrinsics::cttz_nonzero(self) as u32 };
if exp > Self::BITS / power_used { return 0; } // Division of constants is free
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.wrapping_mul(base)
// SAFETY: exp <= Self::BITS / power_used
unsafe { intrinsics::unchecked_shl(
1 as Self,
intrinsics::unchecked_mul(power_used, exp) as Self
)}
// LLVM doesn't always optimize out the checks
// at the ir level.
} else {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc: Self = 1;
while exp > 1 {
if (exp & 1) == 1 {
acc = acc.wrapping_mul(base);
}
exp /= 2;
base = base.wrapping_mul(base);
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc.wrapping_mul(base)
}
}
/// Calculates `self` + `rhs`
@ -2341,37 +2383,58 @@ macro_rules! uint_impl {
#[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")]
#[must_use = "this returns the result of the operation, \
without modifying the original"]
#[rustc_allow_const_fn_unstable(is_val_statically_known, const_int_unchecked_arith)]
#[inline]
pub const fn overflowing_pow(self, mut exp: u32) -> (Self, bool) {
if exp == 0{
return (1,false);
}
let mut base = self;
let mut acc: Self = 1;
let mut overflown = false;
// Scratch space for storing results of overflowing_mul.
let mut r;
// SAFETY: This path has the same behavior as the other.
if unsafe { intrinsics::is_val_statically_known(self) }
&& self.is_power_of_two()
{
if self == 1 { // Avoid divide by zero
return (1, false);
}
// SAFETY: We just checked this is a power of two. and above zero.
let power_used = unsafe { intrinsics::cttz_nonzero(self) as u32 };
if exp > Self::BITS / power_used { return (0, true); } // Division of constants is free
while exp > 1 {
if (exp & 1) == 1 {
r = acc.overflowing_mul(base);
acc = r.0;
// SAFETY: exp <= Self::BITS / power_used
unsafe { (intrinsics::unchecked_shl(
1 as Self,
intrinsics::unchecked_mul(power_used, exp) as Self
), false) }
// LLVM doesn't always optimize out the checks
// at the ir level.
} else {
if exp == 0{
return (1,false);
}
let mut base = self;
let mut acc: Self = 1;
let mut overflown = false;
// Scratch space for storing results of overflowing_mul.
let mut r;
while exp > 1 {
if (exp & 1) == 1 {
r = acc.overflowing_mul(base);
acc = r.0;
overflown |= r.1;
}
exp /= 2;
r = base.overflowing_mul(base);
base = r.0;
overflown |= r.1;
}
exp /= 2;
r = base.overflowing_mul(base);
base = r.0;
overflown |= r.1;
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
r = acc.overflowing_mul(base);
r.1 |= overflown;
r
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
r = acc.overflowing_mul(base);
r.1 |= overflown;
r
}
/// Raises self to the power of `exp`, using exponentiation by squaring.
@ -2387,28 +2450,64 @@ macro_rules! uint_impl {
#[rustc_const_stable(feature = "const_int_pow", since = "1.50.0")]
#[must_use = "this returns the result of the operation, \
without modifying the original"]
#[rustc_allow_const_fn_unstable(is_val_statically_known, const_int_unchecked_arith)]
#[inline]
#[rustc_inherit_overflow_checks]
#[track_caller] // Hides the hackish overflow check for powers of two.
pub const fn pow(self, mut exp: u32) -> Self {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc = 1;
while exp > 1 {
if (exp & 1) == 1 {
acc = acc * base;
// LLVM now knows that `self` is a constant value, but not a
// constant in Rust. This allows us to compute the power used at
// compile-time.
//
// This will likely add a branch in debug builds, but this should
// be ok.
//
// This is a massive performance boost in release builds as you can
// get the power of a power of two and the exponent through a `shl`
// instruction, but we must add a couple more checks for parity with
// our own `pow`.
// SAFETY: This path has the same behavior as the other.
if unsafe { intrinsics::is_val_statically_known(self) }
&& self.is_power_of_two()
{
if self == 1 { // Avoid divide by zero
return 1;
}
// SAFETY: We just checked this is a power of two. and above zero.
let power_used = unsafe { intrinsics::cttz_nonzero(self) as u32 };
if exp > Self::BITS / power_used { // Division of constants is free
#[allow(arithmetic_overflow)]
return Self::MAX * Self::MAX * 0;
}
exp /= 2;
base = base * base;
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc * base
// SAFETY: exp <= Self::BITS / power_used
unsafe { intrinsics::unchecked_shl(
1 as Self,
intrinsics::unchecked_mul(power_used, exp) as Self
)}
// LLVM doesn't always optimize out the checks
// at the ir level.
} else {
if exp == 0 {
return 1;
}
let mut base = self;
let mut acc = 1;
while exp > 1 {
if (exp & 1) == 1 {
acc = acc * base;
}
exp /= 2;
base = base * base;
}
// since exp!=0, finally the exp must be 1.
// Deal with the final bit of the exponent separately, since
// squaring the base afterwards is not necessary and may cause a
// needless overflow.
acc * base
}
}
/// Returns the square root of the number, rounded down.

12
src/tools/miri/src/shims/intrinsics/mod.rs
View File

@ -5,6 +5,7 @@ use std::iter;
use log::trace;
use rand::Rng;
use rustc_apfloat::{Float, Round};
use rustc_middle::ty::layout::LayoutOf;
use rustc_middle::{
@ -141,6 +142,17 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
this.write_pointer(Pointer::new(ptr.provenance, masked_addr), dest)?;
}
// We want to return either `true` or `false` at random, or else something like
// ```
// if !is_val_statically_known(0) { unreachable_unchecked(); }
// ```
// Would not be considered UB, or the other way around (`is_val_statically_known(0)`).
"is_val_statically_known" => {
let [_] = check_arg_count(args)?;
let branch: bool = this.machine.rng.get_mut().gen();
this.write_scalar(Scalar::from_bool(branch), dest)?;
}
// Floating-point operations
"fabsf32" => {
let [f] = check_arg_count(args)?;

15
src/tools/miri/tests/pass/intrinsics.rs
View File

@ -33,6 +33,21 @@ fn main() {
assert_eq!(intrinsics::likely(false), false);
assert_eq!(intrinsics::unlikely(true), true);
let mut saw_true = false;
let mut saw_false = false;
for _ in 0..50 {
if unsafe { intrinsics::is_val_statically_known(0) } {
saw_true = true;
} else {
saw_false = true;
}
}
assert!(
saw_true && saw_false,
"`is_val_statically_known` failed to return both true and false. Congrats, you won the lottery!"
);
intrinsics::forget(Bomb);
let _v = intrinsics::discriminant_value(&Some(()));

48
tests/codegen/is_val_statically_known.rs Normal file
View File

@ -0,0 +1,48 @@
// compile-flags: --crate-type=lib -Zmerge-functions=disabled -O
#![feature(core_intrinsics)]
use std::intrinsics::is_val_statically_known;
pub struct A(u32);
pub enum B {
Ye(u32),
}
#[inline]
pub fn _u32(a: u32) -> i32 {
if unsafe { is_val_statically_known(a) } { 1 } else { 0 }
}
// CHECK-LABEL: @_u32_true(
#[no_mangle]
pub fn _u32_true() -> i32 {
// CHECK: ret i32 1
_u32(1)
}
// CHECK-LABEL: @_u32_false(
#[no_mangle]
pub fn _u32_false(a: u32) -> i32 {
// CHECK: ret i32 0
_u32(a)
}
#[inline]
pub fn _bool(b: bool) -> i32 {
if unsafe { is_val_statically_known(b) } { 3 } else { 2 }
}
// CHECK-LABEL: @_bool_true(
#[no_mangle]
pub fn _bool_true() -> i32 {
// CHECK: ret i32 3
_bool(true)
}
// CHECK-LABEL: @_bool_false(
#[no_mangle]
pub fn _bool_false(b: bool) -> i32 {
// CHECK: ret i32 2
_bool(b)
}

55
tests/codegen/pow_of_two.rs Normal file
View File

@ -0,0 +1,55 @@
// compile-flags: --crate-type=lib -Zmerge-functions=disabled -O -C overflow-checks=false
// CHECK-LABEL: @a(
#[no_mangle]
pub fn a(exp: u32) -> u64 {
// CHECK: %{{[^ ]+}} = icmp ugt i32 %exp, 64
// CHECK: %{{[^ ]+}} = zext i32 %exp to i64
// CHECK: %{{[^ ]+}} = shl nuw i64 {{[^ ]+}}, %{{[^ ]+}}
// CHECK: ret i64 %{{[^ ]+}}
2u64.pow(exp)
}
// CHECK-LABEL: @b(
#[no_mangle]
pub fn b(exp: u32) -> i64 {
// CHECK: %{{[^ ]+}} = icmp ugt i32 %exp, 64
// CHECK: %{{[^ ]+}} = zext i32 %exp to i64
// CHECK: %{{[^ ]+}} = shl nuw i64 {{[^ ]+}}, %{{[^ ]+}}
// CHECK: ret i64 %{{[^ ]+}}
2i64.pow(exp)
}
// CHECK-LABEL: @c(
#[no_mangle]
pub fn c(exp: u32) -> u32 {
// CHECK: %{{[^ ]+}} = icmp ugt i32 %exp, 16
// CHECK: %{{[^ ]+}} = shl nuw nsw i32 %exp, 1
// CHECK: %{{[^ ]+}} = shl nuw i32 1, %{{[^ ]+}}
// CHECK: %{{[^ ]+}} = select i1 %{{[^ ]+}}, i32 0, i32 %{{[^ ]+}}
// CHECK: ret i32 %{{[^ ]+}}
4u32.pow(exp)
}
// CHECK-LABEL: @d(
#[no_mangle]
pub fn d(exp: u32) -> u32 {
// CHECK: %{{[^ ]+}} = icmp ugt i32 %exp, 6
// CHECK: %{{[^ ]+}} = mul nuw nsw i32 %exp, 5
// CHECK: %{{[^ ]+}} = shl nuw nsw i32 1, %{{[^ ]+}}
// CHECK: %{{[^ ]+}} = select i1 {{[^ ]+}}, i32 0, i32 %{{[^ ]+}}
// CHECK: ret i32 %{{[^ ]+}}
32u32.pow(exp)
}
// CHECK-LABEL: @e(
#[no_mangle]
pub fn e(exp: u32) -> i32 {
// CHECK: %{{[^ ]+}} = icmp ugt i32 %exp, 6
// CHECK: %{{[^ ]+}} = mul nuw {{(nsw )?}}i32 %exp, 5
// CHECK: %{{[^ ]+}} = shl nuw {{(nsw )?}}i32 1, %{{[^ ]+}}
// CHECK: %{{[^ ]+}} = select i1 {{[^ ]+}}, i32 0, i32 %{{[^ ]+}}
// CHECK: ret i32 %{{[^ ]+}}
32i32.pow(exp)
}
// note: d and e are expected to yield the same IR

15
tests/ui/consts/is_val_statically_known.rs Normal file
View File

@ -0,0 +1,15 @@
// run-pass
#![feature(core_intrinsics)]
#![feature(is_val_statically_known)]
use std::intrinsics::is_val_statically_known;
const CONST_TEST: bool = unsafe { is_val_statically_known(0) };
fn main() {
if CONST_TEST {
unreachable!("currently expected to return false during const eval");
// but note that this is not a guarantee!
}
}