dnrops
/
rust
mirror of https://github.com/rust-lang/rust.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge commit 'd3a2366ee877075c59b38bd8ced55f224fc7ef51' into sync_cg_clif-2022-07-26

This commit is contained in:
bjorn3 2022-07-26 18:53:46 +02:00
parent c9b31839b6 d3a2366ee8
commit 30a5eb063e
3 changed files with 699 additions and 455 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

145
compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
View File

@ -13,15 +13,11 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
ret: CPlace<'tcx>,
target: Option<BasicBlock>,
) {
intrinsic_match! {
fx, intrinsic, args,
_ => {
fx.tcx.sess.warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic));
crate::trap::trap_unimplemented(fx, intrinsic);
};
match intrinsic {
// Used by `_mm_movemask_epi8` and `_mm256_movemask_epi8`
"llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd", (c a) {
"llvm.x86.sse2.pmovmskb.128" | "llvm.x86.avx2.pmovmskb" | "llvm.x86.sse2.movmsk.pd" => {
intrinsic_args!(fx, args => (a); intrinsic);
let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
let lane_ty = fx.clif_type(lane_ty).unwrap();
assert!(lane_count <= 32);
@ -29,7 +25,8 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
let mut res = fx.bcx.ins().iconst(types::I32, 0);
for lane in (0..lane_count).rev() {
let a_lane = a.value_field(fx, mir::Field::new(lane.try_into().unwrap())).load_scalar(fx);
let a_lane =
a.value_field(fx, mir::Field::new(lane.try_into().unwrap())).load_scalar(fx);
// cast float to int
let a_lane = match lane_ty {
@ -49,26 +46,29 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
let res = CValue::by_val(res, fx.layout_of(fx.tcx.types.i32));
ret.write_cvalue(fx, res);
};
"llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd", (c x, c y, o kind) {
let kind = crate::constant::mir_operand_get_const_val(fx, kind).expect("llvm.x86.sse2.cmp.* kind not const");
let flt_cc = match kind.try_to_bits(Size::from_bytes(1)).unwrap_or_else(|| panic!("kind not scalar: {:?}", kind)) {
}
"llvm.x86.sse2.cmp.ps" | "llvm.x86.sse2.cmp.pd" => {
let (x, y, kind) = match args {
[x, y, kind] => (x, y, kind),
_ => bug!("wrong number of args for intrinsic {intrinsic}"),
};
let x = codegen_operand(fx, x);
let y = codegen_operand(fx, y);
let kind = crate::constant::mir_operand_get_const_val(fx, kind)
.expect("llvm.x86.sse2.cmp.* kind not const");
let flt_cc = match kind
.try_to_bits(Size::from_bytes(1))
.unwrap_or_else(|| panic!("kind not scalar: {:?}", kind))
{
0 => FloatCC::Equal,
1 => FloatCC::LessThan,
2 => FloatCC::LessThanOrEqual,
7 => {
unimplemented!("Compares corresponding elements in `a` and `b` to see if neither is `NaN`.");
}
3 => {
unimplemented!("Compares corresponding elements in `a` and `b` to see if either is `NaN`.");
}
7 => FloatCC::Ordered,
3 => FloatCC::Unordered,
4 => FloatCC::NotEqual,
5 => {
unimplemented!("not less than");
}
6 => {
unimplemented!("not less than or equal");
}
5 => FloatCC::UnorderedOrGreaterThanOrEqual,
6 => FloatCC::UnorderedOrGreaterThan,
kind => unreachable!("kind {:?}", kind),
};
@ -79,50 +79,67 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
};
bool_to_zero_or_max_uint(fx, res_lane_ty, res_lane)
});
};
"llvm.x86.sse2.psrli.d", (c a, o imm8) {
let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
_ => fx.bcx.ins().iconst(types::I32, 0),
}
}
"llvm.x86.sse2.psrli.d" => {
let (a, imm8) = match args {
[a, imm8] => (a, imm8),
_ => bug!("wrong number of args for intrinsic {intrinsic}"),
};
let a = codegen_operand(fx, a);
let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
.expect("llvm.x86.sse2.psrli.d imm8 not const");
simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
.try_to_bits(Size::from_bytes(4))
.unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
{
imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)),
_ => fx.bcx.ins().iconst(types::I32, 0),
});
};
"llvm.x86.sse2.pslli.d", (c a, o imm8) {
let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8).expect("llvm.x86.sse2.psrli.d imm8 not const");
simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| {
match imm8.try_to_bits(Size::from_bytes(4)).unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) {
imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
_ => fx.bcx.ins().iconst(types::I32, 0),
}
}
"llvm.x86.sse2.pslli.d" => {
let (a, imm8) = match args {
[a, imm8] => (a, imm8),
_ => bug!("wrong number of args for intrinsic {intrinsic}"),
};
let a = codegen_operand(fx, a);
let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8)
.expect("llvm.x86.sse2.psrli.d imm8 not const");
simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8
.try_to_bits(Size::from_bytes(4))
.unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8))
{
imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)),
_ => fx.bcx.ins().iconst(types::I32, 0),
});
};
"llvm.x86.sse2.storeu.dq", (v mem_addr, c a) {
}
"llvm.x86.sse2.storeu.dq" => {
intrinsic_args!(fx, args => (mem_addr, a); intrinsic);
let mem_addr = mem_addr.load_scalar(fx);
// FIXME correctly handle the unalignment
let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout());
dest.write_cvalue(fx, a);
};
"llvm.x86.addcarry.64", (v c_in, c a, c b) {
llvm_add_sub(
fx,
BinOp::Add,
ret,
c_in,
a,
b
);
};
"llvm.x86.subborrow.64", (v b_in, c a, c b) {
llvm_add_sub(
fx,
BinOp::Sub,
ret,
b_in,
a,
b
);
};
}
"llvm.x86.addcarry.64" => {
intrinsic_args!(fx, args => (c_in, a, b); intrinsic);
let c_in = c_in.load_scalar(fx);
llvm_add_sub(fx, BinOp::Add, ret, c_in, a, b);
}
"llvm.x86.subborrow.64" => {
intrinsic_args!(fx, args => (b_in, a, b); intrinsic);
let b_in = b_in.load_scalar(fx);
llvm_add_sub(fx, BinOp::Sub, ret, b_in, a, b);
}
_ => {
fx.tcx
.sess
.warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic));
crate::trap::trap_unimplemented(fx, intrinsic);
}
}
let dest = target.expect("all llvm intrinsics used by stdlib should return");

622
compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
View File

@ -1,50 +1,14 @@
//! Codegen of intrinsics. This includes `extern "rust-intrinsic"`, `extern "platform-intrinsic"`
//! and LLVM intrinsics that have symbol names starting with `llvm.`.
macro_rules! intrinsic_pat {
(_) => {
_
};
($name:ident) => {
sym::$name
};
(kw.$name:ident) => {
kw::$name
};
($name:literal) => {
$name
};
}
macro_rules! intrinsic_arg {
(o $fx:expr, $arg:ident) => {};
(c $fx:expr, $arg:ident) => {
let $arg = codegen_operand($fx, $arg);
};
(v $fx:expr, $arg:ident) => {
let $arg = codegen_operand($fx, $arg).load_scalar($fx);
};
}
macro_rules! intrinsic_match {
($fx:expr, $intrinsic:expr, $args:expr,
_ => $unknown:block;
$(
$($($name:tt).*)|+ $(if $cond:expr)?, ($($a:ident $arg:ident),*) $content:block;
)*) => {
match $intrinsic {
$(
$(intrinsic_pat!($($name).*))|* $(if $cond)? => {
if let [$($arg),*] = $args {
$(intrinsic_arg!($a $fx, $arg);)*
$content
} else {
bug!("wrong number of args for intrinsic {:?}", $intrinsic);
}
}
)*
_ => $unknown,
}
macro_rules! intrinsic_args {
($fx:expr, $args:expr => ($($arg:tt),*); $intrinsic:expr) => {
#[allow(unused_parens)]
let ($($arg),*) = if let [$($arg),*] = $args {
($(codegen_operand($fx, $arg)),*)
} else {
$crate::intrinsics::bug_on_incorrect_arg_count($intrinsic);
};
}
}
@ -62,6 +26,10 @@ use rustc_span::symbol::{kw, sym, Symbol};
use crate::prelude::*;
use cranelift_codegen::ir::AtomicRmwOp;
fn bug_on_incorrect_arg_count(intrinsic: impl std::fmt::Display) -> ! {
bug!("wrong number of args for intrinsic {}", intrinsic);
}
fn report_atomic_type_validation_error<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
intrinsic: Symbol,
@ -351,28 +319,31 @@ fn codegen_regular_intrinsic_call<'tcx>(
) {
let usize_layout = fx.layout_of(fx.tcx.types.usize);
intrinsic_match! {
fx, intrinsic, args,
_ => {
fx.tcx.sess.span_fatal(source_info.span, &format!("unsupported intrinsic {}", intrinsic));
};
match intrinsic {
sym::assume => {
intrinsic_args!(fx, args => (_a); intrinsic);
}
sym::likely | sym::unlikely => {
intrinsic_args!(fx, args => (a); intrinsic);
assume, (c _a) {};
likely | unlikely, (c a) {
ret.write_cvalue(fx, a);
};
breakpoint, () {
}
sym::breakpoint => {
intrinsic_args!(fx, args => (); intrinsic);
fx.bcx.ins().debugtrap();
};
copy | copy_nonoverlapping, (v src, v dst, v count) {
}
sym::copy | sym::copy_nonoverlapping => {
intrinsic_args!(fx, args => (src, dst, count); intrinsic);
let src = src.load_scalar(fx);
let dst = dst.load_scalar(fx);
let count = count.load_scalar(fx);
let elem_ty = substs.type_at(0);
let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
assert_eq!(args.len(), 3);
let byte_amount = if elem_size != 1 {
fx.bcx.ins().imul_imm(count, elem_size as i64)
} else {
count
};
let byte_amount =
if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
if intrinsic == sym::copy_nonoverlapping {
// FIXME emit_small_memcpy
@ -381,17 +352,19 @@ fn codegen_regular_intrinsic_call<'tcx>(
// FIXME emit_small_memmove
fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
}
};
// NOTE: the volatile variants have src and dst swapped
volatile_copy_memory | volatile_copy_nonoverlapping_memory, (v dst, v src, v count) {
}
sym::volatile_copy_memory | sym::volatile_copy_nonoverlapping_memory => {
// NOTE: the volatile variants have src and dst swapped
intrinsic_args!(fx, args => (dst, src, count); intrinsic);
let dst = dst.load_scalar(fx);
let src = src.load_scalar(fx);
let count = count.load_scalar(fx);
let elem_ty = substs.type_at(0);
let elem_size: u64 = fx.layout_of(elem_ty).size.bytes();
assert_eq!(args.len(), 3);
let byte_amount = if elem_size != 1 {
fx.bcx.ins().imul_imm(count, elem_size as i64)
} else {
count
};
let byte_amount =
if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
// FIXME make the copy actually volatile when using emit_small_mem{cpy,move}
if intrinsic == sym::volatile_copy_nonoverlapping_memory {
@ -401,8 +374,10 @@ fn codegen_regular_intrinsic_call<'tcx>(
// FIXME emit_small_memmove
fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
}
};
size_of_val, (c ptr) {
}
sym::size_of_val => {
intrinsic_args!(fx, args => (ptr); intrinsic);
let layout = fx.layout_of(substs.type_at(0));
// Note: Can't use is_unsized here as truly unsized types need to take the fixed size
// branch
@ -411,14 +386,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
let (size, _align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
size
} else {
fx
.bcx
.ins()
.iconst(fx.pointer_type, layout.size.bytes() as i64)
fx.bcx.ins().iconst(fx.pointer_type, layout.size.bytes() as i64)
};
ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
};
min_align_of_val, (c ptr) {
}
sym::min_align_of_val => {
intrinsic_args!(fx, args => (ptr); intrinsic);
let layout = fx.layout_of(substs.type_at(0));
// Note: Can't use is_unsized here as truly unsized types need to take the fixed size
// branch
@ -427,26 +401,37 @@ fn codegen_regular_intrinsic_call<'tcx>(
let (_size, align) = crate::unsize::size_and_align_of_dst(fx, layout, info);
align
} else {
fx
.bcx
.ins()
.iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
fx.bcx.ins().iconst(fx.pointer_type, layout.align.abi.bytes() as i64)
};
ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
};
}
sym::vtable_size => {
intrinsic_args!(fx, args => (vtable); intrinsic);
let vtable = vtable.load_scalar(fx);
vtable_size, (v vtable) {
let size = crate::vtable::size_of_obj(fx, vtable);
ret.write_cvalue(fx, CValue::by_val(size, usize_layout));
};
}
sym::vtable_align => {
intrinsic_args!(fx, args => (vtable); intrinsic);
let vtable = vtable.load_scalar(fx);
vtable_align, (v vtable) {
let align = crate::vtable::min_align_of_obj(fx, vtable);
ret.write_cvalue(fx, CValue::by_val(align, usize_layout));
};
}
sym::unchecked_add
| sym::unchecked_sub
| sym::unchecked_mul
| sym::unchecked_div
| sym::exact_div
| sym::unchecked_rem
| sym::unchecked_shl
| sym::unchecked_shr => {
intrinsic_args!(fx, args => (x, y); intrinsic);
unchecked_add | unchecked_sub | unchecked_mul | unchecked_div | exact_div | unchecked_rem
| unchecked_shl | unchecked_shr, (c x, c y) {
// FIXME trap on overflow
let bin_op = match intrinsic {
sym::unchecked_add => BinOp::Add,
@ -460,8 +445,10 @@ fn codegen_regular_intrinsic_call<'tcx>(
};
let res = crate::num::codegen_int_binop(fx, bin_op, x, y);
ret.write_cvalue(fx, res);
};
add_with_overflow | sub_with_overflow | mul_with_overflow, (c x, c y) {
}
sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => {
intrinsic_args!(fx, args => (x, y); intrinsic);
assert_eq!(x.layout().ty, y.layout().ty);
let bin_op = match intrinsic {
sym::add_with_overflow => BinOp::Add,
@ -470,15 +457,12 @@ fn codegen_regular_intrinsic_call<'tcx>(
_ => unreachable!(),
};
let res = crate::num::codegen_checked_int_binop(
fx,
bin_op,
x,
y,
);
let res = crate::num::codegen_checked_int_binop(fx, bin_op, x, y);
ret.write_cvalue(fx, res);
};
saturating_add | saturating_sub, (c lhs, c rhs) {
}
sym::saturating_add | sym::saturating_sub => {
intrinsic_args!(fx, args => (lhs, rhs); intrinsic);
assert_eq!(lhs.layout().ty, rhs.layout().ty);
let bin_op = match intrinsic {
sym::saturating_add => BinOp::Add,
@ -488,12 +472,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
let signed = type_sign(lhs.layout().ty);
let checked_res = crate::num::codegen_checked_int_binop(
fx,
bin_op,
lhs,
rhs,
);
let checked_res = crate::num::codegen_checked_int_binop(fx, bin_op, lhs, rhs);
let (val, has_overflow) = checked_res.load_scalar_pair(fx);
let clif_ty = fx.clif_type(lhs.layout().ty).unwrap();
@ -505,13 +484,15 @@ fn codegen_regular_intrinsic_call<'tcx>(
(sym::saturating_sub, false) => fx.bcx.ins().select(has_overflow, min, val),
(sym::saturating_add, true) => {
let rhs = rhs.load_scalar(fx);
let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
let rhs_ge_zero =
fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
let sat_val = fx.bcx.ins().select(rhs_ge_zero, max, min);
fx.bcx.ins().select(has_overflow, sat_val, val)
}
(sym::saturating_sub, true) => {
let rhs = rhs.load_scalar(fx);
let rhs_ge_zero = fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
let rhs_ge_zero =
fx.bcx.ins().icmp_imm(IntCC::SignedGreaterThanOrEqual, rhs, 0);
let sat_val = fx.bcx.ins().select(rhs_ge_zero, min, max);
fx.bcx.ins().select(has_overflow, sat_val, val)
}
@ -521,23 +502,32 @@ fn codegen_regular_intrinsic_call<'tcx>(
let res = CValue::by_val(val, lhs.layout());
ret.write_cvalue(fx, res);
};
rotate_left, (c x, v y) {
}
sym::rotate_left => {
intrinsic_args!(fx, args => (x, y); intrinsic);
let y = y.load_scalar(fx);
let layout = x.layout();
let x = x.load_scalar(fx);
let res = fx.bcx.ins().rotl(x, y);
ret.write_cvalue(fx, CValue::by_val(res, layout));
};
rotate_right, (c x, v y) {
}
sym::rotate_right => {
intrinsic_args!(fx, args => (x, y); intrinsic);
let y = y.load_scalar(fx);
let layout = x.layout();
let x = x.load_scalar(fx);
let res = fx.bcx.ins().rotr(x, y);
ret.write_cvalue(fx, CValue::by_val(res, layout));
};
}
// The only difference between offset and arith_offset is regarding UB. Because Cranelift
// doesn't have UB both are codegen'ed the same way
offset | arith_offset, (c base, v offset) {
sym::offset | sym::arith_offset => {
intrinsic_args!(fx, args => (base, offset); intrinsic);
let offset = offset.load_scalar(fx);
let pointee_ty = base.layout().ty.builtin_deref(true).unwrap().ty;
let pointee_size = fx.layout_of(pointee_ty).size.bytes();
let ptr_diff = if pointee_size != 1 {
@ -548,12 +538,18 @@ fn codegen_regular_intrinsic_call<'tcx>(
let base_val = base.load_scalar(fx);
let res = fx.bcx.ins().iadd(base_val, ptr_diff);
ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
};
}
sym::transmute => {
intrinsic_args!(fx, args => (from); intrinsic);
transmute, (c from) {
ret.write_cvalue_transmute(fx, from);
};
write_bytes | volatile_set_memory, (c dst, v val, v count) {
}
sym::write_bytes | sym::volatile_set_memory => {
intrinsic_args!(fx, args => (dst, val, count); intrinsic);
let val = val.load_scalar(fx);
let count = count.load_scalar(fx);
let pointee_ty = dst.layout().ty.builtin_deref(true).unwrap().ty;
let pointee_size = fx.layout_of(pointee_ty).size.bytes();
let count = if pointee_size != 1 {
@ -565,34 +561,42 @@ fn codegen_regular_intrinsic_call<'tcx>(
// FIXME make the memset actually volatile when switching to emit_small_memset
// FIXME use emit_small_memset
fx.bcx.call_memset(fx.target_config, dst_ptr, val, count);
};
ctlz | ctlz_nonzero, (c arg) {
}
sym::ctlz | sym::ctlz_nonzero => {
intrinsic_args!(fx, args => (arg); intrinsic);
let val = arg.load_scalar(fx);
// FIXME trap on `ctlz_nonzero` with zero arg.
let res = fx.bcx.ins().clz(val);
let res = CValue::by_val(res, arg.layout());
ret.write_cvalue(fx, res);
};
cttz | cttz_nonzero, (c arg) {
}
sym::cttz | sym::cttz_nonzero => {
intrinsic_args!(fx, args => (arg); intrinsic);
let val = arg.load_scalar(fx);
// FIXME trap on `cttz_nonzero` with zero arg.
let res = fx.bcx.ins().ctz(val);
let res = CValue::by_val(res, arg.layout());
ret.write_cvalue(fx, res);
};
ctpop, (c arg) {
}
sym::ctpop => {
intrinsic_args!(fx, args => (arg); intrinsic);
let val = arg.load_scalar(fx);
let res = fx.bcx.ins().popcnt(val);
let res = CValue::by_val(res, arg.layout());
ret.write_cvalue(fx, res);
};
bitreverse, (c arg) {
}
sym::bitreverse => {
intrinsic_args!(fx, args => (arg); intrinsic);
let val = arg.load_scalar(fx);
let res = fx.bcx.ins().bitrev(val);
let res = CValue::by_val(res, arg.layout());
ret.write_cvalue(fx, res);
};
bswap, (c arg) {
}
sym::bswap => {
// FIXME(CraneStation/cranelift#794) add bswap instruction to cranelift
fn swap(bcx: &mut FunctionBuilder<'_>, v: Value) -> Value {
match bcx.func.dfg.value_type(v) {
@ -668,11 +672,15 @@ fn codegen_regular_intrinsic_call<'tcx>(
ty => unreachable!("bswap {}", ty),
}
}
intrinsic_args!(fx, args => (arg); intrinsic);
let val = arg.load_scalar(fx);
let res = CValue::by_val(swap(&mut fx.bcx, val), arg.layout());
ret.write_cvalue(fx, res);
};
assert_inhabited | assert_zero_valid | assert_uninit_valid, () {
}
sym::assert_inhabited | sym::assert_zero_valid | sym::assert_uninit_valid => {
intrinsic_args!(fx, args => (); intrinsic);
let layout = fx.layout_of(substs.type_at(0));
if layout.abi.is_uninhabited() {
with_no_trimmed_paths!({
@ -689,7 +697,10 @@ fn codegen_regular_intrinsic_call<'tcx>(
with_no_trimmed_paths!({
crate::base::codegen_panic(
fx,
&format!("attempted to zero-initialize type `{}`, which is invalid", layout.ty),
&format!(
"attempted to zero-initialize type `{}`, which is invalid",
layout.ty
),
source_info,
);
});
@ -700,41 +711,53 @@ fn codegen_regular_intrinsic_call<'tcx>(
with_no_trimmed_paths!({
crate::base::codegen_panic(
fx,
&format!("attempted to leave type `{}` uninitialized, which is invalid", layout.ty),
&format!(
"attempted to leave type `{}` uninitialized, which is invalid",
layout.ty
),
source_info,
)
});
return;
}
};
}
sym::volatile_load | sym::unaligned_volatile_load => {
intrinsic_args!(fx, args => (ptr); intrinsic);
volatile_load | unaligned_volatile_load, (c ptr) {
// Cranelift treats loads as volatile by default
// FIXME correctly handle unaligned_volatile_load
let inner_layout =
fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
let inner_layout = fx.layout_of(ptr.layout().ty.builtin_deref(true).unwrap().ty);
let val = CValue::by_ref(Pointer::new(ptr.load_scalar(fx)), inner_layout);
ret.write_cvalue(fx, val);
};
volatile_store | unaligned_volatile_store, (v ptr, c val) {
}
sym::volatile_store | sym::unaligned_volatile_store => {
intrinsic_args!(fx, args => (ptr, val); intrinsic);
let ptr = ptr.load_scalar(fx);
// Cranelift treats stores as volatile by default
// FIXME correctly handle unaligned_volatile_store
let dest = CPlace::for_ptr(Pointer::new(ptr), val.layout());
dest.write_cvalue(fx, val);
};
}
sym::pref_align_of
| sym::needs_drop
| sym::type_id
| sym::type_name
| sym::variant_count => {
intrinsic_args!(fx, args => (); intrinsic);
pref_align_of | needs_drop | type_id | type_name | variant_count, () {
let const_val =
fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
let val = crate::constant::codegen_const_value(
fx,
const_val,
ret.layout().ty,
);
let val = crate::constant::codegen_const_value(fx, const_val, ret.layout().ty);
ret.write_cvalue(fx, val);
};
}
ptr_offset_from | ptr_offset_from_unsigned, (v ptr, v base) {
sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
intrinsic_args!(fx, args => (ptr, base); intrinsic);
let ptr = ptr.load_scalar(fx);
let base = base.load_scalar(fx);
let ty = substs.type_at(0);
let pointee_size: u64 = fx.layout_of(ty).size.bytes();
@ -750,31 +773,44 @@ fn codegen_regular_intrinsic_call<'tcx>(
CValue::by_val(fx.bcx.ins().sdiv_imm(diff_bytes, pointee_size as i64), isize_layout)
};
ret.write_cvalue(fx, val);
};
}
sym::ptr_guaranteed_eq => {
intrinsic_args!(fx, args => (a, b); intrinsic);
ptr_guaranteed_eq, (c a, c b) {
let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b);
ret.write_cvalue(fx, val);
};
}
sym::ptr_guaranteed_ne => {
intrinsic_args!(fx, args => (a, b); intrinsic);
ptr_guaranteed_ne, (c a, c b) {
let val = crate::num::codegen_ptr_binop(fx, BinOp::Ne, a, b);
ret.write_cvalue(fx, val);
};
}
sym::caller_location => {
intrinsic_args!(fx, args => (); intrinsic);
caller_location, () {
let caller_location = fx.get_caller_location(source_info);
ret.write_cvalue(fx, caller_location);
};
}
_ if intrinsic.as_str().starts_with("atomic_fence") => {
intrinsic_args!(fx, args => (); intrinsic);
_ if intrinsic.as_str().starts_with("atomic_fence"), () {
fx.bcx.ins().fence();
};
_ if intrinsic.as_str().starts_with("atomic_singlethreadfence"), () {
}
_ if intrinsic.as_str().starts_with("atomic_singlethreadfence") => {
intrinsic_args!(fx, args => (); intrinsic);
// FIXME use a compiler fence once Cranelift supports it
fx.bcx.ins().fence();
};
_ if intrinsic.as_str().starts_with("atomic_load"), (v ptr) {
}
_ if intrinsic.as_str().starts_with("atomic_load") => {
intrinsic_args!(fx, args => (ptr); intrinsic);
let ptr = ptr.load_scalar(fx);
let ty = substs.type_at(0);
match ty.kind() {
ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
@ -786,7 +822,9 @@ fn codegen_regular_intrinsic_call<'tcx>(
fx.bcx.ins().jump(ret_block, &[]);
return;
} else {
fx.tcx.sess.span_fatal(source_info.span, "128bit atomics not yet supported");
fx.tcx
.sess
.span_fatal(source_info.span, "128bit atomics not yet supported");
}
}
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -801,8 +839,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
let val = CValue::by_val(val, fx.layout_of(ty));
ret.write_cvalue(fx, val);
};
_ if intrinsic.as_str().starts_with("atomic_store"), (v ptr, c val) {
}
_ if intrinsic.as_str().starts_with("atomic_store") => {
intrinsic_args!(fx, args => (ptr, val); intrinsic);
let ptr = ptr.load_scalar(fx);
let ty = substs.type_at(0);
match ty.kind() {
ty::Uint(UintTy::U128) | ty::Int(IntTy::I128) => {
@ -814,7 +855,9 @@ fn codegen_regular_intrinsic_call<'tcx>(
fx.bcx.ins().jump(ret_block, &[]);
return;
} else {
fx.tcx.sess.span_fatal(source_info.span, "128bit atomics not yet supported");
fx.tcx
.sess
.span_fatal(source_info.span, "128bit atomics not yet supported");
}
}
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -827,8 +870,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
let val = val.load_scalar(fx);
fx.bcx.ins().atomic_store(MemFlags::trusted(), val, ptr);
};
_ if intrinsic.as_str().starts_with("atomic_xchg"), (v ptr, c new) {
}
_ if intrinsic.as_str().starts_with("atomic_xchg") => {
intrinsic_args!(fx, args => (ptr, new); intrinsic);
let ptr = ptr.load_scalar(fx);
let layout = new.layout();
match layout.ty.kind() {
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -845,8 +891,12 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.as_str().starts_with("atomic_cxchg"), (v ptr, c test_old, c new) { // both atomic_cxchg_* and atomic_cxchgweak_*
}
_ if intrinsic.as_str().starts_with("atomic_cxchg") => {
// both atomic_cxchg_* and atomic_cxchgweak_*
intrinsic_args!(fx, args => (ptr, test_old, new); intrinsic);
let ptr = ptr.load_scalar(fx);
let layout = new.layout();
match layout.ty.kind() {
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -862,11 +912,15 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = fx.bcx.ins().atomic_cas(MemFlags::trusted(), ptr, test_old, new);
let is_eq = fx.bcx.ins().icmp(IntCC::Equal, old, test_old);
let ret_val = CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
let ret_val =
CValue::by_val_pair(old, fx.bcx.ins().bint(types::I8, is_eq), ret.layout());
ret.write_cvalue(fx, ret_val)
};
}
_ if intrinsic.as_str().starts_with("atomic_xadd") => {
intrinsic_args!(fx, args => (ptr, amount); intrinsic);
let ptr = ptr.load_scalar(fx);
_ if intrinsic.as_str().starts_with("atomic_xadd"), (v ptr, c amount) {
let layout = amount.layout();
match layout.ty.kind() {
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -879,12 +933,16 @@ fn codegen_regular_intrinsic_call<'tcx>(
let amount = amount.load_scalar(fx);
let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount);
let old =
fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Add, ptr, amount);
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.as_str().starts_with("atomic_xsub"), (v ptr, c amount) {
}
_ if intrinsic.as_str().starts_with("atomic_xsub") => {
intrinsic_args!(fx, args => (ptr, amount); intrinsic);
let ptr = ptr.load_scalar(fx);
let layout = amount.layout();
match layout.ty.kind() {
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -897,12 +955,16 @@ fn codegen_regular_intrinsic_call<'tcx>(
let amount = amount.load_scalar(fx);
let old = fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount);
let old =
fx.bcx.ins().atomic_rmw(ty, MemFlags::trusted(), AtomicRmwOp::Sub, ptr, amount);
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.as_str().starts_with("atomic_and"), (v ptr, c src) {
}
_ if intrinsic.as_str().starts_with("atomic_and") => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
let layout = src.layout();
match layout.ty.kind() {
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -919,8 +981,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.as_str().starts_with("atomic_or"), (v ptr, c src) {
}
_ if intrinsic.as_str().starts_with("atomic_or") => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
let layout = src.layout();
match layout.ty.kind() {
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -937,8 +1002,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.as_str().starts_with("atomic_xor"), (v ptr, c src) {
}
_ if intrinsic.as_str().starts_with("atomic_xor") => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
let layout = src.layout();
match layout.ty.kind() {
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -955,8 +1023,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.as_str().starts_with("atomic_nand"), (v ptr, c src) {
}
_ if intrinsic.as_str().starts_with("atomic_nand") => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
let layout = src.layout();
match layout.ty.kind() {
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -973,8 +1044,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.as_str().starts_with("atomic_max"), (v ptr, c src) {
}
_ if intrinsic.as_str().starts_with("atomic_max") => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
let layout = src.layout();
match layout.ty.kind() {
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -991,8 +1065,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.as_str().starts_with("atomic_umax"), (v ptr, c src) {
}
_ if intrinsic.as_str().starts_with("atomic_umax") => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
let layout = src.layout();
match layout.ty.kind() {
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -1009,8 +1086,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.as_str().starts_with("atomic_min"), (v ptr, c src) {
}
_ if intrinsic.as_str().starts_with("atomic_min") => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
let layout = src.layout();
match layout.ty.kind() {
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -1027,8 +1107,11 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
_ if intrinsic.as_str().starts_with("atomic_umin"), (v ptr, c src) {
}
_ if intrinsic.as_str().starts_with("atomic_umin") => {
intrinsic_args!(fx, args => (ptr, src); intrinsic);
let ptr = ptr.load_scalar(fx);
let layout = src.layout();
match layout.ty.kind() {
ty::Uint(_) | ty::Int(_) | ty::RawPtr(..) => {}
@ -1045,30 +1128,51 @@ fn codegen_regular_intrinsic_call<'tcx>(
let old = CValue::by_val(old, layout);
ret.write_cvalue(fx, old);
};
}
sym::minnumf32 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
let b = b.load_scalar(fx);
minnumf32, (v a, v b) {
let val = crate::num::codegen_float_min(fx, a, b);
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
ret.write_cvalue(fx, val);
};
minnumf64, (v a, v b) {
}
sym::minnumf64 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
let b = b.load_scalar(fx);
let val = crate::num::codegen_float_min(fx, a, b);
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
ret.write_cvalue(fx, val);
};
maxnumf32, (v a, v b) {
}
sym::maxnumf32 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
let b = b.load_scalar(fx);
let val = crate::num::codegen_float_max(fx, a, b);
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f32));
ret.write_cvalue(fx, val);
};
maxnumf64, (v a, v b) {
}
sym::maxnumf64 => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let a = a.load_scalar(fx);
let b = b.load_scalar(fx);
let val = crate::num::codegen_float_max(fx, a, b);
let val = CValue::by_val(val, fx.layout_of(fx.tcx.types.f64));
ret.write_cvalue(fx, val);
};
}
kw::Try => {
intrinsic_args!(fx, args => (f, data, catch_fn); intrinsic);
let f = f.load_scalar(fx);
let data = data.load_scalar(fx);
let _catch_fn = catch_fn.load_scalar(fx);
kw.Try, (v f, v data, v _catch_fn) {
// FIXME once unwinding is supported, change this to actually catch panics
let f_sig = fx.bcx.func.import_signature(Signature {
call_conv: fx.target_config.default_call_conv,
@ -1081,20 +1185,30 @@ fn codegen_regular_intrinsic_call<'tcx>(
let layout = ret.layout();
let ret_val = CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size));
ret.write_cvalue(fx, ret_val);
};
}
fadd_fast | fsub_fast | fmul_fast | fdiv_fast | frem_fast, (c x, c y) {
let res = crate::num::codegen_float_binop(fx, match intrinsic {
sym::fadd_fast => BinOp::Add,
sym::fsub_fast => BinOp::Sub,
sym::fmul_fast => BinOp::Mul,
sym::fdiv_fast => BinOp::Div,
sym::frem_fast => BinOp::Rem,
_ => unreachable!(),
}, x, y);
sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
intrinsic_args!(fx, args => (x, y); intrinsic);
let res = crate::num::codegen_float_binop(
fx,
match intrinsic {
sym::fadd_fast => BinOp::Add,
sym::fsub_fast => BinOp::Sub,
sym::fmul_fast => BinOp::Mul,
sym::fdiv_fast => BinOp::Div,
sym::frem_fast => BinOp::Rem,
_ => unreachable!(),
},
x,
y,
);
ret.write_cvalue(fx, res);
};
float_to_int_unchecked, (v f) {
}
sym::float_to_int_unchecked => {
intrinsic_args!(fx, args => (f); intrinsic);
let f = f.load_scalar(fx);
let res = crate::cast::clif_int_or_float_cast(
fx,
f,
@ -1103,66 +1217,74 @@ fn codegen_regular_intrinsic_call<'tcx>(
type_sign(ret.layout().ty),
);
ret.write_cvalue(fx, CValue::by_val(res, ret.layout()));
};
}
sym::raw_eq => {
intrinsic_args!(fx, args => (lhs_ref, rhs_ref); intrinsic);
let lhs_ref = lhs_ref.load_scalar(fx);
let rhs_ref = rhs_ref.load_scalar(fx);
raw_eq, (v lhs_ref, v rhs_ref) {
let size = fx.layout_of(substs.type_at(0)).layout.size();
// FIXME add and use emit_small_memcmp
let is_eq_value =
if size == Size::ZERO {
// No bytes means they're trivially equal
fx.bcx.ins().iconst(types::I8, 1)
} else if let Some(clty) = size.bits().try_into().ok().and_then(Type::int) {
// Can't use `trusted` for these loads; they could be unaligned.
let mut flags = MemFlags::new();
flags.set_notrap();
let lhs_val = fx.bcx.ins().load(clty, flags, lhs_ref, 0);
let rhs_val = fx.bcx.ins().load(clty, flags, rhs_ref, 0);
let eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_val, rhs_val);
fx.bcx.ins().bint(types::I8, eq)
} else {
// Just call `memcmp` (like slices do in core) when the
// size is too large or it's not a power-of-two.
let signed_bytes = i64::try_from(size.bytes()).unwrap();
let bytes_val = fx.bcx.ins().iconst(fx.pointer_type, signed_bytes);
let params = vec![AbiParam::new(fx.pointer_type); 3];
let returns = vec![AbiParam::new(types::I32)];
let args = &[lhs_ref, rhs_ref, bytes_val];
let cmp = fx.lib_call("memcmp", params, returns, args)[0];
let eq = fx.bcx.ins().icmp_imm(IntCC::Equal, cmp, 0);
fx.bcx.ins().bint(types::I8, eq)
};
let is_eq_value = if size == Size::ZERO {
// No bytes means they're trivially equal
fx.bcx.ins().iconst(types::I8, 1)
} else if let Some(clty) = size.bits().try_into().ok().and_then(Type::int) {
// Can't use `trusted` for these loads; they could be unaligned.
let mut flags = MemFlags::new();
flags.set_notrap();
let lhs_val = fx.bcx.ins().load(clty, flags, lhs_ref, 0);
let rhs_val = fx.bcx.ins().load(clty, flags, rhs_ref, 0);
let eq = fx.bcx.ins().icmp(IntCC::Equal, lhs_val, rhs_val);
fx.bcx.ins().bint(types::I8, eq)
} else {
// Just call `memcmp` (like slices do in core) when the
// size is too large or it's not a power-of-two.
let signed_bytes = i64::try_from(size.bytes()).unwrap();
let bytes_val = fx.bcx.ins().iconst(fx.pointer_type, signed_bytes);
let params = vec![AbiParam::new(fx.pointer_type); 3];
let returns = vec![AbiParam::new(types::I32)];
let args = &[lhs_ref, rhs_ref, bytes_val];
let cmp = fx.lib_call("memcmp", params, returns, args)[0];
let eq = fx.bcx.ins().icmp_imm(IntCC::Equal, cmp, 0);
fx.bcx.ins().bint(types::I8, eq)
};
ret.write_cvalue(fx, CValue::by_val(is_eq_value, ret.layout()));
};
}
sym::const_allocate => {
intrinsic_args!(fx, args => (_size, _align); intrinsic);
const_allocate, (c _size, c _align) {
// returns a null pointer at runtime.
let null = fx.bcx.ins().iconst(fx.pointer_type, 0);
ret.write_cvalue(fx, CValue::by_val(null, ret.layout()));
};
}
const_deallocate, (c _ptr, c _size, c _align) {
sym::const_deallocate => {
intrinsic_args!(fx, args => (_ptr, _size, _align); intrinsic);
// nop at runtime.
};
}
sym::black_box => {
intrinsic_args!(fx, args => (a); intrinsic);
black_box, (c a) {
// FIXME implement black_box semantics
ret.write_cvalue(fx, a);
};
}
// FIXME implement variadics in cranelift
va_copy, (o _dest, o _src) {
sym::va_copy | sym::va_arg | sym::va_end => {
fx.tcx.sess.span_fatal(
source_info.span,
"Defining variadic functions is not yet supported by Cranelift",
);
};
va_arg | va_end, (o _valist) {
fx.tcx.sess.span_fatal(
source_info.span,
"Defining variadic functions is not yet supported by Cranelift",
);
};
}
_ => {
fx.tcx
.sess
.span_fatal(source_info.span, &format!("unsupported intrinsic {}", intrinsic));
}
}
let ret_block = fx.get_block(destination.unwrap());

387
compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
View File

@ -25,13 +25,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
ret: CPlace<'tcx>,
span: Span,
) {
intrinsic_match! {
fx, intrinsic, args,
_ => {
fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
};
match intrinsic {
sym::simd_cast => {
intrinsic_args!(fx, args => (a); intrinsic);
simd_cast, (c a) {
if !a.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
return;
@ -45,9 +42,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
clif_int_or_float_cast(fx, lane, from_signed, ret_lane_clif_ty, to_signed)
});
};
}
sym::simd_eq | sym::simd_ne | sym::simd_lt | sym::simd_le | sym::simd_gt | sym::simd_ge => {
intrinsic_args!(fx, args => (x, y); intrinsic);
simd_eq | simd_ne | simd_lt | simd_le | simd_gt | simd_ge, (c x, c y) {
if !x.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
return;
@ -57,7 +56,9 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, res_lane_ty, x_lane, y_lane| {
let res_lane = match (lane_ty.kind(), intrinsic) {
(ty::Uint(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
(ty::Uint(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
(ty::Uint(_), sym::simd_ne) => {
fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane)
}
(ty::Uint(_), sym::simd_lt) => {
fx.bcx.ins().icmp(IntCC::UnsignedLessThan, x_lane, y_lane)
}
@ -72,8 +73,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
}
(ty::Int(_), sym::simd_eq) => fx.bcx.ins().icmp(IntCC::Equal, x_lane, y_lane),
(ty::Int(_), sym::simd_ne) => fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane),
(ty::Int(_), sym::simd_lt) => fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane),
(ty::Int(_), sym::simd_ne) => {
fx.bcx.ins().icmp(IntCC::NotEqual, x_lane, y_lane)
}
(ty::Int(_), sym::simd_lt) => {
fx.bcx.ins().icmp(IntCC::SignedLessThan, x_lane, y_lane)
}
(ty::Int(_), sym::simd_le) => {
fx.bcx.ins().icmp(IntCC::SignedLessThanOrEqual, x_lane, y_lane)
}
@ -84,13 +89,21 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
fx.bcx.ins().icmp(IntCC::SignedGreaterThanOrEqual, x_lane, y_lane)
}
(ty::Float(_), sym::simd_eq) => fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane),
(ty::Float(_), sym::simd_ne) => fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane),
(ty::Float(_), sym::simd_lt) => fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane),
(ty::Float(_), sym::simd_eq) => {
fx.bcx.ins().fcmp(FloatCC::Equal, x_lane, y_lane)
}
(ty::Float(_), sym::simd_ne) => {
fx.bcx.ins().fcmp(FloatCC::NotEqual, x_lane, y_lane)
}
(ty::Float(_), sym::simd_lt) => {
fx.bcx.ins().fcmp(FloatCC::LessThan, x_lane, y_lane)
}
(ty::Float(_), sym::simd_le) => {
fx.bcx.ins().fcmp(FloatCC::LessThanOrEqual, x_lane, y_lane)
}
(ty::Float(_), sym::simd_gt) => fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane),
(ty::Float(_), sym::simd_gt) => {
fx.bcx.ins().fcmp(FloatCC::GreaterThan, x_lane, y_lane)
}
(ty::Float(_), sym::simd_ge) => {
fx.bcx.ins().fcmp(FloatCC::GreaterThanOrEqual, x_lane, y_lane)
}
@ -103,10 +116,19 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let res_lane = fx.bcx.ins().bint(ty, res_lane);
fx.bcx.ins().ineg(res_lane)
});
};
}
// simd_shuffle32<T, U>(x: T, y: T, idx: [u32; 32]) -> U
_ if intrinsic.as_str().starts_with("simd_shuffle"), (c x, c y, o idx) {
_ if intrinsic.as_str().starts_with("simd_shuffle") => {
let (x, y, idx) = match args {
[x, y, idx] => (x, y, idx),
_ => {
bug!("wrong number of args for intrinsic {intrinsic}");
}
};
let x = codegen_operand(fx, x);
let y = codegen_operand(fx, y);
if !x.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
return;
@ -119,11 +141,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
// version of this intrinsic.
let idx_ty = fx.monomorphize(idx.ty(fx.mir, fx.tcx));
match idx_ty.kind() {
ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => {
len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| {
ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => len
.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all())
.unwrap_or_else(|| {
span_bug!(span, "could not evaluate shuffle index array length")
}).try_into().unwrap()
}
})
.try_into()
.unwrap(),
_ => {
fx.tcx.sess.span_err(
span,
@ -154,24 +178,30 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let indexes = {
use rustc_middle::mir::interpret::*;
let idx_const = crate::constant::mir_operand_get_const_val(fx, idx).expect("simd_shuffle* idx not const");
let idx_const = crate::constant::mir_operand_get_const_val(fx, idx)
.expect("simd_shuffle* idx not const");
let idx_bytes = match idx_const {
ConstValue::ByRef { alloc, offset } => {
let size = Size::from_bytes(4 * ret_lane_count /* size_of([u32; ret_lane_count]) */);
let size = Size::from_bytes(
4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */
);
alloc.inner().get_bytes(fx, alloc_range(offset, size)).unwrap()
}
_ => unreachable!("{:?}", idx_const),
};
(0..ret_lane_count).map(|i| {
let i = usize::try_from(i).unwrap();
let idx = rustc_middle::mir::interpret::read_target_uint(
fx.tcx.data_layout.endian,
&idx_bytes[4*i.. 4*i + 4],
).expect("read_target_uint");
u16::try_from(idx).expect("try_from u32")
}).collect::<Vec<u16>>()
(0..ret_lane_count)
.map(|i| {
let i = usize::try_from(i).unwrap();
let idx = rustc_middle::mir::interpret::read_target_uint(
fx.tcx.data_layout.endian,
&idx_bytes[4 * i..4 * i + 4],
)
.expect("read_target_uint");
u16::try_from(idx).expect("try_from u32")
})
.collect::<Vec<u16>>()
};
for &idx in &indexes {
@ -187,43 +217,63 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap());
out_lane.write_cvalue(fx, in_lane);
}
};
}
sym::simd_insert => {
let (base, idx, val) = match args {
[base, idx, val] => (base, idx, val),
_ => {
bug!("wrong number of args for intrinsic {intrinsic}");
}
};
let base = codegen_operand(fx, base);
let val = codegen_operand(fx, val);
simd_insert, (c base, o idx, c val) {
// FIXME validate
let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
let idx_const = if let Some(idx_const) =
crate::constant::mir_operand_get_const_val(fx, idx)
{
idx_const
} else {
fx.tcx.sess.span_fatal(
span,
"Index argument for `simd_insert` is not a constant",
);
fx.tcx.sess.span_fatal(span, "Index argument for `simd_insert` is not a constant");
};
let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
let idx = idx_const
.try_to_bits(Size::from_bytes(4 /* u32*/))
.unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
let (lane_count, _lane_ty) = base.layout().ty.simd_size_and_type(fx.tcx);
if idx >= lane_count.into() {
fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count));
fx.tcx.sess.span_fatal(
fx.mir.span,
&format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count),
);
}
ret.write_cvalue(fx, base);
let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
ret_lane.write_cvalue(fx, val);
};
}
sym::simd_extract => {
let (v, idx) = match args {
[v, idx] => (v, idx),
_ => {
bug!("wrong number of args for intrinsic {intrinsic}");
}
};
let v = codegen_operand(fx, v);
simd_extract, (c v, o idx) {
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
let idx_const = if let Some(idx_const) = crate::constant::mir_operand_get_const_val(fx, idx) {
let idx_const = if let Some(idx_const) =
crate::constant::mir_operand_get_const_val(fx, idx)
{
idx_const
} else {
fx.tcx.sess.span_warn(
span,
"Index argument for `simd_extract` is not a constant",
);
fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
let res = crate::trap::trap_unimplemented_ret_value(
fx,
ret.layout(),
@ -233,89 +283,105 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
return;
};
let idx = idx_const.try_to_bits(Size::from_bytes(4 /* u32*/)).unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
let idx = idx_const
.try_to_bits(Size::from_bytes(4 /* u32*/))
.unwrap_or_else(|| panic!("kind not scalar: {:?}", idx_const));
let (lane_count, _lane_ty) = v.layout().ty.simd_size_and_type(fx.tcx);
if idx >= lane_count.into() {
fx.tcx.sess.span_fatal(fx.mir.span, &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count));
fx.tcx.sess.span_fatal(
fx.mir.span,
&format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count),
);
}
let ret_lane = v.value_lane(fx, idx.try_into().unwrap());
ret.write_cvalue(fx, ret_lane);
};
}
sym::simd_neg => {
intrinsic_args!(fx, args => (a); intrinsic);
simd_neg, (c a) {
if !a.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
return;
}
simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
match lane_ty.kind() {
simd_for_each_lane(
fx,
a,
ret,
&|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() {
ty::Int(_) => fx.bcx.ins().ineg(lane),
ty::Float(_) => fx.bcx.ins().fneg(lane),
_ => unreachable!(),
}
});
};
},
);
}
simd_add | simd_sub | simd_mul | simd_div | simd_rem
| simd_shl | simd_shr | simd_and | simd_or | simd_xor, (c x, c y) {
if !x.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
return;
}
sym::simd_add
| sym::simd_sub
| sym::simd_mul
| sym::simd_div
| sym::simd_rem
| sym::simd_shl
| sym::simd_shr
| sym::simd_and
| sym::simd_or
| sym::simd_xor => {
intrinsic_args!(fx, args => (x, y); intrinsic);
// FIXME use vector instructions when possible
simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| match (
lane_ty.kind(),
intrinsic,
) {
(ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
(ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
(ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
(ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane),
(ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane),
simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
match (lane_ty.kind(), intrinsic) {
(ty::Uint(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
(ty::Uint(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
(ty::Uint(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
(ty::Uint(_), sym::simd_div) => fx.bcx.ins().udiv(x_lane, y_lane),
(ty::Uint(_), sym::simd_rem) => fx.bcx.ins().urem(x_lane, y_lane),
(ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
(ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
(ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
(ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
(ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane),
(ty::Int(_), sym::simd_add) => fx.bcx.ins().iadd(x_lane, y_lane),
(ty::Int(_), sym::simd_sub) => fx.bcx.ins().isub(x_lane, y_lane),
(ty::Int(_), sym::simd_mul) => fx.bcx.ins().imul(x_lane, y_lane),
(ty::Int(_), sym::simd_div) => fx.bcx.ins().sdiv(x_lane, y_lane),
(ty::Int(_), sym::simd_rem) => fx.bcx.ins().srem(x_lane, y_lane),
(ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
(ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
(ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
(ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
(ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call(
"fmodf",
vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
vec![AbiParam::new(types::F32)],
&[x_lane, y_lane],
)[0],
(ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call(
"fmod",
vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
vec![AbiParam::new(types::F64)],
&[x_lane, y_lane],
)[0],
(ty::Float(_), sym::simd_add) => fx.bcx.ins().fadd(x_lane, y_lane),
(ty::Float(_), sym::simd_sub) => fx.bcx.ins().fsub(x_lane, y_lane),
(ty::Float(_), sym::simd_mul) => fx.bcx.ins().fmul(x_lane, y_lane),
(ty::Float(_), sym::simd_div) => fx.bcx.ins().fdiv(x_lane, y_lane),
(ty::Float(FloatTy::F32), sym::simd_rem) => fx.lib_call(
"fmodf",
vec![AbiParam::new(types::F32), AbiParam::new(types::F32)],
vec![AbiParam::new(types::F32)],
&[x_lane, y_lane],
)[0],
(ty::Float(FloatTy::F64), sym::simd_rem) => fx.lib_call(
"fmod",
vec![AbiParam::new(types::F64), AbiParam::new(types::F64)],
vec![AbiParam::new(types::F64)],
&[x_lane, y_lane],
)[0],
(ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
(ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
(ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
(ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
(ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
(ty::Uint(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
(ty::Uint(_), sym::simd_shr) => fx.bcx.ins().ushr(x_lane, y_lane),
(ty::Uint(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
(ty::Uint(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
(ty::Uint(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
(ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
(ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane),
(ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
(ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
(ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
(ty::Int(_), sym::simd_shl) => fx.bcx.ins().ishl(x_lane, y_lane),
(ty::Int(_), sym::simd_shr) => fx.bcx.ins().sshr(x_lane, y_lane),
(ty::Int(_), sym::simd_and) => fx.bcx.ins().band(x_lane, y_lane),
(ty::Int(_), sym::simd_or) => fx.bcx.ins().bor(x_lane, y_lane),
(ty::Int(_), sym::simd_xor) => fx.bcx.ins().bxor(x_lane, y_lane),
_ => unreachable!(),
_ => unreachable!(),
}
});
};
}
sym::simd_fma => {
intrinsic_args!(fx, args => (a, b, c); intrinsic);
simd_fma, (c a, c b, c c) {
if !a.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
return;
@ -333,16 +399,22 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let c_lane = c.value_lane(fx, lane);
let res_lane = match lane_ty.kind() {
ty::Float(FloatTy::F32) => fx.easy_call("fmaf", &[a_lane, b_lane, c_lane], lane_ty),
ty::Float(FloatTy::F64) => fx.easy_call("fma", &[a_lane, b_lane, c_lane], lane_ty),
ty::Float(FloatTy::F32) => {
fx.easy_call("fmaf", &[a_lane, b_lane, c_lane], lane_ty)
}
ty::Float(FloatTy::F64) => {
fx.easy_call("fma", &[a_lane, b_lane, c_lane], lane_ty)
}
_ => unreachable!(),
};
ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
}
};
}
sym::simd_fmin | sym::simd_fmax => {
intrinsic_args!(fx, args => (x, y); intrinsic);
simd_fmin | simd_fmax, (c x, c y) {
if !x.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty);
return;
@ -351,7 +423,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
// FIXME use vector instructions when possible
simd_pair_for_each_lane(fx, x, y, ret, &|fx, lane_ty, _ret_lane_ty, x_lane, y_lane| {
match lane_ty.kind() {
ty::Float(_) => {},
ty::Float(_) => {}
_ => unreachable!("{:?}", lane_ty),
}
match intrinsic {
@ -360,16 +432,21 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
_ => unreachable!(),
}
});
};
}
sym::simd_round => {
intrinsic_args!(fx, args => (a); intrinsic);
simd_round, (c a) {
if !a.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
return;
}
simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
match lane_ty.kind() {
simd_for_each_lane(
fx,
a,
ret,
&|fx, lane_ty, _ret_lane_ty, lane| match lane_ty.kind() {
ty::Float(FloatTy::F32) => fx.lib_call(
"roundf",
vec![AbiParam::new(types::F32)],
@ -383,11 +460,13 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
&[lane],
)[0],
_ => unreachable!("{:?}", lane_ty),
}
});
};
},
);
}
sym::simd_fabs | sym::simd_fsqrt | sym::simd_ceil | sym::simd_floor | sym::simd_trunc => {
intrinsic_args!(fx, args => (a); intrinsic);
simd_fabs | simd_fsqrt | simd_ceil | simd_floor | simd_trunc, (c a) {
if !a.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
return;
@ -395,7 +474,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
simd_for_each_lane(fx, a, ret, &|fx, lane_ty, _ret_lane_ty, lane| {
match lane_ty.kind() {
ty::Float(_) => {},
ty::Float(_) => {}
_ => unreachable!("{:?}", lane_ty),
}
match intrinsic {
@ -407,9 +486,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
_ => unreachable!(),
}
});
};
}
sym::simd_reduce_add_ordered | sym::simd_reduce_add_unordered => {
intrinsic_args!(fx, args => (v, acc); intrinsic);
let acc = acc.load_scalar(fx);
simd_reduce_add_ordered | simd_reduce_add_unordered, (c v, v acc) {
// FIXME there must be no acc param for integer vectors
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
@ -423,9 +505,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
fx.bcx.ins().iadd(a, b)
}
});
};
}
sym::simd_reduce_mul_ordered | sym::simd_reduce_mul_unordered => {
intrinsic_args!(fx, args => (v, acc); intrinsic);
let acc = acc.load_scalar(fx);
simd_reduce_mul_ordered | simd_reduce_mul_unordered, (c v, v acc) {
// FIXME there must be no acc param for integer vectors
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
@ -439,54 +524,66 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
fx.bcx.ins().imul(a, b)
}
});
};
}
sym::simd_reduce_all => {
intrinsic_args!(fx, args => (v); intrinsic);
simd_reduce_all, (c v) {
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().band(a, b));
};
}
sym::simd_reduce_any => {
intrinsic_args!(fx, args => (v); intrinsic);
simd_reduce_any, (c v) {
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce_bool(fx, v, ret, &|fx, a, b| fx.bcx.ins().bor(a, b));
};
}
sym::simd_reduce_and => {
intrinsic_args!(fx, args => (v); intrinsic);
simd_reduce_and, (c v) {
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().band(a, b));
};
}
sym::simd_reduce_or => {
intrinsic_args!(fx, args => (v); intrinsic);
simd_reduce_or, (c v) {
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bor(a, b));
};
}
sym::simd_reduce_xor => {
intrinsic_args!(fx, args => (v); intrinsic);
simd_reduce_xor, (c v) {
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
}
simd_reduce(fx, v, None, ret, &|fx, _ty, a, b| fx.bcx.ins().bxor(a, b));
};
}
sym::simd_reduce_min => {
intrinsic_args!(fx, args => (v); intrinsic);
simd_reduce_min, (c v) {
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
@ -501,9 +598,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
};
fx.bcx.ins().select(lt, a, b)
});
};
}
sym::simd_reduce_max => {
intrinsic_args!(fx, args => (v); intrinsic);
simd_reduce_max, (c v) {
if !v.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, v.layout().ty);
return;
@ -518,9 +617,11 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
};
fx.bcx.ins().select(gt, a, b)
});
};
}
sym::simd_select => {
intrinsic_args!(fx, args => (m, a, b); intrinsic);
simd_select, (c m, c a, c b) {
if !m.layout().ty.is_simd() {
report_simd_type_validation_error(fx, intrinsic, span, m.layout().ty);
return;
@ -540,15 +641,19 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let b_lane = b.value_lane(fx, lane).load_scalar(fx);
let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0);
let res_lane = CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);
let res_lane =
CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);
ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
}
};
}
// simd_saturating_*
// simd_bitmask
// simd_scatter
// simd_gather
_ => {
fx.tcx.sess.span_fatal(span, &format!("Unknown SIMD intrinsic {}", intrinsic));
}
}
}