Lower unsigned vsetcc to psubus in certain cases
The current approach to lower a vsetult is to flip the sign bit of the
operands, swap the operands and then use a (signed) pcmpgt. psubus (unsigned
saturating subtract) can be used to emulate a vsetult more efficiently:
+ case ISD::SETULT: {
+ // If the comparison is against a constant we can turn this into a
+ // setule. With psubus, setule does not require a swap. This is
+ // beneficial because the constant in the register is no longer
+ // destructed as the destination so it can be hoisted out of a loop.
I also enable lowering via psubus in a few other cases where it's clearly
beneficial: setule and setuge if minu/maxu cannot be used.
rdar://problem/14338765
Patch by Adam Nemet <anemet@apple.com>.
llvm-svn: 202301
This commit is contained in:
Quentin Colombet
parent
8e9791f62c
commit
85c9e16291
|
|
@ -10009,6 +10009,37 @@ static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG,
|
|||
DAG.getConstant(SSECC, MVT::i8));
|
||||
}
|
||||
|
||||
/// \brief Try to turn a VSETULT into a VSETULE by modifying its second
|
||||
/// operand \p Op1. If non-trivial (for example because it's not constant)
|
||||
/// return an empty value.
|
||||
static SDValue ChangeVSETULTtoVSETULE(SDValue Op1, SelectionDAG &DAG)
|
||||
{
|
||||
BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Op1.getNode());
|
||||
if (!BV)
|
||||
return SDValue();
|
||||
|
||||
MVT VT = Op1.getSimpleValueType();
|
||||
MVT EVT = VT.getVectorElementType();
|
||||
unsigned n = VT.getVectorNumElements();
|
||||
SmallVector<SDValue, 8> ULTOp1;
|
||||
|
||||
for (unsigned i = 0; i < n; ++i) {
|
||||
ConstantSDNode *Elt = dyn_cast<ConstantSDNode>(BV->getOperand(i));
|
||||
if (!Elt || Elt->isOpaque() || Elt->getValueType(0) != EVT)
|
||||
return SDValue();
|
||||
|
||||
// Avoid underflow.
|
||||
APInt Val = Elt->getAPIntValue();
|
||||
if (Val == 0)
|
||||
return SDValue();
|
||||
|
||||
ULTOp1.push_back(DAG.getConstant(Val - 1, EVT));
|
||||
}
|
||||
|
||||
return DAG.getNode(ISD::BUILD_VECTOR, SDLoc(Op1), VT, ULTOp1.data(),
|
||||
ULTOp1.size());
|
||||
}
|
||||
|
||||
static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
|
||||
SelectionDAG &DAG) {
|
||||
SDValue Op0 = Op.getOperand(0);
|
||||
|
|
@ -10080,6 +10111,7 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
|
|||
// operations may be required for some comparisons.
|
||||
unsigned Opc;
|
||||
bool Swap = false, Invert = false, FlipSigns = false, MinMax = false;
|
||||
bool Subus = false;
|
||||
|
||||
switch (SetCCOpcode) {
|
||||
default: llvm_unreachable("Unexpected SETCC condition");
|
||||
|
|
@ -10114,6 +10146,40 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
|
|||
if (MinMax) { Swap = false; Invert = false; FlipSigns = false; }
|
||||
}
|
||||
|
||||
bool hasSubus = Subtarget->hasSSE2() && (VET == MVT::i8 || VET == MVT::i16);
|
||||
if (!MinMax && hasSubus) {
|
||||
// As another special case, use PSUBUS[BW] when it's profitable. E.g. for
|
||||
// Op0 u<= Op1:
|
||||
// t = psubus Op0, Op1
|
||||
// pcmpeq t, <0..0>
|
||||
switch (SetCCOpcode) {
|
||||
default: break;
|
||||
case ISD::SETULT: {
|
||||
// If the comparison is against a constant we can turn this into a
|
||||
// setule. With psubus, setule does not require a swap. This is
|
||||
// beneficial because the constant in the register is no longer
|
||||
// destructed as the destination so it can be hoisted out of a loop.
|
||||
// Only do this pre-AVX since vpcmp* is no longer destructive.
|
||||
if (Subtarget->hasAVX())
|
||||
break;
|
||||
SDValue ULEOp1 = ChangeVSETULTtoVSETULE(Op1, DAG);
|
||||
if (ULEOp1.getNode()) {
|
||||
Op1 = ULEOp1;
|
||||
Subus = true; Invert = false; Swap = false;
|
||||
}
|
||||
break;
|
||||
}
|
||||
// Psubus is better than flip-sign because it requires no inversion.
|
||||
case ISD::SETUGE: Subus = true; Invert = false; Swap = true; break;
|
||||
case ISD::SETULE: Subus = true; Invert = false; Swap = false; break;
|
||||
}
|
||||
|
||||
if (Subus) {
|
||||
Opc = X86ISD::SUBUS;
|
||||
FlipSigns = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (Swap)
|
||||
std::swap(Op0, Op1);
|
||||
|
||||
|
|
@ -10204,6 +10270,10 @@ static SDValue LowerVSETCC(SDValue Op, const X86Subtarget *Subtarget,
|
|||
if (MinMax)
|
||||
Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);
|
||||
|
||||
if (Subus)
|
||||
Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,
|
||||
getZeroVector(VT, Subtarget, DAG, dl));
|
||||
|
||||
return Result;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
; RUN: llc < %s -o - -mcpu=generic -mtriple=x86_64-apple-darwin -mattr=+sse2 | FileCheck %s
|
||||
; RUN: llc < %s -o - -mcpu=generic -mtriple=x86_64-apple-darwin -mattr=+sse4.2 | FileCheck %s
|
||||
|
||||
; For a setult against a constant, turn it into a setule and lower via psubusw.
|
||||
|
||||
define void @loop_no_const_reload(<2 x i64>* %in, <2 x i64>* %out, i32 %n) {
|
||||
; CHECK: .short 25
|
||||
; CHECK-NEXT: .short 25
|
||||
; CHECK-NEXT: .short 25
|
||||
; CHECK-NEXT: .short 25
|
||||
; CHECK-NEXT: .short 25
|
||||
; CHECK-NEXT: .short 25
|
||||
; CHECK-NEXT: .short 25
|
||||
; CHECK-NEXT: .short 25
|
||||
; CHECK-LABEL: loop_no_const_reload:
|
||||
; CHECK: psubusw
|
||||
|
||||
; Constant is no longer clobbered so no need to reload it in the loop.
|
||||
|
||||
; CHECK-NOT: movdqa {{%xmm[0-9]+}}, {{%xmm[0-9]+}}
|
||||
entry:
|
||||
%cmp9 = icmp eq i32 %n, 0
|
||||
br i1 %cmp9, label %for.end, label %for.body
|
||||
|
||||
for.body: ; preds = %for.body, %entry
|
||||
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
|
||||
%arrayidx1 = getelementptr inbounds <2 x i64>* %in, i64 %indvars.iv
|
||||
%arrayidx1.val = load <2 x i64>* %arrayidx1, align 16
|
||||
%0 = bitcast <2 x i64> %arrayidx1.val to <8 x i16>
|
||||
%cmp.i.i = icmp ult <8 x i16> %0, <i16 26, i16 26, i16 26, i16 26, i16 26, i16 26, i16 26, i16 26>
|
||||
%sext.i.i = sext <8 x i1> %cmp.i.i to <8 x i16>
|
||||
%1 = bitcast <8 x i16> %sext.i.i to <2 x i64>
|
||||
%arrayidx5 = getelementptr inbounds <2 x i64>* %out, i64 %indvars.iv
|
||||
store <2 x i64> %1, <2 x i64>* %arrayidx5, align 16
|
||||
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
|
||||
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
||||
%exitcond = icmp eq i32 %lftr.wideiv, %n
|
||||
br i1 %exitcond, label %for.end, label %for.body
|
||||
|
||||
for.end: ; preds = %for.body, %entry
|
||||
ret void
|
||||
}
|
||||
|
||||
; Be careful if decrementing the constant would undeflow.
|
||||
|
||||
define void @loop_const_folding_underflow(<2 x i64>* %in, <2 x i64>* %out, i32 %n) {
|
||||
; CHECK-NOT: .short 25
|
||||
; CHECK-LABEL: loop_const_folding_underflow:
|
||||
; CHECK-NOT: psubusw
|
||||
entry:
|
||||
%cmp9 = icmp eq i32 %n, 0
|
||||
br i1 %cmp9, label %for.end, label %for.body
|
||||
|
||||
for.body: ; preds = %for.body, %entry
|
||||
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body ], [ 0, %entry ]
|
||||
%arrayidx1 = getelementptr inbounds <2 x i64>* %in, i64 %indvars.iv
|
||||
%arrayidx1.val = load <2 x i64>* %arrayidx1, align 16
|
||||
%0 = bitcast <2 x i64> %arrayidx1.val to <8 x i16>
|
||||
%cmp.i.i = icmp ult <8 x i16> %0, <i16 0, i16 26, i16 26, i16 26, i16 26, i16 26, i16 26, i16 26>
|
||||
%sext.i.i = sext <8 x i1> %cmp.i.i to <8 x i16>
|
||||
%1 = bitcast <8 x i16> %sext.i.i to <2 x i64>
|
||||
%arrayidx5 = getelementptr inbounds <2 x i64>* %out, i64 %indvars.iv
|
||||
store <2 x i64> %1, <2 x i64>* %arrayidx5, align 16
|
||||
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
|
||||
%lftr.wideiv = trunc i64 %indvars.iv.next to i32
|
||||
%exitcond = icmp eq i32 %lftr.wideiv, %n
|
||||
br i1 %exitcond, label %for.end, label %for.body
|
||||
|
||||
for.end: ; preds = %for.body, %entry
|
||||
ret void
|
||||
}
|
||||
|
||||
; Test for PSUBUSB
|
||||
|
||||
define <16 x i8> @test_ult_byte(<16 x i8> %a) {
|
||||
; CHECK: .space 16,10
|
||||
; CHECK-LABEL: test_ult_byte:
|
||||
; CHECK: psubus
|
||||
entry:
|
||||
%icmp = icmp ult <16 x i8> %a, <i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11, i8 11>
|
||||
%sext = sext <16 x i1> %icmp to <16 x i8>
|
||||
ret <16 x i8> %sext
|
||||
}
|
||||
|
||||
; Only do this when we can turn the comparison into a setule. I.e. not for
|
||||
; register operands.
|
||||
|
||||
define <8 x i16> @test_ult_register(<8 x i16> %a, <8 x i16> %b) {
|
||||
; CHECK-LABEL: test_ult_register:
|
||||
; CHECK-NOT: psubus
|
||||
entry:
|
||||
%icmp = icmp ult <8 x i16> %a, %b
|
||||
%sext = sext <8 x i1> %icmp to <8 x i16>
|
||||
ret <8 x i16> %sext
|
||||
}
|
||||
|
|
@ -42,12 +42,9 @@ define <8 x i16> @v8i16_icmp_uge(<8 x i16> %a, <8 x i16> %b) nounwind readnone s
|
|||
%2 = sext <8 x i1> %1 to <8 x i16>
|
||||
ret <8 x i16> %2
|
||||
; SSE2-LABEL: v8i16_icmp_uge:
|
||||
; SSE2: movdqa {{.*}}(%rip), %xmm2
|
||||
; SEE2: pxor %xmm2, %xmm0
|
||||
; SSE2: pxor %xmm1, %xmm2
|
||||
; SSE2: pcmpgtw %xmm0, %xmm2
|
||||
; SSE2: pcmpeqd %xmm0, %xmm0
|
||||
; SSE2: pxor %xmm2, %xmm0
|
||||
; SSE2: psubusw %xmm0, %xmm1
|
||||
; SEE2: pxor %xmm0, %xmm0
|
||||
; SSE2: pcmpeqw %xmm1, %xmm0
|
||||
|
||||
; SSE41-LABEL: v8i16_icmp_uge:
|
||||
; SSE41: pmaxuw %xmm0, %xmm1
|
||||
|
|
@ -63,12 +60,9 @@ define <8 x i16> @v8i16_icmp_ule(<8 x i16> %a, <8 x i16> %b) nounwind readnone s
|
|||
%2 = sext <8 x i1> %1 to <8 x i16>
|
||||
ret <8 x i16> %2
|
||||
; SSE2-LABEL: v8i16_icmp_ule:
|
||||
; SSE2: movdqa {{.*}}(%rip), %xmm2
|
||||
; SSE2: pxor %xmm2, %xmm1
|
||||
; SSE2: pxor %xmm2, %xmm0
|
||||
; SSE2: pcmpgtw %xmm1, %xmm0
|
||||
; SSE2: pcmpeqd %xmm1, %xmm1
|
||||
; SSE2: pxor %xmm0, %xmm1
|
||||
; SSE2: psubusw %xmm1, %xmm0
|
||||
; SSE2: pxor %xmm1, %xmm1
|
||||
; SSE2: pcmpeqw %xmm0, %xmm1
|
||||
; SSE2: movdqa %xmm1, %xmm0
|
||||
|
||||
; SSE41-LABEL: v8i16_icmp_ule:
|
||||
|
|
|
|||
Loading…
Reference in New Issue