Mark x86's V_SET0 and V_SETALLONES with isSimpleLoad, and teach X86's
foldMemoryOperand how to "fold" them, by converting them into constant-pool loads. When they aren't folded, they use xorps/cmpeqd, but for example when register pressure is high, they may now be folded as memory operands, which reduces register pressure. Also, mark V_SET0 isAsCheapAsAMove so that two-address-elimination will remat it instead of copying zeros around (V_SETALLONES was already marked). llvm-svn: 60461
This commit is contained in:
Dan Gohman
parent
e3402692d8
commit
cc78cdf275
|
|
@ -189,7 +189,7 @@ class Instruction {
|
|||
bit isIndirectBranch = 0; // Is this instruction an indirect branch?
|
||||
bit isBarrier = 0; // Can control flow fall through this instruction?
|
||||
bit isCall = 0; // Is this instruction a call instruction?
|
||||
bit isSimpleLoad = 0; // Is this just a load instruction?
|
||||
bit isSimpleLoad = 0; // Can this be folded as a memory operand?
|
||||
bit mayLoad = 0; // Is it possible for this inst to read memory?
|
||||
bit mayStore = 0; // Is it possible for this inst to write memory?
|
||||
bit isTwoAddress = 0; // Is this a two address instruction?
|
||||
|
|
|
|||
|
|
@ -301,11 +301,14 @@ public:
|
|||
return Flags & (1 << TID::DelaySlot);
|
||||
}
|
||||
|
||||
/// isSimpleLoad - Return true for instructions that are simple loads from
|
||||
/// memory. This should only be set on instructions that load a value from
|
||||
/// memory and return it in their only virtual register definition.
|
||||
/// Instructions that return a value loaded from memory and then modified in
|
||||
/// some way should not return true for this.
|
||||
/// isSimpleLoad - Return true for instructions that can be folded as
|
||||
/// memory operands in other instructions. The most common use for this
|
||||
/// is instructions that are simple loads from memory that don't modify
|
||||
/// the loaded value in any way, but it can also be used for instructions
|
||||
/// that can be expressed as constant-pool loads, such as V_SETALLONES
|
||||
/// on x86, to allow them to be folded when it is beneficial.
|
||||
/// This should only be set on instructions that return a value in their
|
||||
/// only virtual register definition.
|
||||
bool isSimpleLoad() const {
|
||||
return Flags & (1 << TID::SimpleLoad);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@
|
|||
#include "X86Subtarget.h"
|
||||
#include "X86TargetMachine.h"
|
||||
#include "llvm/ADT/STLExtras.h"
|
||||
#include "llvm/CodeGen/MachineConstantPool.h"
|
||||
#include "llvm/CodeGen/MachineFrameInfo.h"
|
||||
#include "llvm/CodeGen/MachineInstrBuilder.h"
|
||||
#include "llvm/CodeGen/MachineRegisterInfo.h"
|
||||
|
|
@ -2127,9 +2128,36 @@ MachineInstr* X86InstrInfo::foldMemoryOperand(MachineFunction &MF,
|
|||
return NULL;
|
||||
|
||||
SmallVector<MachineOperand,4> MOs;
|
||||
unsigned NumOps = LoadMI->getDesc().getNumOperands();
|
||||
for (unsigned i = NumOps - 4; i != NumOps; ++i)
|
||||
MOs.push_back(LoadMI->getOperand(i));
|
||||
if (LoadMI->getOpcode() == X86::V_SET0 ||
|
||||
LoadMI->getOpcode() == X86::V_SETALLONES) {
|
||||
// Folding a V_SET0 or V_SETALLONES as a load, to ease register pressure.
|
||||
// Create a constant-pool entry and operands to load from it.
|
||||
|
||||
// x86-32 PIC requires a PIC base register for constant pools.
|
||||
unsigned PICBase = 0;
|
||||
if (TM.getRelocationModel() == Reloc::PIC_ &&
|
||||
!TM.getSubtarget<X86Subtarget>().is64Bit())
|
||||
PICBase = TM.getInstrInfo()->getGlobalBaseReg(&MF);
|
||||
|
||||
// Create a v4i32 constant-pool entry.
|
||||
MachineConstantPool &MCP = *MF.getConstantPool();
|
||||
const VectorType *Ty = VectorType::get(Type::Int32Ty, 4);
|
||||
Constant *C = LoadMI->getOpcode() == X86::V_SET0 ?
|
||||
ConstantVector::getNullValue(Ty) :
|
||||
ConstantVector::getAllOnesValue(Ty);
|
||||
unsigned CPI = MCP.getConstantPoolIndex(C, /*AlignmentLog2=*/4);
|
||||
|
||||
// Create operands to load from the constant pool entry.
|
||||
MOs.push_back(MachineOperand::CreateReg(PICBase, false));
|
||||
MOs.push_back(MachineOperand::CreateImm(1));
|
||||
MOs.push_back(MachineOperand::CreateReg(0, false));
|
||||
MOs.push_back(MachineOperand::CreateCPI(CPI, 0));
|
||||
} else {
|
||||
// Folding a normal load. Just copy the load's address operands.
|
||||
unsigned NumOps = LoadMI->getDesc().getNumOperands();
|
||||
for (unsigned i = NumOps - 4; i != NumOps; ++i)
|
||||
MOs.push_back(LoadMI->getOperand(i));
|
||||
}
|
||||
return foldMemoryOperand(MF, MI, Ops[0], MOs);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -987,7 +987,9 @@ def STMXCSR : PSI<0xAE, MRM3m, (outs), (ins i32mem:$dst),
|
|||
"stmxcsr\t$dst", [(int_x86_sse_stmxcsr addr:$dst)]>;
|
||||
|
||||
// Alias instructions that map zero vector to pxor / xorp* for sse.
|
||||
let isReMaterializable = 1 in
|
||||
// We set isSimpleLoad because this can be converted to a constant-pool
|
||||
// load of an all-zeros value if folding it would be beneficial.
|
||||
let isReMaterializable = 1, isAsCheapAsAMove = 1, isSimpleLoad = 1 in
|
||||
def V_SET0 : PSI<0x57, MRMInitReg, (outs VR128:$dst), (ins),
|
||||
"xorps\t$dst, $dst",
|
||||
[(set VR128:$dst, (v4i32 immAllZerosV))]>;
|
||||
|
|
@ -2253,7 +2255,9 @@ def : Pat<(membarrier (i8 imm:$ll), (i8 imm:$ls), (i8 imm:$sl), (i8 imm:$ss),
|
|||
(i8 1)), (MFENCE)>;
|
||||
|
||||
// Alias instructions that map zero vector to pxor / xorp* for sse.
|
||||
let isReMaterializable = 1, isAsCheapAsAMove = 1 in
|
||||
// We set isSimpleLoad because this can be converted to a constant-pool
|
||||
// load of an all-ones value if folding it would be beneficial.
|
||||
let isReMaterializable = 1, isAsCheapAsAMove = 1, isSimpleLoad = 1 in
|
||||
def V_SETALLONES : PDI<0x76, MRMInitReg, (outs VR128:$dst), (ins),
|
||||
"pcmpeqd\t$dst, $dst",
|
||||
[(set VR128:$dst, (v4i32 immAllOnesV))]>;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,107 @@
|
|||
; RUN: llvm-as < %s | llc | not grep pcmpeqd
|
||||
; RUN: llvm-as < %s | llc -march=x86-64 | grep pcmpeqd | count 1
|
||||
|
||||
; On x86-64, this testcase shouldn't need to spill the -1 value,
|
||||
; so it should just use pcmpeqd to materialize an all-ones vector.
|
||||
; On x86-32, there aren't enough registers, so an all-ones
|
||||
; constant pool should be created so it can be folded.
|
||||
|
||||
target datalayout = "e-p:32:32:32-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-f32:32:32-f64:32:64-v64:64:64-v128:128:128-a0:0:64-f80:32:32"
|
||||
target triple = "i686-apple-cl.1.0"
|
||||
%struct.__ImageExecInfo = type <{ <4 x i32>, <4 x float>, <2 x i64>, i8*, i8*, i8*, i32, i32, i32, i32, i32 }>
|
||||
%struct._cl_image_format_t = type <{ i32, i32, i32 }>
|
||||
%struct._image2d_t = type <{ i8*, %struct._cl_image_format_t, i32, i32, i32, i32, i32, i32 }>
|
||||
|
||||
define void @program_1(%struct._image2d_t* %dest, %struct._image2d_t* %t0, <4 x float> %p0, <4 x float> %p1, <4 x float> %p4, <4 x float> %p5, <4 x float> %p6) nounwind {
|
||||
entry:
|
||||
%tmp3.i = load i32* null ; <i32> [#uses=1]
|
||||
%cmp = icmp sgt i32 %tmp3.i, 200 ; <i1> [#uses=1]
|
||||
br i1 %cmp, label %forcond, label %ifthen
|
||||
|
||||
ifthen: ; preds = %entry
|
||||
ret void
|
||||
|
||||
forcond: ; preds = %entry
|
||||
%tmp3.i536 = load i32* null ; <i32> [#uses=1]
|
||||
%cmp12 = icmp slt i32 0, %tmp3.i536 ; <i1> [#uses=1]
|
||||
br i1 %cmp12, label %forbody, label %afterfor
|
||||
|
||||
forbody: ; preds = %forcond
|
||||
%bitcast204.i313 = bitcast <4 x i32> zeroinitializer to <4 x float> ; <<4 x float>> [#uses=1]
|
||||
%mul233 = mul <4 x float> %bitcast204.i313, zeroinitializer ; <<4 x float>> [#uses=1]
|
||||
%mul257 = mul <4 x float> %mul233, zeroinitializer ; <<4 x float>> [#uses=1]
|
||||
%mul275 = mul <4 x float> %mul257, zeroinitializer ; <<4 x float>> [#uses=1]
|
||||
%tmp51 = call <4 x float> @llvm.x86.sse.max.ps(<4 x float> %mul275, <4 x float> zeroinitializer) nounwind ; <<4 x float>> [#uses=1]
|
||||
%bitcast198.i182 = bitcast <4 x float> zeroinitializer to <4 x i32> ; <<4 x i32>> [#uses=0]
|
||||
%bitcast204.i185 = bitcast <4 x i32> zeroinitializer to <4 x float> ; <<4 x float>> [#uses=1]
|
||||
%tmp69 = call <4 x i32> @llvm.x86.sse2.cvttps2dq(<4 x float> zeroinitializer) nounwind ; <<4 x i32>> [#uses=1]
|
||||
%tmp70 = call <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32> %tmp69) nounwind ; <<4 x float>> [#uses=1]
|
||||
%sub140.i78 = sub <4 x float> zeroinitializer, %tmp70 ; <<4 x float>> [#uses=2]
|
||||
%mul166.i86 = mul <4 x float> zeroinitializer, %sub140.i78 ; <<4 x float>> [#uses=1]
|
||||
%add167.i87 = add <4 x float> %mul166.i86, < float 0x3FE62ACB60000000, float 0x3FE62ACB60000000, float 0x3FE62ACB60000000, float 0x3FE62ACB60000000 > ; <<4 x float>> [#uses=1]
|
||||
%mul171.i88 = mul <4 x float> %add167.i87, %sub140.i78 ; <<4 x float>> [#uses=1]
|
||||
%add172.i89 = add <4 x float> %mul171.i88, < float 0x3FF0000A40000000, float 0x3FF0000A40000000, float 0x3FF0000A40000000, float 0x3FF0000A40000000 > ; <<4 x float>> [#uses=1]
|
||||
%bitcast176.i90 = bitcast <4 x float> %add172.i89 to <4 x i32> ; <<4 x i32>> [#uses=1]
|
||||
%andnps178.i92 = and <4 x i32> %bitcast176.i90, zeroinitializer ; <<4 x i32>> [#uses=1]
|
||||
%bitcast179.i93 = bitcast <4 x i32> %andnps178.i92 to <4 x float> ; <<4 x float>> [#uses=1]
|
||||
%mul186.i96 = mul <4 x float> %bitcast179.i93, zeroinitializer ; <<4 x float>> [#uses=1]
|
||||
%bitcast190.i98 = bitcast <4 x float> %mul186.i96 to <4 x i32> ; <<4 x i32>> [#uses=1]
|
||||
%andnps192.i100 = and <4 x i32> %bitcast190.i98, zeroinitializer ; <<4 x i32>> [#uses=1]
|
||||
%xorps.i102 = xor <4 x i32> zeroinitializer, < i32 -1, i32 -1, i32 -1, i32 -1 > ; <<4 x i32>> [#uses=1]
|
||||
%orps203.i103 = or <4 x i32> %andnps192.i100, %xorps.i102 ; <<4 x i32>> [#uses=1]
|
||||
%bitcast204.i104 = bitcast <4 x i32> %orps203.i103 to <4 x float> ; <<4 x float>> [#uses=1]
|
||||
%cmple.i = call <4 x float> @llvm.x86.sse.cmp.ps(<4 x float> zeroinitializer, <4 x float> %tmp51, i8 2) nounwind ; <<4 x float>> [#uses=1]
|
||||
%tmp80 = call <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32> zeroinitializer) nounwind ; <<4 x float>> [#uses=1]
|
||||
%sub140.i = sub <4 x float> zeroinitializer, %tmp80 ; <<4 x float>> [#uses=1]
|
||||
%bitcast148.i = bitcast <4 x float> zeroinitializer to <4 x i32> ; <<4 x i32>> [#uses=1]
|
||||
%andnps150.i = and <4 x i32> %bitcast148.i, < i32 -2139095041, i32 -2139095041, i32 -2139095041, i32 -2139095041 > ; <<4 x i32>> [#uses=0]
|
||||
%mul171.i = mul <4 x float> zeroinitializer, %sub140.i ; <<4 x float>> [#uses=1]
|
||||
%add172.i = add <4 x float> %mul171.i, < float 0x3FF0000A40000000, float 0x3FF0000A40000000, float 0x3FF0000A40000000, float 0x3FF0000A40000000 > ; <<4 x float>> [#uses=1]
|
||||
%bitcast176.i = bitcast <4 x float> %add172.i to <4 x i32> ; <<4 x i32>> [#uses=1]
|
||||
%andnps178.i = and <4 x i32> %bitcast176.i, zeroinitializer ; <<4 x i32>> [#uses=1]
|
||||
%bitcast179.i = bitcast <4 x i32> %andnps178.i to <4 x float> ; <<4 x float>> [#uses=1]
|
||||
%mul186.i = mul <4 x float> %bitcast179.i, zeroinitializer ; <<4 x float>> [#uses=1]
|
||||
%bitcast189.i = bitcast <4 x float> zeroinitializer to <4 x i32> ; <<4 x i32>> [#uses=0]
|
||||
%bitcast190.i = bitcast <4 x float> %mul186.i to <4 x i32> ; <<4 x i32>> [#uses=1]
|
||||
%andnps192.i = and <4 x i32> %bitcast190.i, zeroinitializer ; <<4 x i32>> [#uses=1]
|
||||
%bitcast198.i = bitcast <4 x float> %cmple.i to <4 x i32> ; <<4 x i32>> [#uses=1]
|
||||
%xorps.i = xor <4 x i32> %bitcast198.i, < i32 -1, i32 -1, i32 -1, i32 -1 > ; <<4 x i32>> [#uses=1]
|
||||
%orps203.i = or <4 x i32> %andnps192.i, %xorps.i ; <<4 x i32>> [#uses=1]
|
||||
%bitcast204.i = bitcast <4 x i32> %orps203.i to <4 x float> ; <<4 x float>> [#uses=1]
|
||||
%mul307 = mul <4 x float> %bitcast204.i185, zeroinitializer ; <<4 x float>> [#uses=1]
|
||||
%mul310 = mul <4 x float> %bitcast204.i104, zeroinitializer ; <<4 x float>> [#uses=2]
|
||||
%mul313 = mul <4 x float> %bitcast204.i, zeroinitializer ; <<4 x float>> [#uses=1]
|
||||
%tmp82 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %mul307, <4 x float> zeroinitializer) nounwind ; <<4 x float>> [#uses=1]
|
||||
%bitcast11.i15 = bitcast <4 x float> %tmp82 to <4 x i32> ; <<4 x i32>> [#uses=1]
|
||||
%andnps.i17 = and <4 x i32> %bitcast11.i15, zeroinitializer ; <<4 x i32>> [#uses=1]
|
||||
%orps.i18 = or <4 x i32> %andnps.i17, zeroinitializer ; <<4 x i32>> [#uses=1]
|
||||
%bitcast17.i19 = bitcast <4 x i32> %orps.i18 to <4 x float> ; <<4 x float>> [#uses=1]
|
||||
%tmp83 = call <4 x float> @llvm.x86.sse.min.ps(<4 x float> %mul310, <4 x float> zeroinitializer) nounwind ; <<4 x float>> [#uses=1]
|
||||
%bitcast.i3 = bitcast <4 x float> %mul310 to <4 x i32> ; <<4 x i32>> [#uses=1]
|
||||
%bitcast6.i4 = bitcast <4 x float> zeroinitializer to <4 x i32> ; <<4 x i32>> [#uses=2]
|
||||
%andps.i5 = and <4 x i32> %bitcast.i3, %bitcast6.i4 ; <<4 x i32>> [#uses=1]
|
||||
%bitcast11.i6 = bitcast <4 x float> %tmp83 to <4 x i32> ; <<4 x i32>> [#uses=1]
|
||||
%not.i7 = xor <4 x i32> %bitcast6.i4, < i32 -1, i32 -1, i32 -1, i32 -1 > ; <<4 x i32>> [#uses=1]
|
||||
%andnps.i8 = and <4 x i32> %bitcast11.i6, %not.i7 ; <<4 x i32>> [#uses=1]
|
||||
%orps.i9 = or <4 x i32> %andnps.i8, %andps.i5 ; <<4 x i32>> [#uses=1]
|
||||
%bitcast17.i10 = bitcast <4 x i32> %orps.i9 to <4 x float> ; <<4 x float>> [#uses=1]
|
||||
%bitcast.i = bitcast <4 x float> %mul313 to <4 x i32> ; <<4 x i32>> [#uses=1]
|
||||
%andps.i = and <4 x i32> %bitcast.i, zeroinitializer ; <<4 x i32>> [#uses=1]
|
||||
%orps.i = or <4 x i32> zeroinitializer, %andps.i ; <<4 x i32>> [#uses=1]
|
||||
%bitcast17.i = bitcast <4 x i32> %orps.i to <4 x float> ; <<4 x float>> [#uses=1]
|
||||
call void null(<4 x float> %bitcast17.i19, <4 x float> %bitcast17.i10, <4 x float> %bitcast17.i, <4 x float> zeroinitializer, %struct.__ImageExecInfo* null, <4 x i32> zeroinitializer) nounwind
|
||||
unreachable
|
||||
|
||||
afterfor: ; preds = %forcond
|
||||
ret void
|
||||
}
|
||||
|
||||
declare <4 x float> @llvm.x86.sse.cmp.ps(<4 x float>, <4 x float>, i8) nounwind readnone
|
||||
|
||||
declare <4 x float> @llvm.x86.sse2.cvtdq2ps(<4 x i32>) nounwind readnone
|
||||
|
||||
declare <4 x i32> @llvm.x86.sse2.cvttps2dq(<4 x float>) nounwind readnone
|
||||
|
||||
declare <4 x float> @llvm.x86.sse.max.ps(<4 x float>, <4 x float>) nounwind readnone
|
||||
|
||||
declare <4 x float> @llvm.x86.sse.min.ps(<4 x float>, <4 x float>) nounwind readnone
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
; RUN: llvm-as < %s | llc -march=x86 -mattr=+sse2 > %t
|
||||
; RUN: grep pcmpeqd %t | count 1
|
||||
; RUN: grep xor %t | count 1
|
||||
; RUN: not grep LCP %t
|
||||
|
||||
define <2 x double> @foo() nounwind {
|
||||
ret <2 x double> bitcast (<2 x i64><i64 -1, i64 -1> to <2 x double>)
|
||||
}
|
||||
define <2 x double> @bar() nounwind {
|
||||
ret <2 x double> bitcast (<2 x i64><i64 0, i64 0> to <2 x double>)
|
||||
}
|
||||
|
|
@ -1759,15 +1759,6 @@ static void InferFromPattern(const CodeGenInstruction &Inst,
|
|||
MayLoad = true;
|
||||
}
|
||||
|
||||
// Sanity-check the isSimpleLoad flag.
|
||||
if (Inst.isSimpleLoad) {
|
||||
if (!MayLoad)
|
||||
fprintf(stderr,
|
||||
"Warning: mayLoad flag not set or inferred for instruction '%s'"
|
||||
" which has isSimpleLoad set.\n",
|
||||
Inst.TheDef->getName().c_str());
|
||||
}
|
||||
|
||||
if (Inst.neverHasSideEffects) {
|
||||
if (HadPattern)
|
||||
fprintf(stderr, "Warning: neverHasSideEffects set on instruction '%s' "
|
||||
|
|
|
|||
Loading…
Reference in New Issue