Misc optimizer+codegen work for 'cmpxchg' and 'atomicrmw'. They appear to be
working on x86 (at least for trivial testcases); other architectures will need more work so that they actually emit the appropriate instructions for orderings stricter than 'monotonic'. (As far as I can tell, the ARM, PPC, Mips, and Alpha backends need such changes.) llvm-svn: 136457
This commit is contained in:
parent
0978426843
commit
adec587d5c
|
@ -342,6 +342,10 @@ public:
|
||||||
case Instruction::Load: return getModRefInfo((const LoadInst*)I, Loc);
|
case Instruction::Load: return getModRefInfo((const LoadInst*)I, Loc);
|
||||||
case Instruction::Store: return getModRefInfo((const StoreInst*)I, Loc);
|
case Instruction::Store: return getModRefInfo((const StoreInst*)I, Loc);
|
||||||
case Instruction::Fence: return getModRefInfo((const FenceInst*)I, Loc);
|
case Instruction::Fence: return getModRefInfo((const FenceInst*)I, Loc);
|
||||||
|
case Instruction::AtomicCmpXchg:
|
||||||
|
return getModRefInfo((const AtomicCmpXchgInst*)I, Loc);
|
||||||
|
case Instruction::AtomicRMW:
|
||||||
|
return getModRefInfo((const AtomicRMWInst*)I, Loc);
|
||||||
case Instruction::Call: return getModRefInfo((const CallInst*)I, Loc);
|
case Instruction::Call: return getModRefInfo((const CallInst*)I, Loc);
|
||||||
case Instruction::Invoke: return getModRefInfo((const InvokeInst*)I,Loc);
|
case Instruction::Invoke: return getModRefInfo((const InvokeInst*)I,Loc);
|
||||||
default: return NoModRef;
|
default: return NoModRef;
|
||||||
|
@ -420,6 +424,32 @@ public:
|
||||||
return getModRefInfo(S, Location(P, Size));
|
return getModRefInfo(S, Location(P, Size));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// getModRefInfo (for cmpxchges) - Return whether information about whether
|
||||||
|
/// a particular cmpxchg modifies or reads the specified memory location.
|
||||||
|
ModRefResult getModRefInfo(const AtomicCmpXchgInst *CX, const Location &Loc) {
|
||||||
|
// Conservatively correct. (But there are obvious ways to be smarter.)
|
||||||
|
return ModRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// getModRefInfo (for cmpxchges) - A convenience wrapper.
|
||||||
|
ModRefResult getModRefInfo(const AtomicCmpXchgInst *CX,
|
||||||
|
const Value *P, unsigned Size) {
|
||||||
|
return getModRefInfo(CX, Location(P, Size));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// getModRefInfo (for atomicrmws) - Return whether information about whether
|
||||||
|
/// a particular atomicrmw modifies or reads the specified memory location.
|
||||||
|
ModRefResult getModRefInfo(const AtomicRMWInst *RMW, const Location &Loc) {
|
||||||
|
// Conservatively correct. (But there are obvious ways to be smarter.)
|
||||||
|
return ModRef;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// getModRefInfo (for atomicrmws) - A convenience wrapper.
|
||||||
|
ModRefResult getModRefInfo(const AtomicRMWInst *RMW,
|
||||||
|
const Value *P, unsigned Size) {
|
||||||
|
return getModRefInfo(RMW, Location(P, Size));
|
||||||
|
}
|
||||||
|
|
||||||
/// getModRefInfo (for va_args) - Return whether information about whether
|
/// getModRefInfo (for va_args) - Return whether information about whether
|
||||||
/// a particular va_arg modifies or reads the specified memory location.
|
/// a particular va_arg modifies or reads the specified memory location.
|
||||||
ModRefResult getModRefInfo(const VAArgInst* I, const Location &Loc);
|
ModRefResult getModRefInfo(const VAArgInst* I, const Location &Loc);
|
||||||
|
|
|
@ -589,19 +589,27 @@ public:
|
||||||
/// takes 3 operands
|
/// takes 3 operands
|
||||||
SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
|
SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
|
||||||
SDValue Ptr, SDValue Cmp, SDValue Swp,
|
SDValue Ptr, SDValue Cmp, SDValue Swp,
|
||||||
MachinePointerInfo PtrInfo, unsigned Alignment=0);
|
MachinePointerInfo PtrInfo, unsigned Alignment,
|
||||||
|
AtomicOrdering Ordering,
|
||||||
|
SynchronizationScope SynchScope);
|
||||||
SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
|
SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
|
||||||
SDValue Ptr, SDValue Cmp, SDValue Swp,
|
SDValue Ptr, SDValue Cmp, SDValue Swp,
|
||||||
MachineMemOperand *MMO);
|
MachineMemOperand *MMO,
|
||||||
|
AtomicOrdering Ordering,
|
||||||
|
SynchronizationScope SynchScope);
|
||||||
|
|
||||||
/// getAtomic - Gets a node for an atomic op, produces result and chain and
|
/// getAtomic - Gets a node for an atomic op, produces result and chain and
|
||||||
/// takes 2 operands.
|
/// takes 2 operands.
|
||||||
SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
|
SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
|
||||||
SDValue Ptr, SDValue Val, const Value* PtrVal,
|
SDValue Ptr, SDValue Val, const Value* PtrVal,
|
||||||
unsigned Alignment = 0);
|
unsigned Alignment,
|
||||||
|
AtomicOrdering Ordering,
|
||||||
|
SynchronizationScope SynchScope);
|
||||||
SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
|
SDValue getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT, SDValue Chain,
|
||||||
SDValue Ptr, SDValue Val,
|
SDValue Ptr, SDValue Val,
|
||||||
MachineMemOperand *MMO);
|
MachineMemOperand *MMO,
|
||||||
|
AtomicOrdering Ordering,
|
||||||
|
SynchronizationScope SynchScope);
|
||||||
|
|
||||||
/// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a
|
/// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a
|
||||||
/// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
|
/// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
|
||||||
|
|
|
@ -20,6 +20,7 @@
|
||||||
#define LLVM_CODEGEN_SELECTIONDAGNODES_H
|
#define LLVM_CODEGEN_SELECTIONDAGNODES_H
|
||||||
|
|
||||||
#include "llvm/Constants.h"
|
#include "llvm/Constants.h"
|
||||||
|
#include "llvm/Instructions.h"
|
||||||
#include "llvm/ADT/FoldingSet.h"
|
#include "llvm/ADT/FoldingSet.h"
|
||||||
#include "llvm/ADT/GraphTraits.h"
|
#include "llvm/ADT/GraphTraits.h"
|
||||||
#include "llvm/ADT/ilist_node.h"
|
#include "llvm/ADT/ilist_node.h"
|
||||||
|
@ -917,6 +918,13 @@ public:
|
||||||
bool isVolatile() const { return (SubclassData >> 5) & 1; }
|
bool isVolatile() const { return (SubclassData >> 5) & 1; }
|
||||||
bool isNonTemporal() const { return (SubclassData >> 6) & 1; }
|
bool isNonTemporal() const { return (SubclassData >> 6) & 1; }
|
||||||
|
|
||||||
|
AtomicOrdering getOrdering() const {
|
||||||
|
return AtomicOrdering((SubclassData >> 7) & 15);
|
||||||
|
}
|
||||||
|
SynchronizationScope getSynchScope() const {
|
||||||
|
return SynchronizationScope((SubclassData >> 11) & 1);
|
||||||
|
}
|
||||||
|
|
||||||
/// Returns the SrcValue and offset that describes the location of the access
|
/// Returns the SrcValue and offset that describes the location of the access
|
||||||
const Value *getSrcValue() const { return MMO->getValue(); }
|
const Value *getSrcValue() const { return MMO->getValue(); }
|
||||||
int64_t getSrcValueOffset() const { return MMO->getOffset(); }
|
int64_t getSrcValueOffset() const { return MMO->getOffset(); }
|
||||||
|
@ -977,6 +985,21 @@ public:
|
||||||
class AtomicSDNode : public MemSDNode {
|
class AtomicSDNode : public MemSDNode {
|
||||||
SDUse Ops[4];
|
SDUse Ops[4];
|
||||||
|
|
||||||
|
void InitAtomic(AtomicOrdering Ordering, SynchronizationScope SynchScope) {
|
||||||
|
// This must match encodeMemSDNodeFlags() in SelectionDAG.cpp.
|
||||||
|
assert((Ordering & 15) == Ordering &&
|
||||||
|
"Ordering may not require more than 4 bits!");
|
||||||
|
assert((SynchScope & 1) == SynchScope &&
|
||||||
|
"SynchScope may not require more than 1 bit!");
|
||||||
|
SubclassData |= Ordering << 7;
|
||||||
|
SubclassData |= SynchScope << 11;
|
||||||
|
assert(getOrdering() == Ordering && "Ordering encoding error!");
|
||||||
|
assert(getSynchScope() == SynchScope && "Synch-scope encoding error!");
|
||||||
|
|
||||||
|
assert(readMem() && "Atomic MachineMemOperand is not a load!");
|
||||||
|
assert(writeMem() && "Atomic MachineMemOperand is not a store!");
|
||||||
|
}
|
||||||
|
|
||||||
public:
|
public:
|
||||||
// Opc: opcode for atomic
|
// Opc: opcode for atomic
|
||||||
// VTL: value type list
|
// VTL: value type list
|
||||||
|
@ -988,18 +1011,18 @@ public:
|
||||||
// Align: alignment of memory
|
// Align: alignment of memory
|
||||||
AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
|
AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
|
||||||
SDValue Chain, SDValue Ptr,
|
SDValue Chain, SDValue Ptr,
|
||||||
SDValue Cmp, SDValue Swp, MachineMemOperand *MMO)
|
SDValue Cmp, SDValue Swp, MachineMemOperand *MMO,
|
||||||
|
AtomicOrdering Ordering, SynchronizationScope SynchScope)
|
||||||
: MemSDNode(Opc, dl, VTL, MemVT, MMO) {
|
: MemSDNode(Opc, dl, VTL, MemVT, MMO) {
|
||||||
assert(readMem() && "Atomic MachineMemOperand is not a load!");
|
InitAtomic(Ordering, SynchScope);
|
||||||
assert(writeMem() && "Atomic MachineMemOperand is not a store!");
|
|
||||||
InitOperands(Ops, Chain, Ptr, Cmp, Swp);
|
InitOperands(Ops, Chain, Ptr, Cmp, Swp);
|
||||||
}
|
}
|
||||||
AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
|
AtomicSDNode(unsigned Opc, DebugLoc dl, SDVTList VTL, EVT MemVT,
|
||||||
SDValue Chain, SDValue Ptr,
|
SDValue Chain, SDValue Ptr,
|
||||||
SDValue Val, MachineMemOperand *MMO)
|
SDValue Val, MachineMemOperand *MMO,
|
||||||
|
AtomicOrdering Ordering, SynchronizationScope SynchScope)
|
||||||
: MemSDNode(Opc, dl, VTL, MemVT, MMO) {
|
: MemSDNode(Opc, dl, VTL, MemVT, MMO) {
|
||||||
assert(readMem() && "Atomic MachineMemOperand is not a load!");
|
InitAtomic(Ordering, SynchScope);
|
||||||
assert(writeMem() && "Atomic MachineMemOperand is not a store!");
|
|
||||||
InitOperands(Ops, Chain, Ptr, Val);
|
InitOperands(Ops, Chain, Ptr, Val);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -770,6 +770,16 @@ public:
|
||||||
SynchronizationScope SynchScope = CrossThread) {
|
SynchronizationScope SynchScope = CrossThread) {
|
||||||
return Insert(new FenceInst(Context, Ordering, SynchScope));
|
return Insert(new FenceInst(Context, Ordering, SynchScope));
|
||||||
}
|
}
|
||||||
|
FenceInst *CreateAtomicCmpXchg(Value *Ptr, Value *Cmp, Value *New,
|
||||||
|
AtomicOrdering Ordering,
|
||||||
|
SynchronizationScope SynchScope = CrossThread){
|
||||||
|
return Insert(new AtomicCmpXchgInst(Ptr, Cmp, New, Ordering, SynchScope));
|
||||||
|
}
|
||||||
|
FenceInst *CreateAtomicRMW(AtomicRMWInst::BinOp Op, Value *Ptr, Value *Val,
|
||||||
|
AtomicOrdering Ordering,
|
||||||
|
SynchronizationScope SynchScope = CrossThread) {
|
||||||
|
return Insert(new AtomicRMWInst(Op, Ptr, Val, Ordering, SynchScope));
|
||||||
|
}
|
||||||
Value *CreateGEP(Value *Ptr, ArrayRef<Value *> IdxList,
|
Value *CreateGEP(Value *Ptr, ArrayRef<Value *> IdxList,
|
||||||
const Twine &Name = "") {
|
const Twine &Name = "") {
|
||||||
if (Constant *PC = dyn_cast<Constant>(Ptr)) {
|
if (Constant *PC = dyn_cast<Constant>(Ptr)) {
|
||||||
|
|
|
@ -155,7 +155,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
|
||||||
SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
|
SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
|
||||||
N->getMemoryVT(),
|
N->getMemoryVT(),
|
||||||
N->getChain(), N->getBasePtr(),
|
N->getChain(), N->getBasePtr(),
|
||||||
Op2, N->getMemOperand());
|
Op2, N->getMemOperand(), N->getOrdering(),
|
||||||
|
N->getSynchScope());
|
||||||
// Legalized the chain result - switch anything that used the old chain to
|
// Legalized the chain result - switch anything that used the old chain to
|
||||||
// use the new one.
|
// use the new one.
|
||||||
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
|
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
|
||||||
|
@ -167,7 +168,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Atomic2(AtomicSDNode *N) {
|
||||||
SDValue Op3 = GetPromotedInteger(N->getOperand(3));
|
SDValue Op3 = GetPromotedInteger(N->getOperand(3));
|
||||||
SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
|
SDValue Res = DAG.getAtomic(N->getOpcode(), N->getDebugLoc(),
|
||||||
N->getMemoryVT(), N->getChain(), N->getBasePtr(),
|
N->getMemoryVT(), N->getChain(), N->getBasePtr(),
|
||||||
Op2, Op3, N->getMemOperand());
|
Op2, Op3, N->getMemOperand(), N->getOrdering(),
|
||||||
|
N->getSynchScope());
|
||||||
// Legalized the chain result - switch anything that used the old chain to
|
// Legalized the chain result - switch anything that used the old chain to
|
||||||
// use the new one.
|
// use the new one.
|
||||||
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
|
ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
|
||||||
|
|
|
@ -3815,7 +3815,9 @@ SDValue SelectionDAG::getMemset(SDValue Chain, DebugLoc dl, SDValue Dst,
|
||||||
SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
|
SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
|
||||||
SDValue Chain, SDValue Ptr, SDValue Cmp,
|
SDValue Chain, SDValue Ptr, SDValue Cmp,
|
||||||
SDValue Swp, MachinePointerInfo PtrInfo,
|
SDValue Swp, MachinePointerInfo PtrInfo,
|
||||||
unsigned Alignment) {
|
unsigned Alignment,
|
||||||
|
AtomicOrdering Ordering,
|
||||||
|
SynchronizationScope SynchScope) {
|
||||||
if (Alignment == 0) // Ensure that codegen never sees alignment 0
|
if (Alignment == 0) // Ensure that codegen never sees alignment 0
|
||||||
Alignment = getEVTAlignment(MemVT);
|
Alignment = getEVTAlignment(MemVT);
|
||||||
|
|
||||||
|
@ -3828,13 +3830,16 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
|
||||||
MachineMemOperand *MMO =
|
MachineMemOperand *MMO =
|
||||||
MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
|
MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment);
|
||||||
|
|
||||||
return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO);
|
return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Cmp, Swp, MMO,
|
||||||
|
Ordering, SynchScope);
|
||||||
}
|
}
|
||||||
|
|
||||||
SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
|
SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
|
||||||
SDValue Chain,
|
SDValue Chain,
|
||||||
SDValue Ptr, SDValue Cmp,
|
SDValue Ptr, SDValue Cmp,
|
||||||
SDValue Swp, MachineMemOperand *MMO) {
|
SDValue Swp, MachineMemOperand *MMO,
|
||||||
|
AtomicOrdering Ordering,
|
||||||
|
SynchronizationScope SynchScope) {
|
||||||
assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op");
|
assert(Opcode == ISD::ATOMIC_CMP_SWAP && "Invalid Atomic Op");
|
||||||
assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
|
assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types");
|
||||||
|
|
||||||
|
@ -3851,7 +3856,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
|
||||||
return SDValue(E, 0);
|
return SDValue(E, 0);
|
||||||
}
|
}
|
||||||
SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
|
SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
|
||||||
Ptr, Cmp, Swp, MMO);
|
Ptr, Cmp, Swp, MMO, Ordering,
|
||||||
|
SynchScope);
|
||||||
CSEMap.InsertNode(N, IP);
|
CSEMap.InsertNode(N, IP);
|
||||||
AllNodes.push_back(N);
|
AllNodes.push_back(N);
|
||||||
return SDValue(N, 0);
|
return SDValue(N, 0);
|
||||||
|
@ -3861,7 +3867,9 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
|
||||||
SDValue Chain,
|
SDValue Chain,
|
||||||
SDValue Ptr, SDValue Val,
|
SDValue Ptr, SDValue Val,
|
||||||
const Value* PtrVal,
|
const Value* PtrVal,
|
||||||
unsigned Alignment) {
|
unsigned Alignment,
|
||||||
|
AtomicOrdering Ordering,
|
||||||
|
SynchronizationScope SynchScope) {
|
||||||
if (Alignment == 0) // Ensure that codegen never sees alignment 0
|
if (Alignment == 0) // Ensure that codegen never sees alignment 0
|
||||||
Alignment = getEVTAlignment(MemVT);
|
Alignment = getEVTAlignment(MemVT);
|
||||||
|
|
||||||
|
@ -3875,13 +3883,16 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
|
||||||
MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
|
MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags,
|
||||||
MemVT.getStoreSize(), Alignment);
|
MemVT.getStoreSize(), Alignment);
|
||||||
|
|
||||||
return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO);
|
return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO,
|
||||||
|
Ordering, SynchScope);
|
||||||
}
|
}
|
||||||
|
|
||||||
SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
|
SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
|
||||||
SDValue Chain,
|
SDValue Chain,
|
||||||
SDValue Ptr, SDValue Val,
|
SDValue Ptr, SDValue Val,
|
||||||
MachineMemOperand *MMO) {
|
MachineMemOperand *MMO,
|
||||||
|
AtomicOrdering Ordering,
|
||||||
|
SynchronizationScope SynchScope) {
|
||||||
assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
|
assert((Opcode == ISD::ATOMIC_LOAD_ADD ||
|
||||||
Opcode == ISD::ATOMIC_LOAD_SUB ||
|
Opcode == ISD::ATOMIC_LOAD_SUB ||
|
||||||
Opcode == ISD::ATOMIC_LOAD_AND ||
|
Opcode == ISD::ATOMIC_LOAD_AND ||
|
||||||
|
@ -3908,7 +3919,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, DebugLoc dl, EVT MemVT,
|
||||||
return SDValue(E, 0);
|
return SDValue(E, 0);
|
||||||
}
|
}
|
||||||
SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
|
SDNode *N = new (NodeAllocator) AtomicSDNode(Opcode, dl, VTs, MemVT, Chain,
|
||||||
Ptr, Val, MMO);
|
Ptr, Val, MMO,
|
||||||
|
Ordering, SynchScope);
|
||||||
CSEMap.InsertNode(N, IP);
|
CSEMap.InsertNode(N, IP);
|
||||||
AllNodes.push_back(N);
|
AllNodes.push_back(N);
|
||||||
return SDValue(N, 0);
|
return SDValue(N, 0);
|
||||||
|
|
|
@ -3257,9 +3257,46 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) {
|
||||||
}
|
}
|
||||||
|
|
||||||
void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
|
void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) {
|
||||||
|
SDValue Root = getRoot();
|
||||||
|
SDValue L =
|
||||||
|
DAG.getAtomic(ISD::ATOMIC_CMP_SWAP, getCurDebugLoc(),
|
||||||
|
getValue(I.getCompareOperand()).getValueType().getSimpleVT(),
|
||||||
|
Root,
|
||||||
|
getValue(I.getPointerOperand()),
|
||||||
|
getValue(I.getCompareOperand()),
|
||||||
|
getValue(I.getNewValOperand()),
|
||||||
|
MachinePointerInfo(I.getPointerOperand()), 0 /* Alignment */,
|
||||||
|
I.getOrdering(), I.getSynchScope());
|
||||||
|
setValue(&I, L);
|
||||||
|
DAG.setRoot(L.getValue(1));
|
||||||
}
|
}
|
||||||
|
|
||||||
void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
|
void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) {
|
||||||
|
ISD::NodeType NT;
|
||||||
|
switch (I.getOperation()) {
|
||||||
|
default: llvm_unreachable("Unknown atomicrmw operation"); return;
|
||||||
|
case AtomicRMWInst::Xchg: NT = ISD::ATOMIC_SWAP; break;
|
||||||
|
case AtomicRMWInst::Add: NT = ISD::ATOMIC_LOAD_ADD; break;
|
||||||
|
case AtomicRMWInst::Sub: NT = ISD::ATOMIC_LOAD_SUB; break;
|
||||||
|
case AtomicRMWInst::And: NT = ISD::ATOMIC_LOAD_AND; break;
|
||||||
|
case AtomicRMWInst::Nand: NT = ISD::ATOMIC_LOAD_NAND; break;
|
||||||
|
case AtomicRMWInst::Or: NT = ISD::ATOMIC_LOAD_OR; break;
|
||||||
|
case AtomicRMWInst::Xor: NT = ISD::ATOMIC_LOAD_XOR; break;
|
||||||
|
case AtomicRMWInst::Max: NT = ISD::ATOMIC_LOAD_MAX; break;
|
||||||
|
case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break;
|
||||||
|
case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break;
|
||||||
|
case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break;
|
||||||
|
}
|
||||||
|
SDValue L =
|
||||||
|
DAG.getAtomic(NT, getCurDebugLoc(),
|
||||||
|
getValue(I.getValOperand()).getValueType().getSimpleVT(),
|
||||||
|
getRoot(),
|
||||||
|
getValue(I.getPointerOperand()),
|
||||||
|
getValue(I.getValOperand()),
|
||||||
|
I.getPointerOperand(), 0 /* Alignment */,
|
||||||
|
I.getOrdering(), I.getSynchScope());
|
||||||
|
setValue(&I, L);
|
||||||
|
DAG.setRoot(L.getValue(1));
|
||||||
}
|
}
|
||||||
|
|
||||||
void SelectionDAGBuilder::visitFence(const FenceInst &I) {
|
void SelectionDAGBuilder::visitFence(const FenceInst &I) {
|
||||||
|
@ -3410,7 +3447,8 @@ SelectionDAGBuilder::implVisitBinaryAtomic(const CallInst& I,
|
||||||
Root,
|
Root,
|
||||||
getValue(I.getArgOperand(0)),
|
getValue(I.getArgOperand(0)),
|
||||||
getValue(I.getArgOperand(1)),
|
getValue(I.getArgOperand(1)),
|
||||||
I.getArgOperand(0));
|
I.getArgOperand(0), 0 /* Alignment */,
|
||||||
|
Monotonic, CrossThread);
|
||||||
setValue(&I, L);
|
setValue(&I, L);
|
||||||
DAG.setRoot(L.getValue(1));
|
DAG.setRoot(L.getValue(1));
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -4935,7 +4973,8 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
|
||||||
getValue(I.getArgOperand(0)),
|
getValue(I.getArgOperand(0)),
|
||||||
getValue(I.getArgOperand(1)),
|
getValue(I.getArgOperand(1)),
|
||||||
getValue(I.getArgOperand(2)),
|
getValue(I.getArgOperand(2)),
|
||||||
MachinePointerInfo(I.getArgOperand(0)));
|
MachinePointerInfo(I.getArgOperand(0)), 0 /* Alignment */,
|
||||||
|
Monotonic, CrossThread);
|
||||||
setValue(&I, L);
|
setValue(&I, L);
|
||||||
DAG.setRoot(L.getValue(1));
|
DAG.setRoot(L.getValue(1));
|
||||||
return 0;
|
return 0;
|
||||||
|
|
|
@ -9562,7 +9562,9 @@ SDValue X86TargetLowering::LowerLOAD_SUB(SDValue Op, SelectionDAG &DAG) const {
|
||||||
Node->getOperand(0),
|
Node->getOperand(0),
|
||||||
Node->getOperand(1), negOp,
|
Node->getOperand(1), negOp,
|
||||||
cast<AtomicSDNode>(Node)->getSrcValue(),
|
cast<AtomicSDNode>(Node)->getSrcValue(),
|
||||||
cast<AtomicSDNode>(Node)->getAlignment());
|
cast<AtomicSDNode>(Node)->getAlignment(),
|
||||||
|
cast<AtomicSDNode>(Node)->getOrdering(),
|
||||||
|
cast<AtomicSDNode>(Node)->getSynchScope());
|
||||||
}
|
}
|
||||||
|
|
||||||
static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
|
static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
|
||||||
|
|
|
@ -317,22 +317,22 @@ bool FunctionComparator::isEquivalentOperation(const Instruction *I1,
|
||||||
if (const InvokeInst *CI = dyn_cast<InvokeInst>(I1))
|
if (const InvokeInst *CI = dyn_cast<InvokeInst>(I1))
|
||||||
return CI->getCallingConv() == cast<InvokeInst>(I2)->getCallingConv() &&
|
return CI->getCallingConv() == cast<InvokeInst>(I2)->getCallingConv() &&
|
||||||
CI->getAttributes() == cast<InvokeInst>(I2)->getAttributes();
|
CI->getAttributes() == cast<InvokeInst>(I2)->getAttributes();
|
||||||
if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(I1)) {
|
if (const InsertValueInst *IVI = dyn_cast<InsertValueInst>(I1))
|
||||||
if (IVI->getNumIndices() != cast<InsertValueInst>(I2)->getNumIndices())
|
return IVI->getIndices() == cast<InsertValueInst>(I2)->getIndices();
|
||||||
return false;
|
if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(I1))
|
||||||
for (unsigned i = 0, e = IVI->getNumIndices(); i != e; ++i)
|
return EVI->getIndices() == cast<ExtractValueInst>(I2)->getIndices();
|
||||||
if (IVI->idx_begin()[i] != cast<InsertValueInst>(I2)->idx_begin()[i])
|
if (const FenceInst *FI = dyn_cast<FenceInst>(I1))
|
||||||
return false;
|
return FI->getOrdering() == cast<FenceInst>(I2)->getOrdering() &&
|
||||||
return true;
|
FI->getSynchScope() == cast<FenceInst>(I2)->getSynchScope();
|
||||||
}
|
if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(I1))
|
||||||
if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(I1)) {
|
return CXI->isVolatile() == cast<AtomicCmpXchgInst>(I2)->isVolatile() &&
|
||||||
if (EVI->getNumIndices() != cast<ExtractValueInst>(I2)->getNumIndices())
|
CXI->getOrdering() == cast<AtomicCmpXchgInst>(I2)->getOrdering() &&
|
||||||
return false;
|
CXI->getSynchScope() == cast<AtomicCmpXchgInst>(I2)->getSynchScope();
|
||||||
for (unsigned i = 0, e = EVI->getNumIndices(); i != e; ++i)
|
if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(I1))
|
||||||
if (EVI->idx_begin()[i] != cast<ExtractValueInst>(I2)->idx_begin()[i])
|
return RMWI->getOperation() == cast<AtomicRMWInst>(I2)->getOperation() &&
|
||||||
return false;
|
RMWI->isVolatile() == cast<AtomicRMWInst>(I2)->isVolatile() &&
|
||||||
return true;
|
RMWI->getOrdering() == cast<AtomicRMWInst>(I2)->getOrdering() &&
|
||||||
}
|
RMWI->getSynchScope() == cast<AtomicRMWInst>(I2)->getSynchScope();
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
|
@ -115,6 +115,76 @@ static bool LowerAtomicIntrinsic(IntrinsicInst *II) {
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool LowerAtomicCmpXchgInst(AtomicCmpXchgInst *CXI) {
|
||||||
|
IRBuilder<> Builder(CXI->getParent(), CXI);
|
||||||
|
Value *Ptr = CXI->getPointerOperand();
|
||||||
|
Value *Cmp = CXI->getCompareOperand();
|
||||||
|
Value *Val = CXI->getNewValOperand();
|
||||||
|
|
||||||
|
LoadInst *Orig = Builder.CreateLoad(Ptr);
|
||||||
|
Value *Equal = Builder.CreateICmpEQ(Orig, Cmp);
|
||||||
|
Value *Res = Builder.CreateSelect(Equal, Val, Orig);
|
||||||
|
Builder.CreateStore(Res, Ptr);
|
||||||
|
|
||||||
|
CXI->replaceAllUsesWith(Orig);
|
||||||
|
CXI->eraseFromParent();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool LowerAtomicRMWInst(AtomicRMWInst *RMWI) {
|
||||||
|
IRBuilder<> Builder(RMWI->getParent(), RMWI);
|
||||||
|
Value *Ptr = RMWI->getPointerOperand();
|
||||||
|
Value *Val = RMWI->getValOperand();
|
||||||
|
|
||||||
|
LoadInst *Orig = Builder.CreateLoad(Ptr);
|
||||||
|
Value *Res = NULL;
|
||||||
|
|
||||||
|
switch (RMWI->getOperation()) {
|
||||||
|
default: llvm_unreachable("Unexpected RMW operation");
|
||||||
|
case AtomicRMWInst::Xchg:
|
||||||
|
Res = Val;
|
||||||
|
break;
|
||||||
|
case AtomicRMWInst::Add:
|
||||||
|
Res = Builder.CreateAdd(Orig, Val);
|
||||||
|
break;
|
||||||
|
case AtomicRMWInst::Sub:
|
||||||
|
Res = Builder.CreateSub(Orig, Val);
|
||||||
|
break;
|
||||||
|
case AtomicRMWInst::And:
|
||||||
|
Res = Builder.CreateAnd(Orig, Val);
|
||||||
|
break;
|
||||||
|
case AtomicRMWInst::Nand:
|
||||||
|
Res = Builder.CreateNot(Builder.CreateAnd(Orig, Val));
|
||||||
|
break;
|
||||||
|
case AtomicRMWInst::Or:
|
||||||
|
Res = Builder.CreateOr(Orig, Val);
|
||||||
|
break;
|
||||||
|
case AtomicRMWInst::Xor:
|
||||||
|
Res = Builder.CreateXor(Orig, Val);
|
||||||
|
break;
|
||||||
|
case AtomicRMWInst::Max:
|
||||||
|
Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
|
||||||
|
Val, Orig);
|
||||||
|
break;
|
||||||
|
case AtomicRMWInst::Min:
|
||||||
|
Res = Builder.CreateSelect(Builder.CreateICmpSLT(Orig, Val),
|
||||||
|
Orig, Val);
|
||||||
|
break;
|
||||||
|
case AtomicRMWInst::UMax:
|
||||||
|
Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
|
||||||
|
Val, Orig);
|
||||||
|
break;
|
||||||
|
case AtomicRMWInst::UMin:
|
||||||
|
Res = Builder.CreateSelect(Builder.CreateICmpULT(Orig, Val),
|
||||||
|
Orig, Val);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Builder.CreateStore(Res, Ptr);
|
||||||
|
RMWI->replaceAllUsesWith(Orig);
|
||||||
|
RMWI->eraseFromParent();
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
static bool LowerFenceInst(FenceInst *FI) {
|
static bool LowerFenceInst(FenceInst *FI) {
|
||||||
FI->eraseFromParent();
|
FI->eraseFromParent();
|
||||||
return true;
|
return true;
|
||||||
|
@ -134,6 +204,10 @@ namespace {
|
||||||
Changed |= LowerAtomicIntrinsic(II);
|
Changed |= LowerAtomicIntrinsic(II);
|
||||||
else if (FenceInst *FI = dyn_cast<FenceInst>(Inst))
|
else if (FenceInst *FI = dyn_cast<FenceInst>(Inst))
|
||||||
Changed |= LowerFenceInst(FI);
|
Changed |= LowerFenceInst(FI);
|
||||||
|
else if (AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(Inst))
|
||||||
|
Changed |= LowerAtomicCmpXchgInst(CXI);
|
||||||
|
else if (AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(Inst))
|
||||||
|
Changed |= LowerAtomicRMWInst(RMWI);
|
||||||
}
|
}
|
||||||
return Changed;
|
return Changed;
|
||||||
}
|
}
|
||||||
|
|
|
@ -216,6 +216,15 @@ bool Instruction::isIdenticalToWhenDefined(const Instruction *I) const {
|
||||||
if (const FenceInst *FI = dyn_cast<FenceInst>(this))
|
if (const FenceInst *FI = dyn_cast<FenceInst>(this))
|
||||||
return FI->getOrdering() == cast<FenceInst>(FI)->getOrdering() &&
|
return FI->getOrdering() == cast<FenceInst>(FI)->getOrdering() &&
|
||||||
FI->getSynchScope() == cast<FenceInst>(FI)->getSynchScope();
|
FI->getSynchScope() == cast<FenceInst>(FI)->getSynchScope();
|
||||||
|
if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(this))
|
||||||
|
return CXI->isVolatile() == cast<AtomicCmpXchgInst>(I)->isVolatile() &&
|
||||||
|
CXI->getOrdering() == cast<AtomicCmpXchgInst>(I)->getOrdering() &&
|
||||||
|
CXI->getSynchScope() == cast<AtomicCmpXchgInst>(I)->getSynchScope();
|
||||||
|
if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(this))
|
||||||
|
return RMWI->getOperation() == cast<AtomicRMWInst>(I)->getOperation() &&
|
||||||
|
RMWI->isVolatile() == cast<AtomicRMWInst>(I)->isVolatile() &&
|
||||||
|
RMWI->getOrdering() == cast<AtomicRMWInst>(I)->getOrdering() &&
|
||||||
|
RMWI->getSynchScope() == cast<AtomicRMWInst>(I)->getSynchScope();
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -257,8 +266,17 @@ bool Instruction::isSameOperationAs(const Instruction *I) const {
|
||||||
if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(this))
|
if (const ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(this))
|
||||||
return EVI->getIndices() == cast<ExtractValueInst>(I)->getIndices();
|
return EVI->getIndices() == cast<ExtractValueInst>(I)->getIndices();
|
||||||
if (const FenceInst *FI = dyn_cast<FenceInst>(this))
|
if (const FenceInst *FI = dyn_cast<FenceInst>(this))
|
||||||
return FI->getOrdering() == cast<FenceInst>(FI)->getOrdering() &&
|
return FI->getOrdering() == cast<FenceInst>(I)->getOrdering() &&
|
||||||
FI->getSynchScope() == cast<FenceInst>(FI)->getSynchScope();
|
FI->getSynchScope() == cast<FenceInst>(I)->getSynchScope();
|
||||||
|
if (const AtomicCmpXchgInst *CXI = dyn_cast<AtomicCmpXchgInst>(this))
|
||||||
|
return CXI->isVolatile() == cast<AtomicCmpXchgInst>(I)->isVolatile() &&
|
||||||
|
CXI->getOrdering() == cast<AtomicCmpXchgInst>(I)->getOrdering() &&
|
||||||
|
CXI->getSynchScope() == cast<AtomicCmpXchgInst>(I)->getSynchScope();
|
||||||
|
if (const AtomicRMWInst *RMWI = dyn_cast<AtomicRMWInst>(this))
|
||||||
|
return RMWI->getOperation() == cast<AtomicRMWInst>(I)->getOperation() &&
|
||||||
|
RMWI->isVolatile() == cast<AtomicRMWInst>(I)->isVolatile() &&
|
||||||
|
RMWI->getOrdering() == cast<AtomicRMWInst>(I)->getOrdering() &&
|
||||||
|
RMWI->getSynchScope() == cast<AtomicRMWInst>(I)->getSynchScope();
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -292,6 +310,8 @@ bool Instruction::mayReadFromMemory() const {
|
||||||
case Instruction::VAArg:
|
case Instruction::VAArg:
|
||||||
case Instruction::Load:
|
case Instruction::Load:
|
||||||
case Instruction::Fence: // FIXME: refine definition of mayReadFromMemory
|
case Instruction::Fence: // FIXME: refine definition of mayReadFromMemory
|
||||||
|
case Instruction::AtomicCmpXchg:
|
||||||
|
case Instruction::AtomicRMW:
|
||||||
return true;
|
return true;
|
||||||
case Instruction::Call:
|
case Instruction::Call:
|
||||||
return !cast<CallInst>(this)->doesNotAccessMemory();
|
return !cast<CallInst>(this)->doesNotAccessMemory();
|
||||||
|
@ -310,6 +330,8 @@ bool Instruction::mayWriteToMemory() const {
|
||||||
case Instruction::Fence: // FIXME: refine definition of mayWriteToMemory
|
case Instruction::Fence: // FIXME: refine definition of mayWriteToMemory
|
||||||
case Instruction::Store:
|
case Instruction::Store:
|
||||||
case Instruction::VAArg:
|
case Instruction::VAArg:
|
||||||
|
case Instruction::AtomicCmpXchg:
|
||||||
|
case Instruction::AtomicRMW:
|
||||||
return true;
|
return true;
|
||||||
case Instruction::Call:
|
case Instruction::Call:
|
||||||
return !cast<CallInst>(this)->onlyReadsMemory();
|
return !cast<CallInst>(this)->onlyReadsMemory();
|
||||||
|
|
Loading…
Reference in New Issue