diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 982ff4ccb53a..d6a40a68a2b2 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -436,18 +436,33 @@ public: SDValue getVAArg(MVT VT, SDValue Chain, SDValue Ptr, SDValue SV); - /// getAtomic - Gets a node for an atomic op, produces result and chain, takes - /// 3 operands + /// getAtomic - Gets a node for an atomic op, produces result and chain and + /// takes 3 operands SDValue getAtomic(unsigned Opcode, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, const Value* PtrVal, unsigned Alignment=0); - /// getAtomic - Gets a node for an atomic op, produces result and chain, takes - /// 2 operands + /// getAtomic - Gets a node for an atomic op, produces result and chain and + /// takes 2 operands. SDValue getAtomic(unsigned Opcode, SDValue Chain, SDValue Ptr, SDValue Val, const Value* PtrVal, unsigned Alignment = 0); + /// getMemIntrinsicNode - Creates a MemIntrinsicNode that may produce a + /// result and takes a list of operands. + SDValue getMemIntrinsicNode(unsigned Opcode, + const MVT *VTs, unsigned NumVTs, + const SDValue *Ops, unsigned NumOps, + MVT MemVT, const Value *srcValue, int SVOff, + unsigned Align = 0, bool Vol = false, + bool ReadMem = true, bool WriteMem = true); + + SDValue getMemIntrinsicNode(unsigned Opcode, SDVTList VTList, + const SDValue *Ops, unsigned NumOps, + MVT MemVT, const Value *srcValue, int SVOff, + unsigned Align = 0, bool Vol = false, + bool ReadMem = true, bool WriteMem = true); + /// getMergeValues - Create a MERGE_VALUES node from the given operands. /// Allowed to return something different (and simpler) if Simplify is true. SDValue getMergeValues(const SDValue *Ops, unsigned NumOps, diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 36bac2cb7a36..94c8dadd966d 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1524,6 +1524,10 @@ public: const Value *srcValue, int SVOff, unsigned alignment, bool isvolatile); + MemSDNode(unsigned Opc, SDVTList VTs, const SDValue *Ops, unsigned NumOps, + MVT MemoryVT, const Value *srcValue, int SVOff, + unsigned alignment, bool isvolatile); + /// Returns alignment and volatility of the memory access unsigned getAlignment() const { return (1u << (Flags >> 1)) >> 1; } bool isVolatile() const { return Flags & 1; } @@ -1551,6 +1555,8 @@ public: // Methods to support isa and dyn_cast static bool classof(const MemSDNode *) { return true; } static bool classof(const SDNode *N) { + // For some targets, we lower some target intrinsics to a MemIntrinsicNode + // with either an intrinsic or a target opcode. return N->getOpcode() == ISD::LOAD || N->getOpcode() == ISD::STORE || N->getOpcode() == ISD::ATOMIC_CMP_SWAP_8 || @@ -1603,11 +1609,16 @@ public: N->getOpcode() == ISD::ATOMIC_LOAD_MIN_64 || N->getOpcode() == ISD::ATOMIC_LOAD_MAX_64 || N->getOpcode() == ISD::ATOMIC_LOAD_UMIN_64 || - N->getOpcode() == ISD::ATOMIC_LOAD_UMAX_64; + N->getOpcode() == ISD::ATOMIC_LOAD_UMAX_64 || + + N->getOpcode() == ISD::INTRINSIC_W_CHAIN || + N->getOpcode() == ISD::INTRINSIC_VOID || + N->isTargetOpcode(); } }; -/// Atomic operations node +/// AtomicSDNode - A SDNode reprenting atomic operations. +/// class AtomicSDNode : public MemSDNode { virtual void ANCHOR(); // Out-of-line virtual method to give class a home. SDUse Ops[4]; @@ -1707,6 +1718,36 @@ class AtomicSDNode : public MemSDNode { } }; +/// MemIntrinsicSDNode - This SDNode is used for target intrinsic that touches +/// memory and need an associated memory operand. +/// +class MemIntrinsicSDNode : public MemSDNode { + virtual void ANCHOR(); // Out-of-line virtual method to give class a home. + bool ReadMem; // Intrinsic reads memory + bool WriteMem; // Intrinsic writes memory + public: + MemIntrinsicSDNode(unsigned Opc, SDVTList VTs, + const SDValue *Ops, unsigned NumOps, + MVT MemoryVT, const Value *srcValue, int SVO, + unsigned Align, bool Vol, bool ReadMem, bool WriteMem) + : MemSDNode(Opc, VTs, Ops, NumOps, MemoryVT, srcValue, SVO, Align, Vol), + ReadMem(ReadMem), WriteMem(WriteMem) { + } + + bool readMem() const { return ReadMem; } + bool writeMem() const { return WriteMem; } + + // Methods to support isa and dyn_cast + static bool classof(const MemIntrinsicSDNode *) { return true; } + static bool classof(const SDNode *N) { + // We lower some target intrinsics to their target opcode + // early a node with a target opcode can be of this class + return N->getOpcode() == ISD::INTRINSIC_W_CHAIN || + N->getOpcode() == ISD::INTRINSIC_VOID || + N->isTargetOpcode(); + } +}; + class ConstantSDNode : public SDNode { const ConstantInt *Value; virtual void ANCHOR(); // Out-of-line virtual method to give class a home. diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index f75d39293e23..9cfb1a1a22d8 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -3334,6 +3334,46 @@ SDValue SelectionDAG::getMergeValues(const SDValue *Ops, unsigned NumOps, return getNode(ISD::MERGE_VALUES, getVTList(&VTs[0], NumOps), Ops, NumOps); } +SDValue +SelectionDAG::getMemIntrinsicNode(unsigned Opcode, + const MVT *VTs, unsigned NumVTs, + const SDValue *Ops, unsigned NumOps, + MVT MemVT, const Value *srcValue, int SVOff, + unsigned Align, bool Vol, + bool ReadMem, bool WriteMem) { + return getMemIntrinsicNode(Opcode, makeVTList(VTs, NumVTs), Ops, NumOps, + MemVT, srcValue, SVOff, Align, Vol, + ReadMem, WriteMem); +} + +SDValue +SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDVTList VTList, + const SDValue *Ops, unsigned NumOps, + MVT MemVT, const Value *srcValue, int SVOff, + unsigned Align, bool Vol, + bool ReadMem, bool WriteMem) { + // Memoize the node unless it returns a flag. + MemIntrinsicSDNode *N; + if (VTList.VTs[VTList.NumVTs-1] != MVT::Flag) { + FoldingSetNodeID ID; + AddNodeIDNode(ID, Opcode, VTList, Ops, NumOps); + void *IP = 0; + if (SDNode *E = CSEMap.FindNodeOrInsertPos(ID, IP)) + return SDValue(E, 0); + + N = NodeAllocator.Allocate(); + new (N) MemIntrinsicSDNode(Opcode, VTList, Ops, NumOps, MemVT, + srcValue, SVOff, Align, Vol, ReadMem, WriteMem); + CSEMap.InsertNode(N, IP); + } else { + N = NodeAllocator.Allocate(); + new (N) MemIntrinsicSDNode(Opcode, VTList, Ops, NumOps, MemVT, + srcValue, SVOff, Align, Vol, ReadMem, WriteMem); + } + AllNodes.push_back(N); + return SDValue(N, 0); +} + SDValue SelectionDAG::getCall(unsigned CallingConv, bool IsVarArgs, bool IsTailCall, bool IsInreg, SDVTList VTs, @@ -4678,6 +4718,7 @@ void MemSDNode::ANCHOR() {} void LoadSDNode::ANCHOR() {} void StoreSDNode::ANCHOR() {} void AtomicSDNode::ANCHOR() {} +void MemIntrinsicSDNode::ANCHOR() {} void CallSDNode::ANCHOR() {} HandleSDNode::~HandleSDNode() { @@ -4707,6 +4748,17 @@ MemSDNode::MemSDNode(unsigned Opc, SDVTList VTs, MVT memvt, assert(isVolatile() == vol && "Volatile representation error!"); } +MemSDNode::MemSDNode(unsigned Opc, SDVTList VTs, const SDValue *Ops, + unsigned NumOps, MVT memvt, const Value *srcValue, + int SVO, unsigned alignment, bool vol) + : SDNode(Opc, VTs, Ops, NumOps), + MemoryVT(memvt), SrcValue(srcValue), SVOffset(SVO), + Flags(vol | ((Log2_32(alignment) + 1) << 1)) { + assert(isPowerOf2_32(alignment) && "Alignment is not a power of 2!"); + assert(getAlignment() == alignment && "Alignment representation error!"); + assert(isVolatile() == vol && "Volatile representation error!"); +} + /// getMemOperand - Return a MachineMemOperand object describing the memory /// reference performed by this memory reference. MachineMemOperand MemSDNode::getMemOperand() const { @@ -4715,10 +4767,15 @@ MachineMemOperand MemSDNode::getMemOperand() const { Flags = MachineMemOperand::MOLoad; else if (isa(this)) Flags = MachineMemOperand::MOStore; - else { - assert(isa(this) && "Unknown MemSDNode opcode!"); + else if (isa(this)) { Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; } + else { + const MemIntrinsicSDNode* MemIntrinNode = dyn_cast(this); + assert(MemIntrinNode && "Unknown MemSDNode opcode!"); + if (MemIntrinNode->readMem()) Flags |= MachineMemOperand::MOLoad; + if (MemIntrinNode->writeMem()) Flags |= MachineMemOperand::MOStore; + } int Size = (getMemoryVT().getSizeInBits() + 7) >> 3; if (isVolatile()) Flags |= MachineMemOperand::MOVolatile;