diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 9a64444d8e7d..941de90ff4b7 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -23,6 +23,7 @@ #include "llvm/CodeGen/DAGCombine.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/SelectionDAGNodes.h" +#include "llvm/Support/ArrayRecycler.h" #include "llvm/Support/RecyclingAllocator.h" #include "llvm/Target/TargetMachine.h" #include @@ -208,6 +209,7 @@ class SelectionDAG { /// Pool allocation for machine-opcode SDNode operands. BumpPtrAllocator OperandAllocator; + ArrayRecycler OperandRecycler; /// Pool allocation for misc. objects that are created once per SelectionDAG. BumpPtrAllocator Allocator; @@ -274,6 +276,30 @@ private: SDNodeT(std::forward(Args)...); } + void createOperands(SDNode *Node, ArrayRef Vals) { + assert(!Node->OperandList && "Node already has operands"); + SDUse *Ops = OperandRecycler.allocate( + ArrayRecycler::Capacity::get(Vals.size()), OperandAllocator); + + for (unsigned I = 0; I != Vals.size(); ++I) { + Ops[I].setUser(Node); + Ops[I].setInitial(Vals[I]); + } + Node->NumOperands = Vals.size(); + Node->OperandList = Ops; + checkForCycles(Node); + } + + void removeOperands(SDNode *Node) { + if (!Node->OperandList) + return; + OperandRecycler.deallocate( + ArrayRecycler::Capacity::get(Node->NumOperands), + Node->OperandList); + Node->NumOperands = 0; + Node->OperandList = nullptr; + } + void operator=(const SelectionDAG&) = delete; SelectionDAG(const SelectionDAG&) = delete; @@ -1316,9 +1342,8 @@ private: void allnodes_clear(); - BinarySDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs, - SDValue N1, SDValue N2, - const SDNodeFlags *Flags = nullptr); + SDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1, + SDValue N2, const SDNodeFlags *Flags = nullptr); /// Look up the node specified by ID in CSEMap. If it exists, return it. If /// not, return the insertion token that will make insertion faster. This diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index 375e238261db..a50742e17b5d 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -44,7 +44,7 @@ class GlobalValue; class MachineBasicBlock; class MachineConstantPoolValue; class SDNode; -class BinaryWithFlagsSDNode; +class HandleSDNode; class Value; class MCSymbol; template struct DenseMapInfo; @@ -280,6 +280,8 @@ public: private: friend class SelectionDAG; friend class SDNode; + // TODO: unfriend HandleSDNode once we fix its operand handling. + friend class HandleSDNode; void setUser(SDNode *p) { User = p; } @@ -394,10 +396,6 @@ private: /// The operation that this node performs. int16_t NodeType; - /// This is true if OperandList was new[]'d. If true, - /// then they will be delete[]'d when the node is destroyed. - uint16_t OperandsNeedDelete : 1; - /// This tracks whether this node has one or more dbg_value /// nodes corresponding to it. uint16_t HasDebugValue : 1; @@ -406,7 +404,7 @@ protected: /// This member is defined by this class, but is not used for /// anything. Subclasses can use it to hold whatever state they find useful. /// This field is initialized to zero by the ctor. - uint16_t SubclassData : 14; + uint16_t SubclassData : 15; private: /// Unique id per SDNode in the DAG. @@ -440,6 +438,8 @@ private: friend class SelectionDAG; friend struct ilist_traits; + // TODO: unfriend HandleSDNode once we fix its operand handling. + friend class HandleSDNode; public: /// Unique and persistent id per SDNode in the DAG. @@ -792,101 +792,20 @@ protected: return Ret; } - SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, - ArrayRef Ops) - : NodeType(Opc), OperandsNeedDelete(true), HasDebugValue(false), - SubclassData(0), NodeId(-1), - OperandList(Ops.size() ? new SDUse[Ops.size()] : nullptr), - ValueList(VTs.VTs), UseList(nullptr), NumOperands(Ops.size()), - NumValues(VTs.NumVTs), IROrder(Order), debugLoc(std::move(dl)) { - assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); - assert(NumOperands == Ops.size() && - "NumOperands wasn't wide enough for its operands!"); - assert(NumValues == VTs.NumVTs && - "NumValues wasn't wide enough for its operands!"); - for (unsigned i = 0; i != Ops.size(); ++i) { - assert(OperandList && "no operands available"); - OperandList[i].setUser(this); - OperandList[i].setInitial(Ops[i]); - } - checkForCycles(this); - } - - /// This constructor adds no operands itself; operands can be - /// set later with InitOperands. + /// Create an SDNode. + /// + /// SDNodes are created without any operands, and never own the operand + /// storage. To add operands, see SelectionDAG::createOperands. SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) - : NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false), - SubclassData(0), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs), - UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs), - IROrder(Order), debugLoc(std::move(dl)) { + : NodeType(Opc), HasDebugValue(false), SubclassData(0), NodeId(-1), + OperandList(nullptr), ValueList(VTs.VTs), UseList(nullptr), + NumOperands(0), NumValues(VTs.NumVTs), IROrder(Order), + debugLoc(std::move(dl)) { assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); assert(NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"); } - /// Initialize the operands list of this with 1 operand. - void InitOperands(SDUse *Ops, const SDValue &Op0) { - Ops[0].setUser(this); - Ops[0].setInitial(Op0); - NumOperands = 1; - OperandList = Ops; - checkForCycles(this); - } - - /// Initialize the operands list of this with 2 operands. - void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1) { - Ops[0].setUser(this); - Ops[0].setInitial(Op0); - Ops[1].setUser(this); - Ops[1].setInitial(Op1); - NumOperands = 2; - OperandList = Ops; - checkForCycles(this); - } - - /// Initialize the operands list of this with 3 operands. - void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1, - const SDValue &Op2) { - Ops[0].setUser(this); - Ops[0].setInitial(Op0); - Ops[1].setUser(this); - Ops[1].setInitial(Op1); - Ops[2].setUser(this); - Ops[2].setInitial(Op2); - NumOperands = 3; - OperandList = Ops; - checkForCycles(this); - } - - /// Initialize the operands list of this with 4 operands. - void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1, - const SDValue &Op2, const SDValue &Op3) { - Ops[0].setUser(this); - Ops[0].setInitial(Op0); - Ops[1].setUser(this); - Ops[1].setInitial(Op1); - Ops[2].setUser(this); - Ops[2].setInitial(Op2); - Ops[3].setUser(this); - Ops[3].setInitial(Op3); - NumOperands = 4; - OperandList = Ops; - checkForCycles(this); - } - - /// Initialize the operands list of this with N operands. - void InitOperands(SDUse *Ops, const SDValue *Vals, unsigned N) { - for (unsigned i = 0; i != N; ++i) { - Ops[i].setUser(this); - Ops[i].setInitial(Vals[i]); - } - NumOperands = N; - assert(NumOperands == N && - "NumOperands wasn't wide enough for its operands!"); - OperandList = Ops; - checkForCycles(this); - } - /// Release the operands and set this node to have zero operands. void DropOperands(); }; @@ -1012,30 +931,6 @@ inline void SDUse::setNode(SDNode *N) { if (N) N->addUse(*this); } -/// This class is used for single-operand SDNodes. This is solely -/// to allow co-allocation of node operands with the node itself. -class UnarySDNode : public SDNode { - SDUse Op; -public: - UnarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, - SDValue X) - : SDNode(Opc, Order, dl, VTs) { - InitOperands(&Op, X); - } -}; - -/// This class is used for two-operand SDNodes. This is solely -/// to allow co-allocation of node operands with the node itself. -class BinarySDNode : public SDNode { - SDUse Ops[2]; -public: - BinarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, - SDValue X, SDValue Y) - : SDNode(Opc, Order, dl, VTs) { - InitOperands(Ops, X, Y); - } -}; - /// Returns true if the opcode is a binary operation with flags. static bool isBinOpWithFlags(unsigned Opcode) { switch (Opcode) { @@ -1060,30 +955,17 @@ static bool isBinOpWithFlags(unsigned Opcode) { /// This class is an extension of BinarySDNode /// used from those opcodes that have associated extra flags. -class BinaryWithFlagsSDNode : public BinarySDNode { +class BinaryWithFlagsSDNode : public SDNode { public: SDNodeFlags Flags; BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, - SDValue X, SDValue Y, const SDNodeFlags &NodeFlags) - : BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags(NodeFlags) {} + const SDNodeFlags &NodeFlags) + : SDNode(Opc, Order, dl, VTs), Flags(NodeFlags) {} static bool classof(const SDNode *N) { return isBinOpWithFlags(N->getOpcode()); } }; -/// This class is used for three-operand SDNodes. This is solely -/// to allow co-allocation of node operands with the node itself. -class TernarySDNode : public SDNode { - SDUse Ops[3]; -public: - TernarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, - SDValue X, SDValue Y, SDValue Z) - : SDNode(Opc, Order, dl, VTs) { - InitOperands(Ops, X, Y, Z); - } -}; - - /// This class is used to form a handle around another node that /// is persistent and is updated across invocations of replaceAllUsesWith on its /// operand. This node should be directly created by end-users and not added to @@ -1096,20 +978,28 @@ public: // HandleSDNodes are never inserted into the DAG, so they won't be // auto-numbered. Use ID 65535 as a sentinel. PersistentId = 0xffff; - InitOperands(&Op, X); + + // Manually set up the operand list. This node type is special in that it's + // always stack allocated and SelectionDAG does not manage its operands. + // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not + // be so special. + Op.setUser(this); + Op.setInitial(X); + NumOperands = 1; + OperandList = &Op; } ~HandleSDNode(); const SDValue &getValue() const { return Op; } }; -class AddrSpaceCastSDNode : public UnarySDNode { +class AddrSpaceCastSDNode : public SDNode { private: unsigned SrcAddrSpace; unsigned DestAddrSpace; public: - AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, SDValue X, - unsigned SrcAS, unsigned DestAS); + AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, unsigned SrcAS, + unsigned DestAS); unsigned getSrcAddressSpace() const { return SrcAddrSpace; } unsigned getDestAddressSpace() const { return DestAddrSpace; } @@ -1133,9 +1023,6 @@ public: MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO); - MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, - ArrayRef Ops, EVT MemoryVT, MachineMemOperand *MMO); - bool readMem() const { return MMO->isLoad(); } bool writeMem() const { return MMO->isStore(); } @@ -1238,8 +1125,6 @@ public: /// This is an SDNode representing atomic operations. class AtomicSDNode : public MemSDNode { - SDUse Ops[4]; - /// For cmpxchg instructions, the ordering requirements when a store does not /// occur. AtomicOrdering FailureOrdering; @@ -1265,16 +1150,12 @@ class AtomicSDNode : public MemSDNode { } public: - AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, EVT MemVT, - const SDValue* AllOps, SDUse *DynOps, unsigned NumOps, - MachineMemOperand *MMO, + AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, + EVT MemVT, MachineMemOperand *MMO, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) - : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { + : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { InitAtomic(SuccessOrdering, FailureOrdering, SynchScope); - assert((DynOps || NumOps <= array_lengthof(Ops)) && - "Too many ops for internal storage!"); - InitOperands(DynOps ? DynOps : Ops, AllOps, NumOps); } const SDValue &getBasePtr() const { return getOperand(1); } @@ -1322,9 +1203,8 @@ public: class MemIntrinsicSDNode : public MemSDNode { public: MemIntrinsicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, - ArrayRef Ops, EVT MemoryVT, - MachineMemOperand *MMO) - : MemSDNode(Opc, Order, dl, VTs, Ops, MemoryVT, MMO) { + EVT MemoryVT, MachineMemOperand *MMO) + : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) { SubclassData |= 1u << 13; } @@ -1347,20 +1227,15 @@ public: /// An index of -1 is treated as undef, such that the code generator may put /// any value in the corresponding element of the result. class ShuffleVectorSDNode : public SDNode { - SDUse Ops[2]; - // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and // is freed when the SelectionDAG object is destroyed. const int *Mask; protected: friend class SelectionDAG; - ShuffleVectorSDNode(EVT VT, unsigned Order, DebugLoc dl, SDValue N1, - SDValue N2, const int *M) - : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) { - InitOperands(Ops, N1, N2); - } -public: + ShuffleVectorSDNode(EVT VT, unsigned Order, DebugLoc dl, const int *M) + : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {} +public: ArrayRef getMask() const { EVT VT = getValueType(0); return makeArrayRef(Mask, VT.getVectorNumElements()); @@ -1791,13 +1666,11 @@ public: }; class EHLabelSDNode : public SDNode { - SDUse Chain; MCSymbol *Label; friend class SelectionDAG; - EHLabelSDNode(unsigned Order, DebugLoc dl, SDValue ch, MCSymbol *L) - : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) { - InitOperands(&Chain, ch); - } + EHLabelSDNode(unsigned Order, DebugLoc dl, MCSymbol *L) + : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) {} + public: MCSymbol *getLabel() const { return Label; } @@ -1863,11 +1736,10 @@ class CvtRndSatSDNode : public SDNode { ISD::CvtCode CvtCode; friend class SelectionDAG; explicit CvtRndSatSDNode(EVT VT, unsigned Order, DebugLoc dl, - ArrayRef Ops, ISD::CvtCode Code) - : SDNode(ISD::CONVERT_RNDSAT, Order, dl, getSDVTList(VT), Ops), - CvtCode(Code) { - assert(Ops.size() == 5 && "wrong number of operations"); + ISD::CvtCode Code) + : SDNode(ISD::CONVERT_RNDSAT, Order, dl, getSDVTList(VT)), CvtCode(Code) { } + public: ISD::CvtCode getCvtCode() const { return CvtCode; } @@ -1896,24 +1768,13 @@ public: /// Base class for LoadSDNode and StoreSDNode class LSBaseSDNode : public MemSDNode { - //! Operand array for load and store - /*! - \note Moving this array to the base class captures more - common functionality shared between LoadSDNode and - StoreSDNode - */ - SDUse Ops[4]; public: LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, - SDValue *Operands, unsigned numOperands, SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, MachineMemOperand *MMO) : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { SubclassData |= AM << 2; assert(getAddressingMode() == AM && "MemIndexedMode encoding error!"); - InitOperands(Ops, Operands, numOperands); - assert((getOffset().getOpcode() == ISD::UNDEF || isIndexed()) && - "Only indexed loads and stores have a non-undef offset operand"); } const SDValue &getOffset() const { @@ -1941,10 +1802,9 @@ public: /// This class is used to represent ISD::LOAD nodes. class LoadSDNode : public LSBaseSDNode { friend class SelectionDAG; - LoadSDNode(SDValue *ChainPtrOff, unsigned Order, DebugLoc dl, SDVTList VTs, - ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT, - MachineMemOperand *MMO) - : LSBaseSDNode(ISD::LOAD, Order, dl, ChainPtrOff, 3, VTs, AM, MemVT, MMO) { + LoadSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM, + ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO) + : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) { SubclassData |= (unsigned short)ETy; assert(getExtensionType() == ETy && "LoadExtType encoding error!"); assert(readMem() && "Load MachineMemOperand is not a load!"); @@ -1969,11 +1829,9 @@ public: /// This class is used to represent ISD::STORE nodes. class StoreSDNode : public LSBaseSDNode { friend class SelectionDAG; - StoreSDNode(SDValue *ChainValuePtrOff, unsigned Order, DebugLoc dl, - SDVTList VTs, ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, - MachineMemOperand *MMO) - : LSBaseSDNode(ISD::STORE, Order, dl, ChainValuePtrOff, 4, - VTs, AM, MemVT, MMO) { + StoreSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM, + bool isTrunc, EVT MemVT, MachineMemOperand *MMO) + : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) { SubclassData |= (unsigned short)isTrunc; assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!"); assert(!readMem() && "Store MachineMemOperand is a load!"); @@ -1997,16 +1855,11 @@ public: /// This base class is used to represent MLOAD and MSTORE nodes class MaskedLoadStoreSDNode : public MemSDNode { - // Operands - SDUse Ops[4]; public: friend class SelectionDAG; MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, - SDValue *Operands, unsigned numOperands, SDVTList VTs, - EVT MemVT, MachineMemOperand *MMO) - : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { - InitOperands(Ops, Operands, numOperands); - } + SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) + : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {} // In the both nodes address is Op1, mask is Op2: // MaskedLoadSDNode (Chain, ptr, mask, src0), src0 is a passthru value @@ -2025,11 +1878,9 @@ public: class MaskedLoadSDNode : public MaskedLoadStoreSDNode { public: friend class SelectionDAG; - MaskedLoadSDNode(unsigned Order, DebugLoc dl, SDValue *Operands, - unsigned numOperands, SDVTList VTs, ISD::LoadExtType ETy, - EVT MemVT, MachineMemOperand *MMO) - : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, Operands, numOperands, - VTs, MemVT, MMO) { + MaskedLoadSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, + ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO) + : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, MemVT, MMO) { SubclassData |= (unsigned short)ETy; } @@ -2047,12 +1898,10 @@ class MaskedStoreSDNode : public MaskedLoadStoreSDNode { public: friend class SelectionDAG; - MaskedStoreSDNode(unsigned Order, DebugLoc dl, SDValue *Operands, - unsigned numOperands, SDVTList VTs, bool isTrunc, EVT MemVT, - MachineMemOperand *MMO) - : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, Operands, numOperands, - VTs, MemVT, MMO) { - SubclassData |= (unsigned short)isTrunc; + MaskedStoreSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, bool isTrunc, + EVT MemVT, MachineMemOperand *MMO) + : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) { + SubclassData |= (unsigned short)isTrunc; } /// Return true if the op does a truncation before store. /// For integers this is the same as doing a TRUNCATE and storing the result. @@ -2070,17 +1919,11 @@ public: /// MGATHER and MSCATTER nodes /// class MaskedGatherScatterSDNode : public MemSDNode { - // Operands - SDUse Ops[5]; public: friend class SelectionDAG; MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, - ArrayRef Operands, SDVTList VTs, EVT MemVT, - MachineMemOperand *MMO) - : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { - assert(Operands.size() == 5 && "Incompatible number of operands"); - InitOperands(Ops, Operands.data(), Operands.size()); - } + SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) + : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {} // In the both nodes address is Op1, mask is Op2: // MaskedGatherSDNode (Chain, src0, mask, base, index), src0 is a passthru value @@ -2102,19 +1945,9 @@ public: class MaskedGatherSDNode : public MaskedGatherScatterSDNode { public: friend class SelectionDAG; - MaskedGatherSDNode(unsigned Order, DebugLoc dl, ArrayRef Operands, - SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) - : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, Operands, VTs, MemVT, - MMO) { - assert(getValue().getValueType() == getValueType(0) && - "Incompatible type of the PassThru value in MaskedGatherSDNode"); - assert(getMask().getValueType().getVectorNumElements() == - getValueType(0).getVectorNumElements() && - "Vector width mismatch between mask and data"); - assert(getIndex().getValueType().getVectorNumElements() == - getValueType(0).getVectorNumElements() && - "Vector width mismatch between index and data"); - } + MaskedGatherSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, EVT MemVT, + MachineMemOperand *MMO) + : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO) {} static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MGATHER; @@ -2127,17 +1960,9 @@ class MaskedScatterSDNode : public MaskedGatherScatterSDNode { public: friend class SelectionDAG; - MaskedScatterSDNode(unsigned Order, DebugLoc dl,ArrayRef Operands, - SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) - : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, Operands, VTs, MemVT, - MMO) { - assert(getMask().getValueType().getVectorNumElements() == - getValue().getValueType().getVectorNumElements() && - "Vector width mismatch between mask and data"); - assert(getIndex().getValueType().getVectorNumElements() == - getValue().getValueType().getVectorNumElements() && - "Vector width mismatch between index and data"); - } + MaskedScatterSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, EVT MemVT, + MachineMemOperand *MMO) + : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO) {} static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MSCATTER; @@ -2156,10 +1981,6 @@ private: MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc DL, SDVTList VTs) : SDNode(Opc, Order, DL, VTs), MemRefs(nullptr), MemRefsEnd(nullptr) {} - /// Operands for this instruction, if they fit here. If - /// they don't, this field is unused. - SDUse LocalOperands[4]; - /// Memory reference descriptions for this instruction. mmo_iterator MemRefs; mmo_iterator MemRefsEnd; @@ -2234,8 +2055,16 @@ template <> struct GraphTraits { } }; -/// The largest SDNode class. -typedef MaskedGatherScatterSDNode LargestSDNode; +/// A representation of the largest SDNode, for use in sizeof(). +/// +/// This needs to be a union because the largest node differs on 32 bit systems +/// with 4 and 8 byte pointer alignment, respectively. +union LargestSDNode { + AtomicSDNode Atomic; + TargetIndexSDNode TargetIndex; + BlockAddressSDNode BlockAddress; + GlobalAddressSDNode GlobalAddress; +}; /// The SDNode class with the greatest alignment requirement. typedef GlobalAddressSDNode MostAlignedSDNode; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 03179b7c3d3a..592f5152558c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -664,8 +664,8 @@ void SDDbgInfo::erase(const SDNode *Node) { } void SelectionDAG::DeallocateNode(SDNode *N) { - if (N->OperandsNeedDelete) - delete[] N->OperandList; + // If we have operands, deallocate them. + removeOperands(N); // Set the opcode to DELETED_NODE to help catch bugs when node // memory is reallocated. @@ -912,6 +912,7 @@ void SelectionDAG::init(MachineFunction &mf) { SelectionDAG::~SelectionDAG() { assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); allnodes_clear(); + OperandRecycler.clear(OperandAllocator); delete DbgInfo; } @@ -925,24 +926,26 @@ void SelectionDAG::allnodes_clear() { #endif } -BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL, - SDVTList VTs, SDValue N1, - SDValue N2, - const SDNodeFlags *Flags) { +SDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs, + SDValue N1, SDValue N2, + const SDNodeFlags *Flags) { + SDValue Ops[] = {N1, N2}; + if (isBinOpWithFlags(Opcode)) { // If no flags were passed in, use a default flags object. SDNodeFlags F; if (Flags == nullptr) Flags = &F; - auto *FN = newSDNode( - Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, *Flags); + auto *FN = newSDNode(Opcode, DL.getIROrder(), + DL.getDebugLoc(), VTs, *Flags); + createOperands(FN, Ops); return FN; } - auto *N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), - VTs, N1, N2); + auto *N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); + createOperands(N, Ops); return N; } @@ -982,6 +985,7 @@ SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, void SelectionDAG::clear() { allnodes_clear(); + OperandRecycler.clear(OperandAllocator); OperandAllocator.Reset(); CSEMap.clear(); @@ -1638,7 +1642,9 @@ SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); auto *N = newSDNode(VT, dl.getIROrder(), - dl.getDebugLoc(), N1, N2, MaskAlloc); + dl.getDebugLoc(), MaskAlloc); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -1671,8 +1677,10 @@ SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl, if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) return SDValue(E, 0); - auto *N = newSDNode(VT, dl.getIROrder(), dl.getDebugLoc(), - Ops, Code); + auto *N = + newSDNode(VT, dl.getIROrder(), dl.getDebugLoc(), Code); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -1715,8 +1723,9 @@ SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) { if (SDNode *E = FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - auto *N = - newSDNode(dl.getIROrder(), dl.getDebugLoc(), Root, Label); + auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), Label); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -1799,7 +1808,9 @@ SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr, return SDValue(E, 0); auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), - VT, Ptr, SrcAS, DestAS); + VT, SrcAS, DestAS); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -3183,20 +3194,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDNode *N; SDVTList VTs = getVTList(VT); + SDValue Ops[] = {Operand}; if (VT != MVT::Glue) { // Don't CSE flag producing nodes FoldingSetNodeID ID; - SDValue Ops[1] = { Operand }; AddNodeIDNode(ID, Opcode, VTs, Ops); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) return SDValue(E, 0); - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, - Operand); + N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); + createOperands(N, Ops); CSEMap.InsertNode(N, IP); } else { - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, - Operand); + N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); + createOperands(N, Ops); } InsertNode(N); @@ -3895,7 +3906,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, } // Memoize this node if possible. - BinarySDNode *N; + SDNode *N; SDVTList VTs = getVTList(VT); if (VT != MVT::Glue) { SDValue Ops[] = {N1, N2}; @@ -3909,7 +3920,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, } N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags); - CSEMap.InsertNode(N, IP); } else { N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags); @@ -4000,20 +4010,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, // Memoize node if it doesn't produce a flag. SDNode *N; SDVTList VTs = getVTList(VT); + SDValue Ops[] = {N1, N2, N3}; if (VT != MVT::Glue) { - SDValue Ops[] = { N1, N2, N3 }; FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTs, Ops); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) return SDValue(E, 0); - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, - N1, N2, N3); + N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); + createOperands(N, Ops); CSEMap.InsertNode(N, IP); } else { - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, - N1, N2, N3); + N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); + createOperands(N, Ops); } InsertNode(N); @@ -4805,18 +4815,11 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, return SDValue(E, 0); } - // Allocate the operands array for the node out of the BumpPtrAllocator, since - // SDNode doesn't have access to it. This memory will be "leaked" when - // the node is deallocated, but recovered when the allocator is released. - // If the number of operands is less than 5 we use AtomicSDNode's internal - // storage. - unsigned NumOps = Ops.size(); - SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate(NumOps) - : nullptr; + auto *N = newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), + VTList, MemVT, MMO, SuccessOrdering, + FailureOrdering, SynchScope); + createOperands(N, Ops); - auto *N = newSDNode( - Opcode, dl.getIROrder(), dl.getDebugLoc(), VTList, MemVT, Ops.data(), - DynOps, NumOps, MMO, SuccessOrdering, FailureOrdering, SynchScope); CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5009,11 +5012,14 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, } N = newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), - VTList, Ops, MemVT, MMO); - CSEMap.InsertNode(N, IP); + VTList, MemVT, MMO); + createOperands(N, Ops); + + CSEMap.InsertNode(N, IP); } else { N = newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), - VTList, Ops, MemVT, MMO); + VTList, MemVT, MMO); + createOperands(N, Ops); } InsertNode(N); return SDValue(N, 0); @@ -5131,8 +5137,10 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, cast(E)->refineAlignment(MMO); return SDValue(E, 0); } - auto *N = newSDNode(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, - AM, ExtType, MemVT, MMO); + auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, + ExtType, MemVT, MMO); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5238,8 +5246,10 @@ SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, cast(E)->refineAlignment(MMO); return SDValue(E, 0); } - auto *N = newSDNode(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, + auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, ISD::UNINDEXED, false, VT, MMO); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5306,8 +5316,10 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, cast(E)->refineAlignment(MMO); return SDValue(E, 0); } - auto *N = newSDNode(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, + auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, ISD::UNINDEXED, true, SVT, MMO); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5330,9 +5342,11 @@ SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base, if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) return SDValue(E, 0); - auto *N = newSDNode(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, - AM, ST->isTruncatingStore(), - ST->getMemoryVT(), ST->getMemOperand()); + auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, + ST->isTruncatingStore(), ST->getMemoryVT(), + ST->getMemOperand()); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5358,8 +5372,10 @@ SelectionDAG::getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain, cast(E)->refineAlignment(MMO); return SDValue(E, 0); } - auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), Ops, - 4, VTs, ExtTy, MemVT, MMO); + auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, + ExtTy, MemVT, MMO); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5384,8 +5400,10 @@ SDValue SelectionDAG::getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val, cast(E)->refineAlignment(MMO); return SDValue(E, 0); } - auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), Ops, - 4, VTs, isTrunc, MemVT, MMO); + auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), VTs, + isTrunc, MemVT, MMO); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5395,6 +5413,7 @@ SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl, ArrayRef Ops, MachineMemOperand *MMO) { + assert(Ops.size() == 5 && "Incompatible number of operands"); FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); @@ -5409,8 +5428,20 @@ SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl, cast(E)->refineAlignment(MMO); return SDValue(E, 0); } + auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), - Ops, VTs, VT, MMO); + VTs, VT, MMO); + createOperands(N, Ops); + + assert(N->getValue().getValueType() == N->getValueType(0) && + "Incompatible type of the PassThru value in MaskedGatherSDNode"); + assert(N->getMask().getValueType().getVectorNumElements() == + N->getValueType(0).getVectorNumElements() && + "Vector width mismatch between mask and data"); + assert(N->getIndex().getValueType().getVectorNumElements() == + N->getValueType(0).getVectorNumElements() && + "Vector width mismatch between index and data"); + CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5419,6 +5450,8 @@ SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl, SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl, ArrayRef Ops, MachineMemOperand *MMO) { + assert(Ops.size() == 5 && "Incompatible number of operands"); + FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); ID.AddInteger(VT.getRawBits()); @@ -5432,7 +5465,16 @@ SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl, return SDValue(E, 0); } auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), - Ops, VTs, VT, MMO); + VTs, VT, MMO); + createOperands(N, Ops); + + assert(N->getMask().getValueType().getVectorNumElements() == + N->getValue().getValueType().getVectorNumElements() && + "Vector width mismatch between mask and data"); + assert(N->getIndex().getValueType().getVectorNumElements() == + N->getValue().getValueType().getVectorNumElements() && + "Vector width mismatch between index and data"); + CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5511,10 +5553,13 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) return SDValue(E, 0); - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops); + N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); } else { - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops); + N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); + createOperands(N, Ops); } InsertNode(N); @@ -5556,7 +5601,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, // Memoize the node unless it returns a flag. SDNode *N; - unsigned NumOps = Ops.size(); if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTList, Ops); @@ -5564,34 +5608,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) return SDValue(E, 0); - if (NumOps == 1) { - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), - VTList, Ops[0]); - } else if (NumOps == 2) { - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), - VTList, Ops[0], Ops[1]); - } else if (NumOps == 3) { - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), - VTList, Ops[0], Ops[1], Ops[2]); - } else { - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, - Ops); - } + N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); + createOperands(N, Ops); CSEMap.InsertNode(N, IP); } else { - if (NumOps == 1) { - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), - VTList, Ops[0]); - } else if (NumOps == 2) { - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), - VTList, Ops[0], Ops[1]); - } else if (NumOps == 3) { - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), - VTList, Ops[0], Ops[1], Ops[2]); - } else { - N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, - Ops); - } + N = newSDNode(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); + createOperands(N, Ops); } InsertNode(N); return SDValue(N, 0); @@ -5977,7 +5999,6 @@ SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) { /// deleting things. SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef Ops) { - unsigned NumOps = Ops.size(); // If an identical node already exists, use it. void *IP = nullptr; if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { @@ -6006,36 +6027,13 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, DeadNodeSet.insert(Used); } - if (MachineSDNode *MN = dyn_cast(N)) { - // Initialize the memory references information. + // For MachineNode, initialize the memory references information. + if (MachineSDNode *MN = dyn_cast(N)) MN->setMemRefs(nullptr, nullptr); - // If NumOps is larger than the # of operands we can have in a - // MachineSDNode, reallocate the operand list. - if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) { - if (MN->OperandsNeedDelete) - delete[] MN->OperandList; - if (NumOps > array_lengthof(MN->LocalOperands)) - // We're creating a final node that will live unmorphed for the - // remainder of the current SelectionDAG iteration, so we can allocate - // the operands directly out of a pool with no recycling metadata. - MN->InitOperands(OperandAllocator.Allocate(NumOps), - Ops.data(), NumOps); - else - MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps); - MN->OperandsNeedDelete = false; - } else - MN->InitOperands(MN->OperandList, Ops.data(), NumOps); - } else { - // If NumOps is larger than the # of operands we currently have, reallocate - // the operand list. - if (NumOps > N->NumOperands) { - if (N->OperandsNeedDelete) - delete[] N->OperandList; - N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps); - N->OperandsNeedDelete = true; - } else - N->InitOperands(N->OperandList, Ops.data(), NumOps); - } + + // Swap for an appropriately sized array from the recycler. + removeOperands(N); + createOperands(N, Ops); // Delete any nodes that are still dead after adding the uses for the // new operands. @@ -6178,16 +6176,14 @@ SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, MachineSDNode * SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs, - ArrayRef OpsArray) { + ArrayRef Ops) { bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; MachineSDNode *N; void *IP = nullptr; - const SDValue *Ops = OpsArray.data(); - unsigned NumOps = OpsArray.size(); if (DoCSE) { FoldingSetNodeID ID; - AddNodeIDNode(ID, ~Opcode, VTs, OpsArray); + AddNodeIDNode(ID, ~Opcode, VTs, Ops); IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) { return cast(UpdadeSDLocOnMergedSDNode(E, DL)); @@ -6196,17 +6192,7 @@ SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs, // Allocate a new MachineSDNode. N = newSDNode(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); - - // Initialize the operands list. - if (NumOps > array_lengthof(N->LocalOperands)) - // We're creating a final node that will live unmorphed for the - // remainder of the current SelectionDAG iteration, so we can allocate - // the operands directly out of a pool with no recycling metadata. - N->InitOperands(OperandAllocator.Allocate(NumOps), - Ops, NumOps); - else - N->InitOperands(N->LocalOperands, Ops, NumOps); - N->OperandsNeedDelete = false; + createOperands(N, Ops); if (DoCSE) CSEMap.InsertNode(N, IP); @@ -6739,10 +6725,9 @@ GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, } AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, - SDValue X, unsigned SrcAS, - unsigned DestAS) - : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X), - SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} + unsigned SrcAS, unsigned DestAS) + : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)), + SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, EVT memvt, MachineMemOperand *mmo) @@ -6758,16 +6743,6 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!"); } -MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, - ArrayRef Ops, EVT memvt, MachineMemOperand *mmo) - : SDNode(Opc, Order, dl, VTs, Ops), - MemoryVT(memvt), MMO(mmo) { - SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), - MMO->isNonTemporal(), MMO->isInvariant()); - assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); - assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!"); -} - /// Profile - Gather unique data for the node. /// void SDNode::Profile(FoldingSetNodeID &ID) const {