replica
/
hanchenye-llvm-project
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Re-apply "SelectionDAG: Store SDNode operands in an ArrayRecycler" 

This re-applies r262886 with a fix for 32 bit platforms that have 8 byte
pointer alignment, effectively reverting r262892.

Original Message:

  Currently some SDNode operands are malloc'd, some are stored inline in
  subclasses of SDNode, and some are thrown into a BumpPtrAllocator.
  This scheme is complex, inconsistent, and makes refactoring SDNodes
  fairly difficult.

  Instead, we can allocate all of the operands using an ArrayRecycler
  that wraps a BumpPtrAllocator. This keeps the cache locality when
  iterating operands, improves locality when iterating SDNodes without
  looking at operands, and vastly simplifies the ownership semantics.

  It also means we stop overallocating SDNodes by 2-3x and will make it
  simpler to fix the rampant undefined behaviour we have in how we
  mutate SDNodes from one kind to another (See llvm.org/pr26808).

  This is NFC other than the changes in memory behaviour, and I ran some
  LNT tests to make sure this didn't hurt compile time. Not many tests
  changed: there were a couple of 1-2% regressions reported, but there
  were more improvements (of up to 4%) than regressions.

llvm-svn: 262902
This commit is contained in:
Justin Bogner 2016-03-08 03:14:29 +00:00
parent 5e5503099b
commit 671febc0f7
3 changed files with 222 additions and 393 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
llvm/include/llvm/CodeGen/SelectionDAG.h
View File

@ -23,6 +23,7 @@
#include "llvm/CodeGen/DAGCombine.h" #include "llvm/CodeGen/DAGCombine.h"
#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/Support/ArrayRecycler.h"
#include "llvm/Support/RecyclingAllocator.h" #include "llvm/Support/RecyclingAllocator.h"
#include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetMachine.h"
#include <cassert> #include <cassert>
@ -208,6 +209,7 @@ class SelectionDAG {
/// Pool allocation for machine-opcode SDNode operands. /// Pool allocation for machine-opcode SDNode operands.
BumpPtrAllocator OperandAllocator; BumpPtrAllocator OperandAllocator;
ArrayRecycler<SDUse> OperandRecycler;
/// Pool allocation for misc. objects that are created once per SelectionDAG. /// Pool allocation for misc. objects that are created once per SelectionDAG.
BumpPtrAllocator Allocator; BumpPtrAllocator Allocator;
@ -274,6 +276,30 @@ private:
SDNodeT(std::forward<ArgTypes>(Args)...); SDNodeT(std::forward<ArgTypes>(Args)...);
} }
void createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
assert(!Node->OperandList && "Node already has operands");
SDUse *Ops = OperandRecycler.allocate(
ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator);
for (unsigned I = 0; I != Vals.size(); ++I) {
Ops[I].setUser(Node);
Ops[I].setInitial(Vals[I]);
}
Node->NumOperands = Vals.size();
Node->OperandList = Ops;
checkForCycles(Node);
}
void removeOperands(SDNode *Node) {
if (!Node->OperandList)
return;
OperandRecycler.deallocate(
ArrayRecycler<SDUse>::Capacity::get(Node->NumOperands),
Node->OperandList);
Node->NumOperands = 0;
Node->OperandList = nullptr;
}
void operator=(const SelectionDAG&) = delete; void operator=(const SelectionDAG&) = delete;
SelectionDAG(const SelectionDAG&) = delete; SelectionDAG(const SelectionDAG&) = delete;
@ -1316,9 +1342,8 @@ private:
void allnodes_clear(); void allnodes_clear();
BinarySDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1,
SDValue N1, SDValue N2, SDValue N2, const SDNodeFlags *Flags = nullptr);
const SDNodeFlags *Flags = nullptr);
/// Look up the node specified by ID in CSEMap. If it exists, return it. If /// Look up the node specified by ID in CSEMap. If it exists, return it. If
/// not, return the insertion token that will make insertion faster. This /// not, return the insertion token that will make insertion faster. This

323
llvm/include/llvm/CodeGen/SelectionDAGNodes.h
View File

@ -44,7 +44,7 @@ class GlobalValue;
class MachineBasicBlock; class MachineBasicBlock;
class MachineConstantPoolValue; class MachineConstantPoolValue;
class SDNode; class SDNode;
class BinaryWithFlagsSDNode; class HandleSDNode;
class Value; class Value;
class MCSymbol; class MCSymbol;
template <typename T> struct DenseMapInfo; template <typename T> struct DenseMapInfo;
@ -280,6 +280,8 @@ public:
private: private:
friend class SelectionDAG; friend class SelectionDAG;
friend class SDNode; friend class SDNode;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;
void setUser(SDNode *p) { User = p; } void setUser(SDNode *p) { User = p; }
@ -394,10 +396,6 @@ private:
/// The operation that this node performs. /// The operation that this node performs.
int16_t NodeType; int16_t NodeType;
/// This is true if OperandList was new[]'d. If true,
/// then they will be delete[]'d when the node is destroyed.
uint16_t OperandsNeedDelete : 1;
/// This tracks whether this node has one or more dbg_value /// This tracks whether this node has one or more dbg_value
/// nodes corresponding to it. /// nodes corresponding to it.
uint16_t HasDebugValue : 1; uint16_t HasDebugValue : 1;
@ -406,7 +404,7 @@ protected:
/// This member is defined by this class, but is not used for /// This member is defined by this class, but is not used for
/// anything. Subclasses can use it to hold whatever state they find useful. /// anything. Subclasses can use it to hold whatever state they find useful.
/// This field is initialized to zero by the ctor. /// This field is initialized to zero by the ctor.
uint16_t SubclassData : 14; uint16_t SubclassData : 15;
private: private:
/// Unique id per SDNode in the DAG. /// Unique id per SDNode in the DAG.
@ -440,6 +438,8 @@ private:
friend class SelectionDAG; friend class SelectionDAG;
friend struct ilist_traits<SDNode>; friend struct ilist_traits<SDNode>;
// TODO: unfriend HandleSDNode once we fix its operand handling.
friend class HandleSDNode;
public: public:
/// Unique and persistent id per SDNode in the DAG. /// Unique and persistent id per SDNode in the DAG.
@ -792,101 +792,20 @@ protected:
return Ret; return Ret;
} }
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, /// Create an SDNode.
ArrayRef<SDValue> Ops) ///
: NodeType(Opc), OperandsNeedDelete(true), HasDebugValue(false), /// SDNodes are created without any operands, and never own the operand
SubclassData(0), NodeId(-1), /// storage. To add operands, see SelectionDAG::createOperands.
OperandList(Ops.size() ? new SDUse[Ops.size()] : nullptr),
ValueList(VTs.VTs), UseList(nullptr), NumOperands(Ops.size()),
NumValues(VTs.NumVTs), IROrder(Order), debugLoc(std::move(dl)) {
assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
assert(NumOperands == Ops.size() &&
"NumOperands wasn't wide enough for its operands!");
assert(NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!");
for (unsigned i = 0; i != Ops.size(); ++i) {
assert(OperandList && "no operands available");
OperandList[i].setUser(this);
OperandList[i].setInitial(Ops[i]);
}
checkForCycles(this);
}
/// This constructor adds no operands itself; operands can be
/// set later with InitOperands.
SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
: NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false), : NodeType(Opc), HasDebugValue(false), SubclassData(0), NodeId(-1),
SubclassData(0), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs), OperandList(nullptr), ValueList(VTs.VTs), UseList(nullptr),
UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs), NumOperands(0), NumValues(VTs.NumVTs), IROrder(Order),
IROrder(Order), debugLoc(std::move(dl)) { debugLoc(std::move(dl)) {
assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor");
assert(NumValues == VTs.NumVTs && assert(NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!"); "NumValues wasn't wide enough for its operands!");
} }
/// Initialize the operands list of this with 1 operand.
void InitOperands(SDUse *Ops, const SDValue &Op0) {
Ops[0].setUser(this);
Ops[0].setInitial(Op0);
NumOperands = 1;
OperandList = Ops;
checkForCycles(this);
}
/// Initialize the operands list of this with 2 operands.
void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1) {
Ops[0].setUser(this);
Ops[0].setInitial(Op0);
Ops[1].setUser(this);
Ops[1].setInitial(Op1);
NumOperands = 2;
OperandList = Ops;
checkForCycles(this);
}
/// Initialize the operands list of this with 3 operands.
void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1,
const SDValue &Op2) {
Ops[0].setUser(this);
Ops[0].setInitial(Op0);
Ops[1].setUser(this);
Ops[1].setInitial(Op1);
Ops[2].setUser(this);
Ops[2].setInitial(Op2);
NumOperands = 3;
OperandList = Ops;
checkForCycles(this);
}
/// Initialize the operands list of this with 4 operands.
void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1,
const SDValue &Op2, const SDValue &Op3) {
Ops[0].setUser(this);
Ops[0].setInitial(Op0);
Ops[1].setUser(this);
Ops[1].setInitial(Op1);
Ops[2].setUser(this);
Ops[2].setInitial(Op2);
Ops[3].setUser(this);
Ops[3].setInitial(Op3);
NumOperands = 4;
OperandList = Ops;
checkForCycles(this);
}
/// Initialize the operands list of this with N operands.
void InitOperands(SDUse *Ops, const SDValue *Vals, unsigned N) {
for (unsigned i = 0; i != N; ++i) {
Ops[i].setUser(this);
Ops[i].setInitial(Vals[i]);
}
NumOperands = N;
assert(NumOperands == N &&
"NumOperands wasn't wide enough for its operands!");
OperandList = Ops;
checkForCycles(this);
}
/// Release the operands and set this node to have zero operands. /// Release the operands and set this node to have zero operands.
void DropOperands(); void DropOperands();
}; };
@ -1012,30 +931,6 @@ inline void SDUse::setNode(SDNode *N) {
if (N) N->addUse(*this); if (N) N->addUse(*this);
} }
/// This class is used for single-operand SDNodes. This is solely
/// to allow co-allocation of node operands with the node itself.
class UnarySDNode : public SDNode {
SDUse Op;
public:
UnarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
SDValue X)
: SDNode(Opc, Order, dl, VTs) {
InitOperands(&Op, X);
}
};
/// This class is used for two-operand SDNodes. This is solely
/// to allow co-allocation of node operands with the node itself.
class BinarySDNode : public SDNode {
SDUse Ops[2];
public:
BinarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
SDValue X, SDValue Y)
: SDNode(Opc, Order, dl, VTs) {
InitOperands(Ops, X, Y);
}
};
/// Returns true if the opcode is a binary operation with flags. /// Returns true if the opcode is a binary operation with flags.
static bool isBinOpWithFlags(unsigned Opcode) { static bool isBinOpWithFlags(unsigned Opcode) {
switch (Opcode) { switch (Opcode) {
@ -1060,30 +955,17 @@ static bool isBinOpWithFlags(unsigned Opcode) {
/// This class is an extension of BinarySDNode /// This class is an extension of BinarySDNode
/// used from those opcodes that have associated extra flags. /// used from those opcodes that have associated extra flags.
class BinaryWithFlagsSDNode : public BinarySDNode { class BinaryWithFlagsSDNode : public SDNode {
public: public:
SDNodeFlags Flags; SDNodeFlags Flags;
BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
SDValue X, SDValue Y, const SDNodeFlags &NodeFlags) const SDNodeFlags &NodeFlags)
: BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags(NodeFlags) {} : SDNode(Opc, Order, dl, VTs), Flags(NodeFlags) {}
static bool classof(const SDNode *N) { static bool classof(const SDNode *N) {
return isBinOpWithFlags(N->getOpcode()); return isBinOpWithFlags(N->getOpcode());
} }
}; };
/// This class is used for three-operand SDNodes. This is solely
/// to allow co-allocation of node operands with the node itself.
class TernarySDNode : public SDNode {
SDUse Ops[3];
public:
TernarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
SDValue X, SDValue Y, SDValue Z)
: SDNode(Opc, Order, dl, VTs) {
InitOperands(Ops, X, Y, Z);
}
};
/// This class is used to form a handle around another node that /// This class is used to form a handle around another node that
/// is persistent and is updated across invocations of replaceAllUsesWith on its /// is persistent and is updated across invocations of replaceAllUsesWith on its
/// operand. This node should be directly created by end-users and not added to /// operand. This node should be directly created by end-users and not added to
@ -1096,20 +978,28 @@ public:
// HandleSDNodes are never inserted into the DAG, so they won't be // HandleSDNodes are never inserted into the DAG, so they won't be
// auto-numbered. Use ID 65535 as a sentinel. // auto-numbered. Use ID 65535 as a sentinel.
PersistentId = 0xffff; PersistentId = 0xffff;
InitOperands(&Op, X);
// Manually set up the operand list. This node type is special in that it's
// always stack allocated and SelectionDAG does not manage its operands.
// TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
// be so special.
Op.setUser(this);
Op.setInitial(X);
NumOperands = 1;
OperandList = &Op;
} }
~HandleSDNode(); ~HandleSDNode();
const SDValue &getValue() const { return Op; } const SDValue &getValue() const { return Op; }
}; };
class AddrSpaceCastSDNode : public UnarySDNode { class AddrSpaceCastSDNode : public SDNode {
private: private:
unsigned SrcAddrSpace; unsigned SrcAddrSpace;
unsigned DestAddrSpace; unsigned DestAddrSpace;
public: public:
AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, SDValue X, AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, unsigned SrcAS,
unsigned SrcAS, unsigned DestAS); unsigned DestAS);
unsigned getSrcAddressSpace() const { return SrcAddrSpace; } unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
unsigned getDestAddressSpace() const { return DestAddrSpace; } unsigned getDestAddressSpace() const { return DestAddrSpace; }
@ -1133,9 +1023,6 @@ public:
MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
EVT MemoryVT, MachineMemOperand *MMO); EVT MemoryVT, MachineMemOperand *MMO);
MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
ArrayRef<SDValue> Ops, EVT MemoryVT, MachineMemOperand *MMO);
bool readMem() const { return MMO->isLoad(); } bool readMem() const { return MMO->isLoad(); }
bool writeMem() const { return MMO->isStore(); } bool writeMem() const { return MMO->isStore(); }
@ -1238,8 +1125,6 @@ public:
/// This is an SDNode representing atomic operations. /// This is an SDNode representing atomic operations.
class AtomicSDNode : public MemSDNode { class AtomicSDNode : public MemSDNode {
SDUse Ops[4];
/// For cmpxchg instructions, the ordering requirements when a store does not /// For cmpxchg instructions, the ordering requirements when a store does not
/// occur. /// occur.
AtomicOrdering FailureOrdering; AtomicOrdering FailureOrdering;
@ -1265,16 +1150,12 @@ class AtomicSDNode : public MemSDNode {
} }
public: public:
AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, EVT MemVT, AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL,
const SDValue* AllOps, SDUse *DynOps, unsigned NumOps, EVT MemVT, MachineMemOperand *MMO,
MachineMemOperand *MMO,
AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering,
SynchronizationScope SynchScope) SynchronizationScope SynchScope)
: MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
InitAtomic(SuccessOrdering, FailureOrdering, SynchScope); InitAtomic(SuccessOrdering, FailureOrdering, SynchScope);
assert((DynOps || NumOps <= array_lengthof(Ops)) &&
"Too many ops for internal storage!");
InitOperands(DynOps ? DynOps : Ops, AllOps, NumOps);
} }
const SDValue &getBasePtr() const { return getOperand(1); } const SDValue &getBasePtr() const { return getOperand(1); }
@ -1322,9 +1203,8 @@ public:
class MemIntrinsicSDNode : public MemSDNode { class MemIntrinsicSDNode : public MemSDNode {
public: public:
MemIntrinsicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, MemIntrinsicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
ArrayRef<SDValue> Ops, EVT MemoryVT, EVT MemoryVT, MachineMemOperand *MMO)
MachineMemOperand *MMO) : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
: MemSDNode(Opc, Order, dl, VTs, Ops, MemoryVT, MMO) {
SubclassData |= 1u << 13; SubclassData |= 1u << 13;
} }
@ -1347,20 +1227,15 @@ public:
/// An index of -1 is treated as undef, such that the code generator may put /// An index of -1 is treated as undef, such that the code generator may put
/// any value in the corresponding element of the result. /// any value in the corresponding element of the result.
class ShuffleVectorSDNode : public SDNode { class ShuffleVectorSDNode : public SDNode {
SDUse Ops[2];
// The memory for Mask is owned by the SelectionDAG's OperandAllocator, and // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
// is freed when the SelectionDAG object is destroyed. // is freed when the SelectionDAG object is destroyed.
const int *Mask; const int *Mask;
protected: protected:
friend class SelectionDAG; friend class SelectionDAG;
ShuffleVectorSDNode(EVT VT, unsigned Order, DebugLoc dl, SDValue N1, ShuffleVectorSDNode(EVT VT, unsigned Order, DebugLoc dl, const int *M)
SDValue N2, const int *M) : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
: SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {
InitOperands(Ops, N1, N2);
}
public:
public:
ArrayRef<int> getMask() const { ArrayRef<int> getMask() const {
EVT VT = getValueType(0); EVT VT = getValueType(0);
return makeArrayRef(Mask, VT.getVectorNumElements()); return makeArrayRef(Mask, VT.getVectorNumElements());
@ -1791,13 +1666,11 @@ public:
}; };
class EHLabelSDNode : public SDNode { class EHLabelSDNode : public SDNode {
SDUse Chain;
MCSymbol *Label; MCSymbol *Label;
friend class SelectionDAG; friend class SelectionDAG;
EHLabelSDNode(unsigned Order, DebugLoc dl, SDValue ch, MCSymbol *L) EHLabelSDNode(unsigned Order, DebugLoc dl, MCSymbol *L)
: SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) { : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) {}
InitOperands(&Chain, ch);
}
public: public:
MCSymbol *getLabel() const { return Label; } MCSymbol *getLabel() const { return Label; }
@ -1863,11 +1736,10 @@ class CvtRndSatSDNode : public SDNode {
ISD::CvtCode CvtCode; ISD::CvtCode CvtCode;
friend class SelectionDAG; friend class SelectionDAG;
explicit CvtRndSatSDNode(EVT VT, unsigned Order, DebugLoc dl, explicit CvtRndSatSDNode(EVT VT, unsigned Order, DebugLoc dl,
ArrayRef<SDValue> Ops, ISD::CvtCode Code) ISD::CvtCode Code)
: SDNode(ISD::CONVERT_RNDSAT, Order, dl, getSDVTList(VT), Ops), : SDNode(ISD::CONVERT_RNDSAT, Order, dl, getSDVTList(VT)), CvtCode(Code) {
CvtCode(Code) {
assert(Ops.size() == 5 && "wrong number of operations");
} }
public: public:
ISD::CvtCode getCvtCode() const { return CvtCode; } ISD::CvtCode getCvtCode() const { return CvtCode; }
@ -1896,24 +1768,13 @@ public:
/// Base class for LoadSDNode and StoreSDNode /// Base class for LoadSDNode and StoreSDNode
class LSBaseSDNode : public MemSDNode { class LSBaseSDNode : public MemSDNode {
//! Operand array for load and store
/*!
\note Moving this array to the base class captures more
common functionality shared between LoadSDNode and
StoreSDNode
*/
SDUse Ops[4];
public: public:
LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl,
SDValue *Operands, unsigned numOperands,
SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
MachineMemOperand *MMO) MachineMemOperand *MMO)
: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
SubclassData |= AM << 2; SubclassData |= AM << 2;
assert(getAddressingMode() == AM && "MemIndexedMode encoding error!"); assert(getAddressingMode() == AM && "MemIndexedMode encoding error!");
InitOperands(Ops, Operands, numOperands);
assert((getOffset().getOpcode() == ISD::UNDEF || isIndexed()) &&
"Only indexed loads and stores have a non-undef offset operand");
} }
const SDValue &getOffset() const { const SDValue &getOffset() const {
@ -1941,10 +1802,9 @@ public:
/// This class is used to represent ISD::LOAD nodes. /// This class is used to represent ISD::LOAD nodes.
class LoadSDNode : public LSBaseSDNode { class LoadSDNode : public LSBaseSDNode {
friend class SelectionDAG; friend class SelectionDAG;
LoadSDNode(SDValue *ChainPtrOff, unsigned Order, DebugLoc dl, SDVTList VTs, LoadSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM,
ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT, ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO)
MachineMemOperand *MMO) : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
: LSBaseSDNode(ISD::LOAD, Order, dl, ChainPtrOff, 3, VTs, AM, MemVT, MMO) {
SubclassData |= (unsigned short)ETy; SubclassData |= (unsigned short)ETy;
assert(getExtensionType() == ETy && "LoadExtType encoding error!"); assert(getExtensionType() == ETy && "LoadExtType encoding error!");
assert(readMem() && "Load MachineMemOperand is not a load!"); assert(readMem() && "Load MachineMemOperand is not a load!");
@ -1969,11 +1829,9 @@ public:
/// This class is used to represent ISD::STORE nodes. /// This class is used to represent ISD::STORE nodes.
class StoreSDNode : public LSBaseSDNode { class StoreSDNode : public LSBaseSDNode {
friend class SelectionDAG; friend class SelectionDAG;
StoreSDNode(SDValue *ChainValuePtrOff, unsigned Order, DebugLoc dl, StoreSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM,
SDVTList VTs, ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, bool isTrunc, EVT MemVT, MachineMemOperand *MMO)
MachineMemOperand *MMO) : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
: LSBaseSDNode(ISD::STORE, Order, dl, ChainValuePtrOff, 4,
VTs, AM, MemVT, MMO) {
SubclassData |= (unsigned short)isTrunc; SubclassData |= (unsigned short)isTrunc;
assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!"); assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!");
assert(!readMem() && "Store MachineMemOperand is a load!"); assert(!readMem() && "Store MachineMemOperand is a load!");
@ -1997,16 +1855,11 @@ public:
/// This base class is used to represent MLOAD and MSTORE nodes /// This base class is used to represent MLOAD and MSTORE nodes
class MaskedLoadStoreSDNode : public MemSDNode { class MaskedLoadStoreSDNode : public MemSDNode {
// Operands
SDUse Ops[4];
public: public:
friend class SelectionDAG; friend class SelectionDAG;
MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl,
SDValue *Operands, unsigned numOperands, SDVTList VTs, SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
EVT MemVT, MachineMemOperand *MMO) : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {}
: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
InitOperands(Ops, Operands, numOperands);
}
// In the both nodes address is Op1, mask is Op2: // In the both nodes address is Op1, mask is Op2:
// MaskedLoadSDNode (Chain, ptr, mask, src0), src0 is a passthru value // MaskedLoadSDNode (Chain, ptr, mask, src0), src0 is a passthru value
@ -2025,11 +1878,9 @@ public:
class MaskedLoadSDNode : public MaskedLoadStoreSDNode { class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
public: public:
friend class SelectionDAG; friend class SelectionDAG;
MaskedLoadSDNode(unsigned Order, DebugLoc dl, SDValue *Operands, MaskedLoadSDNode(unsigned Order, DebugLoc dl, SDVTList VTs,
unsigned numOperands, SDVTList VTs, ISD::LoadExtType ETy, ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO)
EVT MemVT, MachineMemOperand *MMO) : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, MemVT, MMO) {
: MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, Operands, numOperands,
VTs, MemVT, MMO) {
SubclassData |= (unsigned short)ETy; SubclassData |= (unsigned short)ETy;
} }
@ -2047,12 +1898,10 @@ class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
public: public:
friend class SelectionDAG; friend class SelectionDAG;
MaskedStoreSDNode(unsigned Order, DebugLoc dl, SDValue *Operands, MaskedStoreSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, bool isTrunc,
unsigned numOperands, SDVTList VTs, bool isTrunc, EVT MemVT, EVT MemVT, MachineMemOperand *MMO)
MachineMemOperand *MMO) : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) {
: MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, Operands, numOperands, SubclassData |= (unsigned short)isTrunc;
VTs, MemVT, MMO) {
SubclassData |= (unsigned short)isTrunc;
} }
/// Return true if the op does a truncation before store. /// Return true if the op does a truncation before store.
/// For integers this is the same as doing a TRUNCATE and storing the result. /// For integers this is the same as doing a TRUNCATE and storing the result.
@ -2070,17 +1919,11 @@ public:
/// MGATHER and MSCATTER nodes /// MGATHER and MSCATTER nodes
/// ///
class MaskedGatherScatterSDNode : public MemSDNode { class MaskedGatherScatterSDNode : public MemSDNode {
// Operands
SDUse Ops[5];
public: public:
friend class SelectionDAG; friend class SelectionDAG;
MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl,
ArrayRef<SDValue> Operands, SDVTList VTs, EVT MemVT, SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
MachineMemOperand *MMO) : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {}
: MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
assert(Operands.size() == 5 && "Incompatible number of operands");
InitOperands(Ops, Operands.data(), Operands.size());
}
// In the both nodes address is Op1, mask is Op2: // In the both nodes address is Op1, mask is Op2:
// MaskedGatherSDNode (Chain, src0, mask, base, index), src0 is a passthru value // MaskedGatherSDNode (Chain, src0, mask, base, index), src0 is a passthru value
@ -2102,19 +1945,9 @@ public:
class MaskedGatherSDNode : public MaskedGatherScatterSDNode { class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
public: public:
friend class SelectionDAG; friend class SelectionDAG;
MaskedGatherSDNode(unsigned Order, DebugLoc dl, ArrayRef<SDValue> Operands, MaskedGatherSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, EVT MemVT,
SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) MachineMemOperand *MMO)
: MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, Operands, VTs, MemVT, : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO) {}
MMO) {
assert(getValue().getValueType() == getValueType(0) &&
"Incompatible type of the PassThru value in MaskedGatherSDNode");
assert(getMask().getValueType().getVectorNumElements() ==
getValueType(0).getVectorNumElements() &&
"Vector width mismatch between mask and data");
assert(getIndex().getValueType().getVectorNumElements() ==
getValueType(0).getVectorNumElements() &&
"Vector width mismatch between index and data");
}
static bool classof(const SDNode *N) { static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::MGATHER; return N->getOpcode() == ISD::MGATHER;
@ -2127,17 +1960,9 @@ class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
public: public:
friend class SelectionDAG; friend class SelectionDAG;
MaskedScatterSDNode(unsigned Order, DebugLoc dl,ArrayRef<SDValue> Operands, MaskedScatterSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, EVT MemVT,
SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) MachineMemOperand *MMO)
: MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, Operands, VTs, MemVT, : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO) {}
MMO) {
assert(getMask().getValueType().getVectorNumElements() ==
getValue().getValueType().getVectorNumElements() &&
"Vector width mismatch between mask and data");
assert(getIndex().getValueType().getVectorNumElements() ==
getValue().getValueType().getVectorNumElements() &&
"Vector width mismatch between index and data");
}
static bool classof(const SDNode *N) { static bool classof(const SDNode *N) {
return N->getOpcode() == ISD::MSCATTER; return N->getOpcode() == ISD::MSCATTER;
@ -2156,10 +1981,6 @@ private:
MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc DL, SDVTList VTs) MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc DL, SDVTList VTs)
: SDNode(Opc, Order, DL, VTs), MemRefs(nullptr), MemRefsEnd(nullptr) {} : SDNode(Opc, Order, DL, VTs), MemRefs(nullptr), MemRefsEnd(nullptr) {}
/// Operands for this instruction, if they fit here. If
/// they don't, this field is unused.
SDUse LocalOperands[4];
/// Memory reference descriptions for this instruction. /// Memory reference descriptions for this instruction.
mmo_iterator MemRefs; mmo_iterator MemRefs;
mmo_iterator MemRefsEnd; mmo_iterator MemRefsEnd;
@ -2234,8 +2055,16 @@ template <> struct GraphTraits<SDNode*> {
} }
}; };
/// The largest SDNode class. /// A representation of the largest SDNode, for use in sizeof().
typedef MaskedGatherScatterSDNode LargestSDNode; ///
/// This needs to be a union because the largest node differs on 32 bit systems
/// with 4 and 8 byte pointer alignment, respectively.
union LargestSDNode {
AtomicSDNode Atomic;
TargetIndexSDNode TargetIndex;
BlockAddressSDNode BlockAddress;
GlobalAddressSDNode GlobalAddress;
};
/// The SDNode class with the greatest alignment requirement. /// The SDNode class with the greatest alignment requirement.
typedef GlobalAddressSDNode MostAlignedSDNode; typedef GlobalAddressSDNode MostAlignedSDNode;

261
llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
View File

@ -664,8 +664,8 @@ void SDDbgInfo::erase(const SDNode *Node) {
} }
void SelectionDAG::DeallocateNode(SDNode *N) { void SelectionDAG::DeallocateNode(SDNode *N) {
if (N->OperandsNeedDelete) // If we have operands, deallocate them.
delete[] N->OperandList; removeOperands(N);
// Set the opcode to DELETED_NODE to help catch bugs when node // Set the opcode to DELETED_NODE to help catch bugs when node
// memory is reallocated. // memory is reallocated.
@ -912,6 +912,7 @@ void SelectionDAG::init(MachineFunction &mf) {
SelectionDAG::~SelectionDAG() { SelectionDAG::~SelectionDAG() {
assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); assert(!UpdateListeners && "Dangling registered DAGUpdateListeners");
allnodes_clear(); allnodes_clear();
OperandRecycler.clear(OperandAllocator);
delete DbgInfo; delete DbgInfo;
} }
@ -925,24 +926,26 @@ void SelectionDAG::allnodes_clear() {
#endif #endif
} }
BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL, SDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
SDVTList VTs, SDValue N1, SDValue N1, SDValue N2,
SDValue N2, const SDNodeFlags *Flags) {
const SDNodeFlags *Flags) { SDValue Ops[] = {N1, N2};
if (isBinOpWithFlags(Opcode)) { if (isBinOpWithFlags(Opcode)) {
// If no flags were passed in, use a default flags object. // If no flags were passed in, use a default flags object.
SDNodeFlags F; SDNodeFlags F;
if (Flags == nullptr) if (Flags == nullptr)
Flags = &F; Flags = &F;
auto *FN = newSDNode<BinaryWithFlagsSDNode>( auto *FN = newSDNode<BinaryWithFlagsSDNode>(Opcode, DL.getIROrder(),
Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, *Flags); DL.getDebugLoc(), VTs, *Flags);
createOperands(FN, Ops);
return FN; return FN;
} }
auto *N = newSDNode<BinarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
VTs, N1, N2); createOperands(N, Ops);
return N; return N;
} }
@ -982,6 +985,7 @@ SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID,
void SelectionDAG::clear() { void SelectionDAG::clear() {
allnodes_clear(); allnodes_clear();
OperandRecycler.clear(OperandAllocator);
OperandAllocator.Reset(); OperandAllocator.Reset();
CSEMap.clear(); CSEMap.clear();
@ -1638,7 +1642,9 @@ SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1,
memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int));
auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(), auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(),
dl.getDebugLoc(), N1, N2, MaskAlloc); dl.getDebugLoc(), MaskAlloc);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -1671,8 +1677,10 @@ SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl,
if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
return SDValue(E, 0); return SDValue(E, 0);
auto *N = newSDNode<CvtRndSatSDNode>(VT, dl.getIROrder(), dl.getDebugLoc(), auto *N =
Ops, Code); newSDNode<CvtRndSatSDNode>(VT, dl.getIROrder(), dl.getDebugLoc(), Code);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -1715,8 +1723,9 @@ SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) {
if (SDNode *E = FindNodeOrInsertPos(ID, IP)) if (SDNode *E = FindNodeOrInsertPos(ID, IP))
return SDValue(E, 0); return SDValue(E, 0);
auto *N = auto *N = newSDNode<EHLabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Label);
newSDNode<EHLabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Root, Label); createOperands(N, Ops);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -1799,7 +1808,9 @@ SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr,
return SDValue(E, 0); return SDValue(E, 0);
auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(), auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(),
VT, Ptr, SrcAS, DestAS); VT, SrcAS, DestAS);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -3183,20 +3194,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL,
SDNode *N; SDNode *N;
SDVTList VTs = getVTList(VT); SDVTList VTs = getVTList(VT);
SDValue Ops[] = {Operand};
if (VT != MVT::Glue) { // Don't CSE flag producing nodes if (VT != MVT::Glue) { // Don't CSE flag producing nodes
FoldingSetNodeID ID; FoldingSetNodeID ID;
SDValue Ops[1] = { Operand };
AddNodeIDNode(ID, Opcode, VTs, Ops); AddNodeIDNode(ID, Opcode, VTs, Ops);
void *IP = nullptr; void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0); return SDValue(E, 0);
N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
Operand); createOperands(N, Ops);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
} else { } else {
N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
Operand); createOperands(N, Ops);
} }
InsertNode(N); InsertNode(N);
@ -3895,7 +3906,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
} }
// Memoize this node if possible. // Memoize this node if possible.
BinarySDNode *N; SDNode *N;
SDVTList VTs = getVTList(VT); SDVTList VTs = getVTList(VT);
if (VT != MVT::Glue) { if (VT != MVT::Glue) {
SDValue Ops[] = {N1, N2}; SDValue Ops[] = {N1, N2};
@ -3909,7 +3920,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1,
} }
N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags); N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
} else { } else {
N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags); N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags);
@ -4000,20 +4010,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
// Memoize node if it doesn't produce a flag. // Memoize node if it doesn't produce a flag.
SDNode *N; SDNode *N;
SDVTList VTs = getVTList(VT); SDVTList VTs = getVTList(VT);
SDValue Ops[] = {N1, N2, N3};
if (VT != MVT::Glue) { if (VT != MVT::Glue) {
SDValue Ops[] = { N1, N2, N3 };
FoldingSetNodeID ID; FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTs, Ops); AddNodeIDNode(ID, Opcode, VTs, Ops);
void *IP = nullptr; void *IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0); return SDValue(E, 0);
N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
N1, N2, N3); createOperands(N, Ops);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
} else { } else {
N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
N1, N2, N3); createOperands(N, Ops);
} }
InsertNode(N); InsertNode(N);
@ -4805,18 +4815,11 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT,
return SDValue(E, 0); return SDValue(E, 0);
} }
// Allocate the operands array for the node out of the BumpPtrAllocator, since auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
// SDNode doesn't have access to it. This memory will be "leaked" when VTList, MemVT, MMO, SuccessOrdering,
// the node is deallocated, but recovered when the allocator is released. FailureOrdering, SynchScope);
// If the number of operands is less than 5 we use AtomicSDNode's internal createOperands(N, Ops);
// storage.
unsigned NumOps = Ops.size();
SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps)
: nullptr;
auto *N = newSDNode<AtomicSDNode>(
Opcode, dl.getIROrder(), dl.getDebugLoc(), VTList, MemVT, Ops.data(),
DynOps, NumOps, MMO, SuccessOrdering, FailureOrdering, SynchScope);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -5009,11 +5012,14 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList,
} }
N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
VTList, Ops, MemVT, MMO); VTList, MemVT, MMO);
CSEMap.InsertNode(N, IP); createOperands(N, Ops);
CSEMap.InsertNode(N, IP);
} else { } else {
N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(),
VTList, Ops, MemVT, MMO); VTList, MemVT, MMO);
createOperands(N, Ops);
} }
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -5131,8 +5137,10 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType,
cast<LoadSDNode>(E)->refineAlignment(MMO); cast<LoadSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0); return SDValue(E, 0);
} }
auto *N = newSDNode<LoadSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
AM, ExtType, MemVT, MMO); ExtType, MemVT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -5238,8 +5246,10 @@ SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val,
cast<StoreSDNode>(E)->refineAlignment(MMO); cast<StoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0); return SDValue(E, 0);
} }
auto *N = newSDNode<StoreSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
ISD::UNINDEXED, false, VT, MMO); ISD::UNINDEXED, false, VT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -5306,8 +5316,10 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val,
cast<StoreSDNode>(E)->refineAlignment(MMO); cast<StoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0); return SDValue(E, 0);
} }
auto *N = newSDNode<StoreSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
ISD::UNINDEXED, true, SVT, MMO); ISD::UNINDEXED, true, SVT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -5330,9 +5342,11 @@ SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base,
if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP))
return SDValue(E, 0); return SDValue(E, 0);
auto *N = newSDNode<StoreSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM,
AM, ST->isTruncatingStore(), ST->isTruncatingStore(), ST->getMemoryVT(),
ST->getMemoryVT(), ST->getMemOperand()); ST->getMemOperand());
createOperands(N, Ops);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -5358,8 +5372,10 @@ SelectionDAG::getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain,
cast<MaskedLoadSDNode>(E)->refineAlignment(MMO); cast<MaskedLoadSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0); return SDValue(E, 0);
} }
auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), Ops, auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
4, VTs, ExtTy, MemVT, MMO); ExtTy, MemVT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -5384,8 +5400,10 @@ SDValue SelectionDAG::getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val,
cast<MaskedStoreSDNode>(E)->refineAlignment(MMO); cast<MaskedStoreSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0); return SDValue(E, 0);
} }
auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), Ops, auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
4, VTs, isTrunc, MemVT, MMO); isTrunc, MemVT, MMO);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -5395,6 +5413,7 @@ SDValue
SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl, SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl,
ArrayRef<SDValue> Ops, ArrayRef<SDValue> Ops,
MachineMemOperand *MMO) { MachineMemOperand *MMO) {
assert(Ops.size() == 5 && "Incompatible number of operands");
FoldingSetNodeID ID; FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops);
@ -5409,8 +5428,20 @@ SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl,
cast<MaskedGatherSDNode>(E)->refineAlignment(MMO); cast<MaskedGatherSDNode>(E)->refineAlignment(MMO);
return SDValue(E, 0); return SDValue(E, 0);
} }
auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(),
Ops, VTs, VT, MMO); VTs, VT, MMO);
createOperands(N, Ops);
assert(N->getValue().getValueType() == N->getValueType(0) &&
"Incompatible type of the PassThru value in MaskedGatherSDNode");
assert(N->getMask().getValueType().getVectorNumElements() ==
N->getValueType(0).getVectorNumElements() &&
"Vector width mismatch between mask and data");
assert(N->getIndex().getValueType().getVectorNumElements() ==
N->getValueType(0).getVectorNumElements() &&
"Vector width mismatch between index and data");
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -5419,6 +5450,8 @@ SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl,
SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl, SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl,
ArrayRef<SDValue> Ops, ArrayRef<SDValue> Ops,
MachineMemOperand *MMO) { MachineMemOperand *MMO) {
assert(Ops.size() == 5 && "Incompatible number of operands");
FoldingSetNodeID ID; FoldingSetNodeID ID;
AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops);
ID.AddInteger(VT.getRawBits()); ID.AddInteger(VT.getRawBits());
@ -5432,7 +5465,16 @@ SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl,
return SDValue(E, 0); return SDValue(E, 0);
} }
auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(),
Ops, VTs, VT, MMO); VTs, VT, MMO);
createOperands(N, Ops);
assert(N->getMask().getValueType().getVectorNumElements() ==
N->getValue().getValueType().getVectorNumElements() &&
"Vector width mismatch between mask and data");
assert(N->getIndex().getValueType().getVectorNumElements() ==
N->getValue().getValueType().getVectorNumElements() &&
"Vector width mismatch between index and data");
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -5511,10 +5553,13 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT,
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0); return SDValue(E, 0);
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops); N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
createOperands(N, Ops);
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
} else { } else {
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops); N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
createOperands(N, Ops);
} }
InsertNode(N); InsertNode(N);
@ -5556,7 +5601,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
// Memoize the node unless it returns a flag. // Memoize the node unless it returns a flag.
SDNode *N; SDNode *N;
unsigned NumOps = Ops.size();
if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) {
FoldingSetNodeID ID; FoldingSetNodeID ID;
AddNodeIDNode(ID, Opcode, VTList, Ops); AddNodeIDNode(ID, Opcode, VTList, Ops);
@ -5564,34 +5608,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList,
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP))
return SDValue(E, 0); return SDValue(E, 0);
if (NumOps == 1) { N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), createOperands(N, Ops);
VTList, Ops[0]);
} else if (NumOps == 2) {
N = newSDNode<BinarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
VTList, Ops[0], Ops[1]);
} else if (NumOps == 3) {
N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
VTList, Ops[0], Ops[1], Ops[2]);
} else {
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList,
Ops);
}
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
} else { } else {
if (NumOps == 1) { N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList);
N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), createOperands(N, Ops);
VTList, Ops[0]);
} else if (NumOps == 2) {
N = newSDNode<BinarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
VTList, Ops[0], Ops[1]);
} else if (NumOps == 3) {
N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(),
VTList, Ops[0], Ops[1], Ops[2]);
} else {
N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList,
Ops);
}
} }
InsertNode(N); InsertNode(N);
return SDValue(N, 0); return SDValue(N, 0);
@ -5977,7 +5999,6 @@ SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) {
/// deleting things. /// deleting things.
SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
SDVTList VTs, ArrayRef<SDValue> Ops) { SDVTList VTs, ArrayRef<SDValue> Ops) {
unsigned NumOps = Ops.size();
// If an identical node already exists, use it. // If an identical node already exists, use it.
void *IP = nullptr; void *IP = nullptr;
if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) {
@ -6006,36 +6027,13 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc,
DeadNodeSet.insert(Used); DeadNodeSet.insert(Used);
} }
if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) { // For MachineNode, initialize the memory references information.
// Initialize the memory references information. if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N))
MN->setMemRefs(nullptr, nullptr); MN->setMemRefs(nullptr, nullptr);
// If NumOps is larger than the # of operands we can have in a
// MachineSDNode, reallocate the operand list. // Swap for an appropriately sized array from the recycler.
if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) { removeOperands(N);
if (MN->OperandsNeedDelete) createOperands(N, Ops);
delete[] MN->OperandList;
if (NumOps > array_lengthof(MN->LocalOperands))
// We're creating a final node that will live unmorphed for the
// remainder of the current SelectionDAG iteration, so we can allocate
// the operands directly out of a pool with no recycling metadata.
MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
Ops.data(), NumOps);
else
MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps);
MN->OperandsNeedDelete = false;
} else
MN->InitOperands(MN->OperandList, Ops.data(), NumOps);
} else {
// If NumOps is larger than the # of operands we currently have, reallocate
// the operand list.
if (NumOps > N->NumOperands) {
if (N->OperandsNeedDelete)
delete[] N->OperandList;
N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps);
N->OperandsNeedDelete = true;
} else
N->InitOperands(N->OperandList, Ops.data(), NumOps);
}
// Delete any nodes that are still dead after adding the uses for the // Delete any nodes that are still dead after adding the uses for the
// new operands. // new operands.
@ -6178,16 +6176,14 @@ SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl,
MachineSDNode * MachineSDNode *
SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
ArrayRef<SDValue> OpsArray) { ArrayRef<SDValue> Ops) {
bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue;
MachineSDNode *N; MachineSDNode *N;
void *IP = nullptr; void *IP = nullptr;
const SDValue *Ops = OpsArray.data();
unsigned NumOps = OpsArray.size();
if (DoCSE) { if (DoCSE) {
FoldingSetNodeID ID; FoldingSetNodeID ID;
AddNodeIDNode(ID, ~Opcode, VTs, OpsArray); AddNodeIDNode(ID, ~Opcode, VTs, Ops);
IP = nullptr; IP = nullptr;
if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) { if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) {
return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL)); return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL));
@ -6196,17 +6192,7 @@ SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs,
// Allocate a new MachineSDNode. // Allocate a new MachineSDNode.
N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs);
createOperands(N, Ops);
// Initialize the operands list.
if (NumOps > array_lengthof(N->LocalOperands))
// We're creating a final node that will live unmorphed for the
// remainder of the current SelectionDAG iteration, so we can allocate
// the operands directly out of a pool with no recycling metadata.
N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps),
Ops, NumOps);
else
N->InitOperands(N->LocalOperands, Ops, NumOps);
N->OperandsNeedDelete = false;
if (DoCSE) if (DoCSE)
CSEMap.InsertNode(N, IP); CSEMap.InsertNode(N, IP);
@ -6739,10 +6725,9 @@ GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order,
} }
AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT,
SDValue X, unsigned SrcAS, unsigned SrcAS, unsigned DestAS)
unsigned DestAS) : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)),
: UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X), SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {}
MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
EVT memvt, MachineMemOperand *mmo) EVT memvt, MachineMemOperand *mmo)
@ -6758,16 +6743,6 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!"); assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
} }
MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs,
ArrayRef<SDValue> Ops, EVT memvt, MachineMemOperand *mmo)
: SDNode(Opc, Order, dl, VTs, Ops),
MemoryVT(memvt), MMO(mmo) {
SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(),
MMO->isNonTemporal(), MMO->isInvariant());
assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!");
assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!");
}
/// Profile - Gather unique data for the node. /// Profile - Gather unique data for the node.
/// ///
void SDNode::Profile(FoldingSetNodeID &ID) const { void SDNode::Profile(FoldingSetNodeID &ID) const {