diff --git a/llvm/include/llvm/CodeGen/LiveInterval.h b/llvm/include/llvm/CodeGen/LiveInterval.h index a936ac642553..8b3bb41b7876 100644 --- a/llvm/include/llvm/CodeGen/LiveInterval.h +++ b/llvm/include/llvm/CodeGen/LiveInterval.h @@ -204,6 +204,25 @@ namespace llvm { const_vni_iterator vni_begin() const { return valnos.begin(); } const_vni_iterator vni_end() const { return valnos.end(); } + /// Constructs a new LiveRange object. + LiveRange() { + } + + /// Constructs a new LiveRange object by copying segments and valnos from + /// another LiveRange. + LiveRange(const LiveRange &Other, BumpPtrAllocator &Allocator) { + // Duplicate valnos. + for (LiveRange::const_vni_iterator I = Other.vni_begin(), + E = Other.vni_end(); I != E; ++I) { + createValueCopy(*I, Allocator); + } + // Now we can copy segments and remap their valnos. + for (LiveRange::const_iterator I = Other.begin(), E = Other.end(); + I != E; ++I) { + segments.push_back(Segment(I->start, I->end, valnos[I->valno->id])); + } + } + /// advanceTo - Advance the specified iterator to point to the Segment /// containing the specified position, or end() if the position is past the /// end of the range. If no Segment contains this position, but the @@ -543,11 +562,106 @@ namespace llvm { public: typedef LiveRange super; + /// A live range for subregisters. The LaneMask specifies which parts of the + /// super register are covered by the interval. + /// (@sa TargetRegisterInfo::getSubRegIndexLaneMask()). + class SubRange : public LiveRange { + public: + SubRange *Next; + unsigned LaneMask; + + /// Constructs a new SubRange object. + SubRange(unsigned LaneMask) + : Next(nullptr), LaneMask(LaneMask) { + } + + /// Constructs a new SubRange object by copying liveness from @p Other. + SubRange(unsigned LaneMask, const LiveRange &Other, + BumpPtrAllocator &Allocator) + : LiveRange(Other, Allocator), Next(nullptr), LaneMask(LaneMask) { + } + }; + + private: + SubRange *SubRanges; ///< Single linked list of subregister live ranges. + + public: const unsigned reg; // the register or stack slot of this interval. float weight; // weight of this interval LiveInterval(unsigned Reg, float Weight) - : reg(Reg), weight(Weight) {} + : SubRanges(nullptr), reg(Reg), weight(Weight) {} + + template + class SingleLinkedListIterator { + T *P; + public: + SingleLinkedListIterator(T *P) : P(P) {} + SingleLinkedListIterator &operator++() { + P = P->Next; + return *this; + } + SingleLinkedListIterator &operator++(int) { + SingleLinkedListIterator res = *this; + ++*this; + return res; + } + bool operator!=(const SingleLinkedListIterator &Other) { + return P != Other.operator->(); + } + bool operator==(const SingleLinkedListIterator &Other) { + return P == Other.operator->(); + } + T &operator*() const { + return *P; + } + T *operator->() const { + return P; + } + }; + + typedef SingleLinkedListIterator subrange_iterator; + subrange_iterator subrange_begin() { + return subrange_iterator(SubRanges); + } + subrange_iterator subrange_end() { + return subrange_iterator(nullptr); + } + + typedef SingleLinkedListIterator const_subrange_iterator; + const_subrange_iterator subrange_begin() const { + return const_subrange_iterator(SubRanges); + } + const_subrange_iterator subrange_end() const { + return const_subrange_iterator(nullptr); + } + + /// Creates a new empty subregister live range. The range is added at the + /// beginning of the subrange list; subrange iterators stay valid. + SubRange *createSubRange(BumpPtrAllocator &Allocator, unsigned LaneMask) { + SubRange *Range = new (Allocator) SubRange(LaneMask); + appendSubRange(Range); + return Range; + } + + /// Like createSubRange() but the new range is filled with a copy of the + /// liveness information in @p CopyFrom. + SubRange *createSubRangeFrom(BumpPtrAllocator &Allocator, unsigned LaneMask, + const LiveRange &CopyFrom) { + SubRange *Range = new (Allocator) SubRange(LaneMask, CopyFrom, Allocator); + appendSubRange(Range); + return Range; + } + + /// Returns true if subregister liveness information is available. + bool hasSubRanges() const { + return SubRanges != nullptr; + } + + /// Removes all subregister liveness information. + void clearSubRanges() { + SubRanges = nullptr; + } /// getSize - Returns the sum of sizes of all the LiveRange's. /// @@ -572,9 +686,23 @@ namespace llvm { void print(raw_ostream &OS) const; void dump() const; + /// \brief Walks the interval and assert if any invariants fail to hold. + /// + /// Note that this is a no-op when asserts are disabled. +#ifdef NDEBUG + void verify(const MachineRegisterInfo *MRI = nullptr) const {} +#else + void verify(const MachineRegisterInfo *MRI = nullptr) const; +#endif + private: LiveInterval& operator=(const LiveInterval& rhs) LLVM_DELETED_FUNCTION; + /// Appends @p Range to SubRanges list. + void appendSubRange(SubRange *Range) { + Range->Next = SubRanges; + SubRanges = Range; + } }; inline raw_ostream &operator<<(raw_ostream &OS, const LiveInterval &LI) { diff --git a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h index 2e7f034be6b0..ba86853dc0d5 100644 --- a/llvm/include/llvm/CodeGen/MachineRegisterInfo.h +++ b/llvm/include/llvm/CodeGen/MachineRegisterInfo.h @@ -768,6 +768,10 @@ public: const TargetRegisterInfo &TRI, const TargetInstrInfo &TII); + /// Returns a mask covering all bits that can appear in lane masks of + /// subregisters of the virtual register @p Reg. + unsigned getMaxLaneMaskForVReg(unsigned Reg) const; + /// defusechain_iterator - This class provides iterator support for machine /// operands in the function that use or define a specific register. If /// ReturnUses is true it returns uses of registers, if ReturnDefs is true it diff --git a/llvm/lib/CodeGen/LiveInterval.cpp b/llvm/lib/CodeGen/LiveInterval.cpp index bacd61978087..5162579adf1e 100644 --- a/llvm/lib/CodeGen/LiveInterval.cpp +++ b/llvm/lib/CodeGen/LiveInterval.cpp @@ -26,6 +26,7 @@ #include "llvm/CodeGen/LiveIntervalAnalysis.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/Debug.h" +#include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetRegisterInfo.h" #include @@ -641,6 +642,11 @@ void LiveRange::print(raw_ostream &OS) const { void LiveInterval::print(raw_ostream &OS) const { OS << PrintReg(reg) << ' '; super::print(OS); + // Print subranges + for (const_subrange_iterator I = subrange_begin(), E = subrange_end(); + I != E; ++I) { + OS << format(" L%04X ", I->LaneMask) << *I; + } } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) @@ -669,6 +675,27 @@ void LiveRange::verify() const { } } } + +void LiveInterval::verify(const MachineRegisterInfo *MRI) const { + super::verify(); + + // Make sure SubRanges are fine and LaneMasks are disjunct. + unsigned Mask = 0; + unsigned MaxMask = MRI != nullptr ? MRI->getMaxLaneMaskForVReg(reg) : ~0u; + for (const_subrange_iterator I = subrange_begin(), E = subrange_end(); I != E; + ++I) { + // Subrange lanemask should be disjunct to any previous subrange masks. + assert((Mask & I->LaneMask) == 0); + Mask |= I->LaneMask; + + // subrange mask should not contained in maximum lane mask for the vreg. + assert((Mask & ~MaxMask) == 0); + + I->verify(); + // Main liverange should cover subrange. + assert(covers(*I)); + } +} #endif @@ -959,6 +986,8 @@ void ConnectedVNInfoEqClasses::Distribute(LiveInterval *LIV[], } else *J++ = *I; } + // TODO: do not cheat anymore by simply cleaning all subranges + LI.clearSubRanges(); LI.segments.erase(J, E); // Transfer VNInfos to their new owners and renumber them. diff --git a/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp b/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp index 1742e63b0dff..d07d3cc3a8a2 100644 --- a/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp +++ b/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp @@ -767,6 +767,8 @@ public: continue; if (TargetRegisterInfo::isVirtualRegister(Reg)) { LiveInterval &LI = LIS.getInterval(Reg); + // TODO: handle subranges instead of dropping them + LI.clearSubRanges(); updateRange(LI, Reg); continue; } diff --git a/llvm/lib/CodeGen/MachineRegisterInfo.cpp b/llvm/lib/CodeGen/MachineRegisterInfo.cpp index e9612f3cfde0..c2fc4a7de469 100644 --- a/llvm/lib/CodeGen/MachineRegisterInfo.cpp +++ b/llvm/lib/CodeGen/MachineRegisterInfo.cpp @@ -391,6 +391,14 @@ MachineRegisterInfo::EmitLiveInCopies(MachineBasicBlock *EntryMBB, } } +unsigned MachineRegisterInfo::getMaxLaneMaskForVReg(unsigned Reg) const +{ + // Lane masks are only defined for vregs. + assert(TargetRegisterInfo::isVirtualRegister(Reg)); + const TargetRegisterClass &TRC = *getRegClass(Reg); + return TRC.getLaneMask(); +} + #ifndef NDEBUG void MachineRegisterInfo::dumpUses(unsigned Reg) const { for (MachineInstr &I : use_instructions(Reg)) diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index cd19083f81ba..0586561ecc31 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -42,6 +42,7 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FileSystem.h" +#include "llvm/Support/Format.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetInstrInfo.h" #include "llvm/Target/TargetMachine.h" @@ -215,9 +216,9 @@ namespace { void report(const char *msg, const MachineBasicBlock *MBB, const LiveInterval &LI); void report(const char *msg, const MachineFunction *MF, - const LiveRange &LR, unsigned Reg); + const LiveRange &LR, unsigned Reg, unsigned LaneMask); void report(const char *msg, const MachineBasicBlock *MBB, - const LiveRange &LR, unsigned Reg); + const LiveRange &LR, unsigned Reg, unsigned LaneMask); void verifyInlineAsm(const MachineInstr *MI); @@ -230,10 +231,12 @@ namespace { void verifyLiveVariables(); void verifyLiveIntervals(); void verifyLiveInterval(const LiveInterval&); - void verifyLiveRangeValue(const LiveRange&, const VNInfo*, unsigned); + void verifyLiveRangeValue(const LiveRange&, const VNInfo*, unsigned, + unsigned); void verifyLiveRangeSegment(const LiveRange&, - const LiveRange::const_iterator I, unsigned); - void verifyLiveRange(const LiveRange&, unsigned); + const LiveRange::const_iterator I, unsigned, + unsigned); + void verifyLiveRange(const LiveRange&, unsigned, unsigned LaneMask = 0); void verifyStackFrame(); }; @@ -432,17 +435,23 @@ void MachineVerifier::report(const char *msg, const MachineBasicBlock *MBB, } void MachineVerifier::report(const char *msg, const MachineBasicBlock *MBB, - const LiveRange &LR, unsigned Reg) { + const LiveRange &LR, unsigned Reg, + unsigned LaneMask) { report(msg, MBB); *OS << "- liverange: " << LR << '\n'; *OS << "- register: " << PrintReg(Reg, TRI) << '\n'; + if (LaneMask != 0) + *OS << "- lanemask: " << format("%04X\n", LaneMask); } void MachineVerifier::report(const char *msg, const MachineFunction *MF, - const LiveRange &LR, unsigned Reg) { + const LiveRange &LR, unsigned Reg, + unsigned LaneMask) { report(msg, MF); *OS << "- liverange: " << LR << '\n'; *OS << "- register: " << PrintReg(Reg, TRI) << '\n'; + if (LaneMask != 0) + *OS << "- lanemask: " << format("%04X\n", LaneMask); } void MachineVerifier::markReachable(const MachineBasicBlock *MBB) { @@ -1358,21 +1367,22 @@ void MachineVerifier::verifyLiveIntervals() { } void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR, - const VNInfo *VNI, - unsigned Reg) { + const VNInfo *VNI, unsigned Reg, + unsigned LaneMask) { if (VNI->isUnused()) return; const VNInfo *DefVNI = LR.getVNInfoAt(VNI->def); if (!DefVNI) { - report("Valno not live at def and not marked unused", MF, LR, Reg); + report("Valno not live at def and not marked unused", MF, LR, Reg, + LaneMask); *OS << "Valno #" << VNI->id << '\n'; return; } if (DefVNI != VNI) { - report("Live segment at def has different valno", MF, LR, Reg); + report("Live segment at def has different valno", MF, LR, Reg, LaneMask); *OS << "Valno #" << VNI->id << " is defined at " << VNI->def << " where valno #" << DefVNI->id << " is live\n"; return; @@ -1380,7 +1390,7 @@ void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR, const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(VNI->def); if (!MBB) { - report("Invalid definition index", MF, LR, Reg); + report("Invalid definition index", MF, LR, Reg, LaneMask); *OS << "Valno #" << VNI->id << " is defined at " << VNI->def << " in " << LR << '\n'; return; @@ -1388,7 +1398,8 @@ void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR, if (VNI->isPHIDef()) { if (VNI->def != LiveInts->getMBBStartIdx(MBB)) { - report("PHIDef value is not defined at MBB start", MBB, LR, Reg); + report("PHIDef value is not defined at MBB start", MBB, LR, Reg, + LaneMask); *OS << "Valno #" << VNI->id << " is defined at " << VNI->def << ", not at the beginning of BB#" << MBB->getNumber() << '\n'; } @@ -1398,7 +1409,7 @@ void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR, // Non-PHI def. const MachineInstr *MI = LiveInts->getInstructionFromIndex(VNI->def); if (!MI) { - report("No instruction at def index", MBB, LR, Reg); + report("No instruction at def index", MBB, LR, Reg, LaneMask); *OS << "Valno #" << VNI->id << " is defined at " << VNI->def << '\n'; return; } @@ -1417,6 +1428,9 @@ void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR, !TRI->hasRegUnit(MOI->getReg(), Reg)) continue; } + if (LaneMask != 0 && + (TRI->getSubRegIndexLaneMask(MOI->getSubReg()) & LaneMask) == 0) + continue; hasDef = true; if (MOI->isEarlyClobber()) isEarlyClobber = true; @@ -1432,12 +1446,12 @@ void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR, if (isEarlyClobber) { if (!VNI->def.isEarlyClobber()) { report("Early clobber def must be at an early-clobber slot", MBB, LR, - Reg); + Reg, LaneMask); *OS << "Valno #" << VNI->id << " is defined at " << VNI->def << '\n'; } } else if (!VNI->def.isRegister()) { report("Non-PHI, non-early clobber def must be at a register slot", - MBB, LR, Reg); + MBB, LR, Reg, LaneMask); *OS << "Valno #" << VNI->id << " is defined at " << VNI->def << '\n'; } } @@ -1445,37 +1459,38 @@ void MachineVerifier::verifyLiveRangeValue(const LiveRange &LR, void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR, const LiveRange::const_iterator I, - unsigned Reg) { + unsigned Reg, unsigned LaneMask) { const LiveRange::Segment &S = *I; const VNInfo *VNI = S.valno; assert(VNI && "Live segment has no valno"); if (VNI->id >= LR.getNumValNums() || VNI != LR.getValNumInfo(VNI->id)) { - report("Foreign valno in live segment", MF, LR, Reg); + report("Foreign valno in live segment", MF, LR, Reg, LaneMask); *OS << S << " has a bad valno\n"; } if (VNI->isUnused()) { - report("Live segment valno is marked unused", MF, LR, Reg); + report("Live segment valno is marked unused", MF, LR, Reg, LaneMask); *OS << S << '\n'; } const MachineBasicBlock *MBB = LiveInts->getMBBFromIndex(S.start); if (!MBB) { - report("Bad start of live segment, no basic block", MF, LR, Reg); + report("Bad start of live segment, no basic block", MF, LR, Reg, LaneMask); *OS << S << '\n'; return; } SlotIndex MBBStartIdx = LiveInts->getMBBStartIdx(MBB); if (S.start != MBBStartIdx && S.start != VNI->def) { - report("Live segment must begin at MBB entry or valno def", MBB, LR, Reg); + report("Live segment must begin at MBB entry or valno def", MBB, LR, Reg, + LaneMask); *OS << S << '\n'; } const MachineBasicBlock *EndMBB = LiveInts->getMBBFromIndex(S.end.getPrevSlot()); if (!EndMBB) { - report("Bad end of live segment, no basic block", MF, LR, Reg); + report("Bad end of live segment, no basic block", MF, LR, Reg, LaneMask); *OS << S << '\n'; return; } @@ -1493,14 +1508,16 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR, const MachineInstr *MI = LiveInts->getInstructionFromIndex(S.end.getPrevSlot()); if (!MI) { - report("Live segment doesn't end at a valid instruction", EndMBB, LR, Reg); + report("Live segment doesn't end at a valid instruction", EndMBB, LR, Reg, + LaneMask); *OS << S << '\n'; return; } // The block slot must refer to a basic block boundary. if (S.end.isBlock()) { - report("Live segment ends at B slot of an instruction", EndMBB, LR, Reg); + report("Live segment ends at B slot of an instruction", EndMBB, LR, Reg, + LaneMask); *OS << S << '\n'; } @@ -1509,7 +1526,7 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR, // That means there must be a dead def. if (!SlotIndex::isSameInstr(S.start, S.end)) { report("Live segment ending at dead slot spans instructions", EndMBB, LR, - Reg); + Reg, LaneMask); *OS << S << '\n'; } } @@ -1519,7 +1536,8 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR, if (S.end.isEarlyClobber()) { if (I+1 == LR.end() || (I+1)->start != S.end) { report("Live segment ending at early clobber slot must be " - "redefined by an EC def in the same instruction", EndMBB, LR, Reg); + "redefined by an EC def in the same instruction", EndMBB, LR, Reg, + LaneMask); *OS << S << '\n'; } } @@ -1533,6 +1551,9 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR, for (ConstMIBundleOperands MOI(MI); MOI.isValid(); ++MOI) { if (!MOI->isReg() || MOI->getReg() != Reg) continue; + if (LaneMask != 0 && + (LaneMask & TRI->getSubRegIndexLaneMask(MOI->getSubReg())) == 0) + continue; if (MOI->readsReg()) hasRead = true; } @@ -1577,7 +1598,8 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR, // All predecessors must have a live-out value. if (!PVNI) { - report("Register not marked live out of predecessor", *PI, LR, Reg); + report("Register not marked live out of predecessor", *PI, LR, Reg, + LaneMask); *OS << "Valno #" << VNI->id << " live into BB#" << MFI->getNumber() << '@' << LiveInts->getMBBStartIdx(MFI) << ", not live before " << PEnd << '\n'; @@ -1586,7 +1608,8 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR, // Only PHI-defs can take different predecessor values. if (!IsPHI && PVNI != VNI) { - report("Different value live out of predecessor", *PI, LR, Reg); + report("Different value live out of predecessor", *PI, LR, Reg, + LaneMask); *OS << "Valno #" << PVNI->id << " live out of BB#" << (*PI)->getNumber() << '@' << PEnd << "\nValno #" << VNI->id << " live into BB#" << MFI->getNumber() @@ -1599,18 +1622,38 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR, } } -void MachineVerifier::verifyLiveRange(const LiveRange &LR, unsigned Reg) { +void MachineVerifier::verifyLiveRange(const LiveRange &LR, unsigned Reg, + unsigned LaneMask) { for (LiveRange::const_vni_iterator I = LR.vni_begin(), E = LR.vni_end(); I != E; ++I) - verifyLiveRangeValue(LR, *I, Reg); + verifyLiveRangeValue(LR, *I, Reg, LaneMask); for (LiveRange::const_iterator I = LR.begin(), E = LR.end(); I != E; ++I) - verifyLiveRangeSegment(LR, I, Reg); + verifyLiveRangeSegment(LR, I, Reg, LaneMask); } void MachineVerifier::verifyLiveInterval(const LiveInterval &LI) { verifyLiveRange(LI, LI.reg); + unsigned Reg = LI.reg; + if (TargetRegisterInfo::isVirtualRegister(Reg)) { + unsigned Mask = 0; + unsigned MaxMask = MRI->getMaxLaneMaskForVReg(Reg); + for (LiveInterval::const_subrange_iterator I = LI.subrange_begin(), + E = LI.subrange_end(); I != E; ++I) { + if ((Mask & I->LaneMask) != 0) + report("Lane masks of sub ranges overlap in live interval", MF, LI); + if ((I->LaneMask & ~MaxMask) != 0) + report("Subrange lanemask is invalid", MF, LI); + Mask |= I->LaneMask; + verifyLiveRange(*I, LI.reg, I->LaneMask); + if (!LI.covers(*I)) + report("A Subrange is not covered by the main range", MF, LI); + } + } else if (LI.hasSubRanges()) { + report("subregister liveness only allowed for virtual registers", MF, LI); + } + // Check the LI only has one connected component. if (TargetRegisterInfo::isVirtualRegister(LI.reg)) { ConnectedVNInfoEqClasses ConEQ(*LiveInts);