From 7bbcad7bcd9c2c5818a21a37d6d33ca791ece332 Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Wed, 5 Dec 2012 21:37:47 +0000 Subject: [PATCH] RegisterPressureTracker: unify virtual registers and physical regunits. Now that live register units are tracked individually, the code can be simplified. llvm-svn: 169426 --- .../llvm/CodeGen/LiveIntervalAnalysis.h | 4 + llvm/include/llvm/CodeGen/RegisterPressure.h | 63 ++- llvm/lib/CodeGen/RegisterPressure.cpp | 420 ++++++++---------- 3 files changed, 226 insertions(+), 261 deletions(-) diff --git a/llvm/include/llvm/CodeGen/LiveIntervalAnalysis.h b/llvm/include/llvm/CodeGen/LiveIntervalAnalysis.h index 367f6d0bbb92..9e89519b2875 100644 --- a/llvm/include/llvm/CodeGen/LiveIntervalAnalysis.h +++ b/llvm/include/llvm/CodeGen/LiveIntervalAnalysis.h @@ -347,6 +347,10 @@ namespace llvm { return RegUnitIntervals[Unit]; } + const LiveInterval *getCachedRegUnit(unsigned Unit) const { + return RegUnitIntervals[Unit]; + } + private: /// computeIntervals - Compute live intervals. void computeIntervals(); diff --git a/llvm/include/llvm/CodeGen/RegisterPressure.h b/llvm/include/llvm/CodeGen/RegisterPressure.h index edff782f7bbc..604953235aef 100644 --- a/llvm/include/llvm/CodeGen/RegisterPressure.h +++ b/llvm/include/llvm/CodeGen/RegisterPressure.h @@ -22,6 +22,7 @@ namespace llvm { class LiveIntervals; +class LiveInterval; class RegisterClassInfo; class MachineInstr; @@ -37,17 +38,17 @@ struct RegisterPressure { /// Increase register pressure for each pressure set impacted by this register /// class. Normally called by RegPressureTracker, but may be called manually /// to account for live through (global liveness). - void increase(const TargetRegisterClass *RC, const TargetRegisterInfo *TRI); - - /// Increase pressure for each pressure set impacted by this register unit. - void increase(unsigned RU, const TargetRegisterInfo *TRI); + /// + /// \param Reg is either a virtual register number or register unit number. + void increase(unsigned Reg, const TargetRegisterInfo *TRI, + const MachineRegisterInfo *MRI); /// Decrease register pressure for each pressure set impacted by this register /// class. This is only useful to account for spilling or rematerialization. - void decrease(const TargetRegisterClass *RC, const TargetRegisterInfo *TRI); - - /// Decrease pressure for each pressure set impacted by this register unit. - void decrease(unsigned RU, const TargetRegisterInfo *TRI); + /// + /// \param Reg is either a virtual register number or register unit number. + void decrease(unsigned Reg, const TargetRegisterInfo *TRI, + const MachineRegisterInfo *MRI); void dump(const TargetRegisterInfo *TRI) const; }; @@ -122,6 +123,33 @@ struct RegPressureDelta { RegPressureDelta() {} }; +/// \brief A set of live virtual registers and physical register units. +/// +/// Virtual and physical register numbers require separate sparse sets, but most +/// of the RegisterPressureTracker handles them uniformly. +struct LiveRegSet { + SparseSet PhysRegs; + SparseSet VirtRegs; + + bool contains(unsigned Reg) { + if (TargetRegisterInfo::isVirtualRegister(Reg)) + return VirtRegs.count(Reg); + return PhysRegs.count(Reg); + } + + bool insert(unsigned Reg) { + if (TargetRegisterInfo::isVirtualRegister(Reg)) + return VirtRegs.insert(Reg).second; + return PhysRegs.insert(Reg).second; + } + + bool erase(unsigned Reg) { + if (TargetRegisterInfo::isVirtualRegister(Reg)) + return VirtRegs.erase(Reg); + return PhysRegs.erase(Reg); + } +}; + /// Track the current register pressure at some position in the instruction /// stream, and remember the high water mark within the region traversed. This /// does not automatically consider live-through ranges. The client may @@ -163,9 +191,8 @@ class RegPressureTracker { /// Pressure map indexed by pressure set ID, not class ID. std::vector CurrSetPressure; - /// List of live registers. - SparseSet LivePhysRegs; - SparseSet LiveVirtRegs; + /// Set of live registers. + LiveRegSet LiveRegs; public: RegPressureTracker(IntervalPressure &rp) : @@ -215,11 +242,8 @@ public: /// than the pressure across the traversed region. std::vector &getRegSetPressureAtPos() { return CurrSetPressure; } - void discoverPhysLiveIn(unsigned Reg); - void discoverPhysLiveOut(unsigned Reg); - - void discoverVirtLiveIn(unsigned Reg); - void discoverVirtLiveOut(unsigned Reg); + void discoverLiveOut(unsigned Reg); + void discoverLiveIn(unsigned Reg); bool isTopClosed() const; bool isBottomClosed() const; @@ -283,11 +307,10 @@ public: void dump(const TargetRegisterInfo *TRI) const; protected: - void increasePhysRegPressure(ArrayRef Regs); - void decreasePhysRegPressure(ArrayRef Regs); + const LiveInterval *getInterval(unsigned Reg) const; - void increaseVirtRegPressure(ArrayRef Regs); - void decreaseVirtRegPressure(ArrayRef Regs); + void increaseRegPressure(ArrayRef Regs); + void decreaseRegPressure(ArrayRef Regs); void bumpUpwardPressure(const MachineInstr *MI); void bumpDownwardPressure(const MachineInstr *MI); diff --git a/llvm/lib/CodeGen/RegisterPressure.cpp b/llvm/lib/CodeGen/RegisterPressure.cpp index 4d56ad5ba707..1576b2abbe36 100644 --- a/llvm/lib/CodeGen/RegisterPressure.cpp +++ b/llvm/lib/CodeGen/RegisterPressure.cpp @@ -46,31 +46,33 @@ static void decreaseSetPressure(std::vector &CurrSetPressure, } /// Directly increase pressure only within this RegisterPressure result. -void RegisterPressure::increase(const TargetRegisterClass *RC, - const TargetRegisterInfo *TRI) { - increaseSetPressure(MaxSetPressure, MaxSetPressure, - TRI->getRegClassPressureSets(RC), - TRI->getRegClassWeight(RC).RegWeight); -} - -/// Directly increase pressure only within this RegisterPressure result. -void RegisterPressure::increase(unsigned RU, const TargetRegisterInfo *TRI) { - increaseSetPressure(MaxSetPressure, MaxSetPressure, - TRI->getRegUnitPressureSets(RU), - TRI->getRegUnitWeight(RU)); +void RegisterPressure::increase(unsigned Reg, const TargetRegisterInfo *TRI, + const MachineRegisterInfo *MRI) { + if (TargetRegisterInfo::isVirtualRegister(Reg)) { + const TargetRegisterClass *RC = MRI->getRegClass(Reg); + increaseSetPressure(MaxSetPressure, MaxSetPressure, + TRI->getRegClassPressureSets(RC), + TRI->getRegClassWeight(RC).RegWeight); + } + else { + increaseSetPressure(MaxSetPressure, MaxSetPressure, + TRI->getRegUnitPressureSets(Reg), + TRI->getRegUnitWeight(Reg)); + } } /// Directly decrease pressure only within this RegisterPressure result. -void RegisterPressure::decrease(const TargetRegisterClass *RC, - const TargetRegisterInfo *TRI) { - decreaseSetPressure(MaxSetPressure, TRI->getRegClassPressureSets(RC), - TRI->getRegClassWeight(RC).RegWeight); -} - -/// Directly decrease pressure only within this RegisterPressure result. -void RegisterPressure::decrease(unsigned RU, const TargetRegisterInfo *TRI) { - decreaseSetPressure(MaxSetPressure, TRI->getRegUnitPressureSets(RU), - TRI->getRegUnitWeight(RU)); +void RegisterPressure::decrease(unsigned Reg, const TargetRegisterInfo *TRI, + const MachineRegisterInfo *MRI) { + if (TargetRegisterInfo::isVirtualRegister(Reg)) { + const TargetRegisterClass *RC = MRI->getRegClass(Reg); + decreaseSetPressure(MaxSetPressure, TRI->getRegClassPressureSets(RC), + TRI->getRegClassWeight(RC).RegWeight); + } + else { + decreaseSetPressure(MaxSetPressure, TRI->getRegUnitPressureSets(Reg), + TRI->getRegUnitWeight(Reg)); + } } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) @@ -102,41 +104,37 @@ void RegPressureTracker::dump(const TargetRegisterInfo *TRI) const { } #endif -/// Increase the current pressure as impacted by these register units and bump +/// Increase the current pressure as impacted by these registers and bump /// the high water mark if needed. -void RegPressureTracker::increasePhysRegPressure(ArrayRef Regs) { - for (unsigned I = 0, E = Regs.size(); I != E; ++I) - increaseSetPressure(CurrSetPressure, P.MaxSetPressure, - TRI->getRegUnitPressureSets(Regs[I]), - TRI->getRegUnitWeight(Regs[I])); -} - -/// Simply decrease the current pressure as impacted by these physcial -/// registers. -void RegPressureTracker::decreasePhysRegPressure(ArrayRef Regs) { - for (unsigned I = 0, E = Regs.size(); I != E; ++I) - decreaseSetPressure(CurrSetPressure, TRI->getRegUnitPressureSets(Regs[I]), - TRI->getRegUnitWeight(Regs[I])); -} - -/// Increase the current pressure as impacted by these virtual registers and -/// bump the high water mark if needed. -void RegPressureTracker::increaseVirtRegPressure(ArrayRef Regs) { +void RegPressureTracker::increaseRegPressure(ArrayRef Regs) { for (unsigned I = 0, E = Regs.size(); I != E; ++I) { - const TargetRegisterClass *RC = MRI->getRegClass(Regs[I]); - increaseSetPressure(CurrSetPressure, P.MaxSetPressure, - TRI->getRegClassPressureSets(RC), - TRI->getRegClassWeight(RC).RegWeight); + if (TargetRegisterInfo::isVirtualRegister(Regs[I])) { + const TargetRegisterClass *RC = MRI->getRegClass(Regs[I]); + increaseSetPressure(CurrSetPressure, P.MaxSetPressure, + TRI->getRegClassPressureSets(RC), + TRI->getRegClassWeight(RC).RegWeight); + } + else { + increaseSetPressure(CurrSetPressure, P.MaxSetPressure, + TRI->getRegUnitPressureSets(Regs[I]), + TRI->getRegUnitWeight(Regs[I])); + } } } -/// Simply decrease the current pressure as impacted by these virtual registers. -void RegPressureTracker::decreaseVirtRegPressure(ArrayRef Regs) { +/// Simply decrease the current pressure as impacted by these registers. +void RegPressureTracker::decreaseRegPressure(ArrayRef Regs) { for (unsigned I = 0, E = Regs.size(); I != E; ++I) { - const TargetRegisterClass *RC = MRI->getRegClass(Regs[I]); - decreaseSetPressure(CurrSetPressure, - TRI->getRegClassPressureSets(RC), - TRI->getRegClassWeight(RC).RegWeight); + if (TargetRegisterInfo::isVirtualRegister(Regs[I])) { + const TargetRegisterClass *RC = MRI->getRegClass(Regs[I]); + decreaseSetPressure(CurrSetPressure, + TRI->getRegClassPressureSets(RC), + TRI->getRegClassWeight(RC).RegWeight); + } + else { + decreaseSetPressure(CurrSetPressure, TRI->getRegUnitPressureSets(Regs[I]), + TRI->getRegUnitWeight(Regs[I])); + } } } @@ -189,6 +187,12 @@ void RegionPressure::openBottom(MachineBasicBlock::const_iterator PrevBottom) { LiveInRegs.clear(); } +const LiveInterval *RegPressureTracker::getInterval(unsigned Reg) const { + if (TargetRegisterInfo::isVirtualRegister(Reg)) + return &LIS->getInterval(Reg); + return LIS->getCachedRegUnit(Reg); +} + /// Setup the RegPressureTracker. /// /// TODO: Add support for pressure without LiveIntervals. @@ -218,10 +222,10 @@ void RegPressureTracker::init(const MachineFunction *mf, static_cast(P).reset(); P.MaxSetPressure = CurrSetPressure; - LivePhysRegs.clear(); - LivePhysRegs.setUniverse(TRI->getNumRegs()); - LiveVirtRegs.clear(); - LiveVirtRegs.setUniverse(MRI->getNumVirtRegs()); + LiveRegs.PhysRegs.clear(); + LiveRegs.PhysRegs.setUniverse(TRI->getNumRegs()); + LiveRegs.VirtRegs.clear(); + LiveRegs.VirtRegs.setUniverse(MRI->getNumVirtRegs()); } /// Does this pressure result have a valid top position and live ins. @@ -258,10 +262,10 @@ void RegPressureTracker::closeTop() { static_cast(P).TopPos = CurrPos; assert(P.LiveInRegs.empty() && "inconsistent max pressure result"); - P.LiveInRegs.reserve(LivePhysRegs.size() + LiveVirtRegs.size()); - P.LiveInRegs.append(LivePhysRegs.begin(), LivePhysRegs.end()); + P.LiveInRegs.reserve(LiveRegs.PhysRegs.size() + LiveRegs.VirtRegs.size()); + P.LiveInRegs.append(LiveRegs.PhysRegs.begin(), LiveRegs.PhysRegs.end()); for (SparseSet::const_iterator I = - LiveVirtRegs.begin(), E = LiveVirtRegs.end(); I != E; ++I) + LiveRegs.VirtRegs.begin(), E = LiveRegs.VirtRegs.end(); I != E; ++I) P.LiveInRegs.push_back(*I); std::sort(P.LiveInRegs.begin(), P.LiveInRegs.end()); P.LiveInRegs.erase(std::unique(P.LiveInRegs.begin(), P.LiveInRegs.end()), @@ -276,10 +280,10 @@ void RegPressureTracker::closeBottom() { static_cast(P).BottomPos = CurrPos; assert(P.LiveOutRegs.empty() && "inconsistent max pressure result"); - P.LiveOutRegs.reserve(LivePhysRegs.size() + LiveVirtRegs.size()); - P.LiveOutRegs.append(LivePhysRegs.begin(), LivePhysRegs.end()); + P.LiveOutRegs.reserve(LiveRegs.PhysRegs.size() + LiveRegs.VirtRegs.size()); + P.LiveOutRegs.append(LiveRegs.PhysRegs.begin(), LiveRegs.PhysRegs.end()); for (SparseSet::const_iterator I = - LiveVirtRegs.begin(), E = LiveVirtRegs.end(); I != E; ++I) + LiveRegs.VirtRegs.begin(), E = LiveRegs.VirtRegs.end(); I != E; ++I) P.LiveOutRegs.push_back(*I); std::sort(P.LiveOutRegs.begin(), P.LiveOutRegs.end()); P.LiveOutRegs.erase(std::unique(P.LiveOutRegs.begin(), P.LiveOutRegs.end()), @@ -289,7 +293,7 @@ void RegPressureTracker::closeBottom() { /// Finalize the region boundaries and record live ins and live outs. void RegPressureTracker::closeRegion() { if (!isTopClosed() && !isBottomClosed()) { - assert(LivePhysRegs.empty() && LiveVirtRegs.empty() && + assert(LiveRegs.PhysRegs.empty() && LiveRegs.VirtRegs.empty() && "no region boundary"); return; } @@ -307,34 +311,40 @@ static bool containsReg(ArrayRef Regs, unsigned Reg) { /// Collect this instruction's unique uses and defs into SmallVectors for /// processing defs and uses in order. -template class RegisterOperands { + const TargetRegisterInfo *TRI; + const MachineRegisterInfo *MRI; + public: SmallVector Uses; SmallVector Defs; SmallVector DeadDefs; + RegisterOperands(const TargetRegisterInfo *tri, + const MachineRegisterInfo *mri): TRI(tri), MRI(mri) {} + /// Push this operand's register onto the correct vector. - void collect(const MachineOperand &MO, const TargetRegisterInfo *TRI) { + void collect(const MachineOperand &MO) { + if (!MO.isReg() || !MO.getReg()) + return; if (MO.readsReg()) - pushRegUnits(MO.getReg(), Uses, TRI); + pushRegUnits(MO.getReg(), Uses); if (MO.isDef()) { if (MO.isDead()) - pushRegUnits(MO.getReg(), DeadDefs, TRI); + pushRegUnits(MO.getReg(), DeadDefs); else - pushRegUnits(MO.getReg(), Defs, TRI); + pushRegUnits(MO.getReg(), Defs); } } protected: - void pushRegUnits(unsigned Reg, SmallVectorImpl &Regs, - const TargetRegisterInfo *TRI) { - if (isVReg) { + void pushRegUnits(unsigned Reg, SmallVectorImpl &Regs) { + if (TargetRegisterInfo::isVirtualRegister(Reg)) { if (containsReg(Regs, Reg)) return; Regs.push_back(Reg); } - else { + else if (MRI->isAllocatable(Reg)) { for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units) { if (containsReg(Regs, *Units)) continue; @@ -343,89 +353,49 @@ protected: } } }; -typedef RegisterOperands PhysRegOperands; -typedef RegisterOperands VirtRegOperands; /// Collect physical and virtual register operands. static void collectOperands(const MachineInstr *MI, - PhysRegOperands &PhysRegOpers, - VirtRegOperands &VirtRegOpers, - const TargetRegisterInfo *TRI, - const MachineRegisterInfo *MRI) { - for(ConstMIBundleOperands OperI(MI); OperI.isValid(); ++OperI) { - const MachineOperand &MO = *OperI; - if (!MO.isReg() || !MO.getReg()) - continue; + RegisterOperands &RegOpers) { + for(ConstMIBundleOperands OperI(MI); OperI.isValid(); ++OperI) + RegOpers.collect(*OperI); - if (TargetRegisterInfo::isVirtualRegister(MO.getReg())) - VirtRegOpers.collect(MO, TRI); - else if (MRI->isAllocatable(MO.getReg())) - PhysRegOpers.collect(MO, TRI); - } // Remove redundant physreg dead defs. - for (unsigned i = PhysRegOpers.DeadDefs.size(); i > 0; --i) { - unsigned Reg = PhysRegOpers.DeadDefs[i-1]; - if (containsReg(PhysRegOpers.Defs, Reg)) - PhysRegOpers.DeadDefs.erase(&PhysRegOpers.DeadDefs[i-1]); + for (unsigned i = RegOpers.DeadDefs.size(); i > 0; --i) { + unsigned Reg = RegOpers.DeadDefs[i-1]; + if (containsReg(RegOpers.Defs, Reg)) + RegOpers.DeadDefs.erase(&RegOpers.DeadDefs[i-1]); } } /// Force liveness of registers. void RegPressureTracker::addLiveRegs(ArrayRef Regs) { for (unsigned i = 0, e = Regs.size(); i != e; ++i) { - if (TargetRegisterInfo::isVirtualRegister(Regs[i])) { - if (LiveVirtRegs.insert(Regs[i]).second) - increaseVirtRegPressure(Regs[i]); - } - else { - if (LivePhysRegs.insert(Regs[i]).second) - increasePhysRegPressure(Regs[i]); - } + if (LiveRegs.insert(Regs[i])) + increaseRegPressure(Regs[i]); } } -/// Add PhysReg to the live in set and increase max pressure. -void RegPressureTracker::discoverPhysLiveIn(unsigned Reg) { - assert(!LivePhysRegs.count(Reg) && "avoid bumping max pressure twice"); +/// Add Reg to the live in set and increase max pressure. +void RegPressureTracker::discoverLiveIn(unsigned Reg) { + assert(!LiveRegs.contains(Reg) && "avoid bumping max pressure twice"); if (containsReg(P.LiveInRegs, Reg)) return; // At live in discovery, unconditionally increase the high water mark. P.LiveInRegs.push_back(Reg); - P.increase(Reg, TRI); + P.increase(Reg, TRI, MRI); } -/// Add PhysReg to the live out set and increase max pressure. -void RegPressureTracker::discoverPhysLiveOut(unsigned Reg) { - assert(!LivePhysRegs.count(Reg) && "avoid bumping max pressure twice"); +/// Add Reg to the live out set and increase max pressure. +void RegPressureTracker::discoverLiveOut(unsigned Reg) { + assert(!LiveRegs.contains(Reg) && "avoid bumping max pressure twice"); if (containsReg(P.LiveOutRegs, Reg)) return; // At live out discovery, unconditionally increase the high water mark. P.LiveOutRegs.push_back(Reg); - P.increase(Reg, TRI); -} - -/// Add VirtReg to the live in set and increase max pressure. -void RegPressureTracker::discoverVirtLiveIn(unsigned Reg) { - assert(!LiveVirtRegs.count(Reg) && "avoid bumping max pressure twice"); - if (containsReg(P.LiveInRegs, Reg)) - return; - - // At live in discovery, unconditionally increase the high water mark. - P.LiveInRegs.push_back(Reg); - P.increase(MRI->getRegClass(Reg), TRI); -} - -/// Add VirtReg to the live out set and increase max pressure. -void RegPressureTracker::discoverVirtLiveOut(unsigned Reg) { - assert(!LiveVirtRegs.count(Reg) && "avoid bumping max pressure twice"); - if (containsReg(P.LiveOutRegs, Reg)) - return; - - // At live out discovery, unconditionally increase the high water mark. - P.LiveOutRegs.push_back(Reg); - P.increase(MRI->getRegClass(Reg), TRI); + P.increase(Reg, TRI, MRI); } /// Recede across the previous instruction. @@ -459,50 +429,35 @@ bool RegPressureTracker::recede() { if (RequireIntervals && isTopClosed()) static_cast(P).openTop(SlotIdx); - PhysRegOperands PhysRegOpers; - VirtRegOperands VirtRegOpers; - collectOperands(CurrPos, PhysRegOpers, VirtRegOpers, TRI, MRI); + RegisterOperands RegOpers(TRI, MRI); + collectOperands(CurrPos, RegOpers); // Boost pressure for all dead defs together. - increasePhysRegPressure(PhysRegOpers.DeadDefs); - increaseVirtRegPressure(VirtRegOpers.DeadDefs); - decreasePhysRegPressure(PhysRegOpers.DeadDefs); - decreaseVirtRegPressure(VirtRegOpers.DeadDefs); + increaseRegPressure(RegOpers.DeadDefs); + decreaseRegPressure(RegOpers.DeadDefs); // Kill liveness at live defs. // TODO: consider earlyclobbers? - for (unsigned i = 0, e = PhysRegOpers.Defs.size(); i < e; ++i) { - unsigned Reg = PhysRegOpers.Defs[i]; - if (LivePhysRegs.erase(Reg)) - decreasePhysRegPressure(Reg); + for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) { + unsigned Reg = RegOpers.Defs[i]; + if (LiveRegs.erase(Reg)) + decreaseRegPressure(Reg); else - discoverPhysLiveOut(Reg); - } - for (unsigned i = 0, e = VirtRegOpers.Defs.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Defs[i]; - if (LiveVirtRegs.erase(Reg)) - decreaseVirtRegPressure(Reg); - else - discoverVirtLiveOut(Reg); + discoverLiveOut(Reg); } // Generate liveness for uses. - for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = PhysRegOpers.Uses[i]; - if (LivePhysRegs.insert(Reg).second) - increasePhysRegPressure(Reg); - } - for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Uses[i]; - if (!LiveVirtRegs.count(Reg)) { + for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) { + unsigned Reg = RegOpers.Uses[i]; + if (!LiveRegs.contains(Reg)) { // Adjust liveouts if LiveIntervals are available. if (RequireIntervals) { - const LiveInterval *LI = &LIS->getInterval(Reg); - if (!LI->killedAt(SlotIdx)) - discoverVirtLiveOut(Reg); + const LiveInterval *LI = getInterval(Reg); + if (LI && !LI->killedAt(SlotIdx)) + discoverLiveOut(Reg); } - increaseVirtRegPressure(Reg); - LiveVirtRegs.insert(Reg); + increaseRegPressure(Reg); + LiveRegs.insert(Reg); } } return true; @@ -530,53 +485,43 @@ bool RegPressureTracker::advance() { static_cast(P).openBottom(CurrPos); } - PhysRegOperands PhysRegOpers; - VirtRegOperands VirtRegOpers; - collectOperands(CurrPos, PhysRegOpers, VirtRegOpers, TRI, MRI); + RegisterOperands RegOpers(TRI, MRI); + collectOperands(CurrPos, RegOpers); - // Kill liveness at last uses. - for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = PhysRegOpers.Uses[i]; - // Allocatable physregs are always single-use before register rewriting. - if (LivePhysRegs.erase(Reg)) - decreasePhysRegPressure(Reg); - else - discoverPhysLiveIn(Reg); - } - for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Uses[i]; + for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) { + unsigned Reg = RegOpers.Uses[i]; + // Discover live-ins. + bool isLive = LiveRegs.contains(Reg); + if (!isLive) + discoverLiveIn(Reg); + // Kill liveness at last uses. + bool lastUse = false; if (RequireIntervals) { - const LiveInterval *LI = &LIS->getInterval(Reg); - if (LI->killedAt(SlotIdx)) { - if (LiveVirtRegs.erase(Reg)) - decreaseVirtRegPressure(Reg); - else - discoverVirtLiveIn(Reg); - } + const LiveInterval *LI = getInterval(Reg); + lastUse = LI && LI->killedAt(SlotIdx); } - else if (!LiveVirtRegs.count(Reg)) { - discoverVirtLiveIn(Reg); - increaseVirtRegPressure(Reg); + else { + // Allocatable physregs are always single-use before register rewriting. + lastUse = !TargetRegisterInfo::isVirtualRegister(Reg); } + if (lastUse && isLive) { + LiveRegs.erase(Reg); + decreaseRegPressure(Reg); + } + else if (!lastUse && !isLive) + increaseRegPressure(Reg); } // Generate liveness for defs. - for (unsigned i = 0, e = PhysRegOpers.Defs.size(); i < e; ++i) { - unsigned Reg = PhysRegOpers.Defs[i]; - if (LivePhysRegs.insert(Reg).second) - increasePhysRegPressure(Reg); - } - for (unsigned i = 0, e = VirtRegOpers.Defs.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Defs[i]; - if (LiveVirtRegs.insert(Reg).second) - increaseVirtRegPressure(Reg); + for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) { + unsigned Reg = RegOpers.Defs[i]; + if (LiveRegs.insert(Reg)) + increaseRegPressure(Reg); } // Boost pressure for all dead defs together. - increasePhysRegPressure(PhysRegOpers.DeadDefs); - increaseVirtRegPressure(VirtRegOpers.DeadDefs); - decreasePhysRegPressure(PhysRegOpers.DeadDefs); - decreaseVirtRegPressure(VirtRegOpers.DeadDefs); + increaseRegPressure(RegOpers.DeadDefs); + decreaseRegPressure(RegOpers.DeadDefs); // Find the next instruction. do @@ -667,39 +612,28 @@ static void computeMaxPressureDelta(ArrayRef OldMaxPressureVec, /// This is intended for speculative queries. It leaves pressure inconsistent /// with the current position, so must be restored by the caller. void RegPressureTracker::bumpUpwardPressure(const MachineInstr *MI) { + assert(!MI->isDebugValue() && "Expect a nondebug instruction."); + // Account for register pressure similar to RegPressureTracker::recede(). - PhysRegOperands PhysRegOpers; - VirtRegOperands VirtRegOpers; - collectOperands(MI, PhysRegOpers, VirtRegOpers, TRI, MRI); + RegisterOperands RegOpers(TRI, MRI); + collectOperands(MI, RegOpers); // Boost max pressure for all dead defs together. // Since CurrSetPressure and MaxSetPressure - increasePhysRegPressure(PhysRegOpers.DeadDefs); - increaseVirtRegPressure(VirtRegOpers.DeadDefs); - decreasePhysRegPressure(PhysRegOpers.DeadDefs); - decreaseVirtRegPressure(VirtRegOpers.DeadDefs); + increaseRegPressure(RegOpers.DeadDefs); + decreaseRegPressure(RegOpers.DeadDefs); // Kill liveness at live defs. - for (unsigned i = 0, e = PhysRegOpers.Defs.size(); i < e; ++i) { - unsigned Reg = PhysRegOpers.Defs[i]; - if (!containsReg(PhysRegOpers.Uses, Reg)) - decreasePhysRegPressure(Reg); - } - for (unsigned i = 0, e = VirtRegOpers.Defs.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Defs[i]; - if (!containsReg(VirtRegOpers.Uses, Reg)) - decreaseVirtRegPressure(Reg); + for (unsigned i = 0, e = RegOpers.Defs.size(); i < e; ++i) { + unsigned Reg = RegOpers.Defs[i]; + if (!containsReg(RegOpers.Uses, Reg)) + decreaseRegPressure(Reg); } // Generate liveness for uses. - for (unsigned i = 0, e = PhysRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = PhysRegOpers.Uses[i]; - if (!LivePhysRegs.count(Reg)) - increasePhysRegPressure(Reg); - } - for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Uses[i]; - if (!LiveVirtRegs.count(Reg)) - increaseVirtRegPressure(Reg); + for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) { + unsigned Reg = RegOpers.Uses[i]; + if (!LiveRegs.contains(Reg)) + increaseRegPressure(Reg); } } @@ -760,38 +694,42 @@ static bool findUseBetween(unsigned Reg, /// This is intended for speculative queries. It leaves pressure inconsistent /// with the current position, so must be restored by the caller. void RegPressureTracker::bumpDownwardPressure(const MachineInstr *MI) { + assert(!MI->isDebugValue() && "Expect a nondebug instruction."); + // Account for register pressure similar to RegPressureTracker::recede(). - PhysRegOperands PhysRegOpers; - VirtRegOperands VirtRegOpers; - collectOperands(MI, PhysRegOpers, VirtRegOpers, TRI, MRI); + RegisterOperands RegOpers(TRI, MRI); + collectOperands(MI, RegOpers); // Kill liveness at last uses. Assume allocatable physregs are single-use // rather than checking LiveIntervals. - decreasePhysRegPressure(PhysRegOpers.Uses); - if (RequireIntervals) { - SlotIndex SlotIdx = LIS->getInstructionIndex(MI).getRegSlot(); - for (unsigned i = 0, e = VirtRegOpers.Uses.size(); i < e; ++i) { - unsigned Reg = VirtRegOpers.Uses[i]; - const LiveInterval *LI = &LIS->getInterval(Reg); - // FIXME: allow the caller to pass in the list of vreg uses that remain to - // be bottom-scheduled to avoid searching uses at each query. + SlotIndex SlotIdx; + if (RequireIntervals) + SlotIdx = LIS->getInstructionIndex(MI).getRegSlot(); + + for (unsigned i = 0, e = RegOpers.Uses.size(); i < e; ++i) { + unsigned Reg = RegOpers.Uses[i]; + if (RequireIntervals) { + // FIXME: allow the caller to pass in the list of vreg uses that remain + // to be bottom-scheduled to avoid searching uses at each query. SlotIndex CurrIdx = getCurrSlot(); - if (LI->killedAt(SlotIdx) + const LiveInterval *LI = getInterval(Reg); + if (LI && LI->killedAt(SlotIdx) && !findUseBetween(Reg, CurrIdx, SlotIdx, MRI, LIS)) { - decreaseVirtRegPressure(Reg); + decreaseRegPressure(Reg); } } + else if (!TargetRegisterInfo::isVirtualRegister(Reg)) { + // Allocatable physregs are always single-use before register rewriting. + decreaseRegPressure(Reg); + } } // Generate liveness for defs. - increasePhysRegPressure(PhysRegOpers.Defs); - increaseVirtRegPressure(VirtRegOpers.Defs); + increaseRegPressure(RegOpers.Defs); // Boost pressure for all dead defs together. - increasePhysRegPressure(PhysRegOpers.DeadDefs); - increaseVirtRegPressure(VirtRegOpers.DeadDefs); - decreasePhysRegPressure(PhysRegOpers.DeadDefs); - decreaseVirtRegPressure(VirtRegOpers.DeadDefs); + increaseRegPressure(RegOpers.DeadDefs); + decreaseRegPressure(RegOpers.DeadDefs); } /// Consider the pressure increase caused by traversing this instruction