From d7f890edb092483e6aae69ae9221ea1206d4f86c Mon Sep 17 00:00:00 2001 From: Andrew Trick Date: Sat, 28 Dec 2013 21:56:47 +0000 Subject: [PATCH] Factor MI-Sched in preparation for post-ra scheduling support. Factor the MachineFunctionPass into MachineSchedulerBase. Split the DAG class into ScheduleDAGMI and SchedulerDAGMILive. llvm-svn: 198119 --- llvm/include/llvm/CodeGen/MachineScheduler.h | 188 ++++--- llvm/lib/CodeGen/MachineScheduler.cpp | 494 +++++++++++------- .../Hexagon/HexagonMachineScheduler.cpp | 17 +- .../Target/Hexagon/HexagonMachineScheduler.h | 26 +- llvm/lib/Target/R600/AMDGPUTargetMachine.cpp | 2 +- llvm/lib/Target/R600/R600MachineScheduler.cpp | 7 +- llvm/lib/Target/R600/R600MachineScheduler.h | 2 +- 7 files changed, 450 insertions(+), 286 deletions(-) diff --git a/llvm/include/llvm/CodeGen/MachineScheduler.h b/llvm/include/llvm/CodeGen/MachineScheduler.h index 920abff28c18..4c7f3d94d310 100644 --- a/llvm/include/llvm/CodeGen/MachineScheduler.h +++ b/llvm/include/llvm/CodeGen/MachineScheduler.h @@ -23,7 +23,7 @@ // return new CustomMachineScheduler(C); // } // -// The default scheduler, ScheduleDAGMI, builds the DAG and drives list +// The default scheduler, ScheduleDAGMILive, builds the DAG and drives list // scheduling while updating the instruction stream, register pressure, and live // intervals. Most targets don't need to override the DAG builder and list // schedulier, but subtargets that require custom scheduling heuristics may @@ -155,8 +155,8 @@ struct MachineSchedPolicy { bool OnlyTopDown; bool OnlyBottomUp; - MachineSchedPolicy(): - ShouldTrackPressure(false), OnlyTopDown(false), OnlyBottomUp(false) {} + MachineSchedPolicy(): ShouldTrackPressure(false), OnlyTopDown(false), + OnlyBottomUp(false) {} }; /// MachineSchedStrategy - Interface to the scheduling algorithm used by @@ -214,19 +214,15 @@ public: virtual void apply(ScheduleDAGMI *DAG) = 0; }; -/// ScheduleDAGMI is an implementation of ScheduleDAGInstrs that schedules -/// machine instructions while updating LiveIntervals and tracking regpressure. +/// ScheduleDAGMI is an implementation of ScheduleDAGInstrs that simply +/// schedules machine instructions according to the given MachineSchedStrategy +/// without much extra book-keeping. This is the common functionality between +/// PreRA and PostRA MachineScheduler. class ScheduleDAGMI : public ScheduleDAGInstrs { protected: AliasAnalysis *AA; - RegisterClassInfo *RegClassInfo; MachineSchedStrategy *SchedImpl; - /// Information about DAG subtrees. If DFSResult is NULL, then SchedulerTrees - /// will be empty. - SchedDFSResult *DFSResult; - BitVector ScheduledTrees; - /// Topo - A topological ordering for SUnits which permits fast IsReachable /// and similar queries. ScheduleDAGTopologicalSort Topo; @@ -234,32 +230,11 @@ protected: /// Ordered list of DAG postprocessing steps. std::vector Mutations; - MachineBasicBlock::iterator LiveRegionEnd; - - // Map each SU to its summary of pressure changes. This array is updated for - // liveness during bottom-up scheduling. Top-down scheduling may proceed but - // has no affect on the pressure diffs. - PressureDiffs SUPressureDiffs; - - /// Register pressure in this region computed by initRegPressure. - bool ShouldTrackPressure; - IntervalPressure RegPressure; - RegPressureTracker RPTracker; - - /// List of pressure sets that exceed the target's pressure limit before - /// scheduling, listed in increasing set ID order. Each pressure set is paired - /// with its max pressure in the currently scheduled regions. - std::vector RegionCriticalPSets; - /// The top of the unscheduled zone. MachineBasicBlock::iterator CurrentTop; - IntervalPressure TopPressure; - RegPressureTracker TopRPTracker; /// The bottom of the unscheduled zone. MachineBasicBlock::iterator CurrentBottom; - IntervalPressure BotPressure; - RegPressureTracker BotRPTracker; /// Record the next node in a scheduled cluster. const SUnit *NextClusterPred; @@ -270,14 +245,11 @@ protected: /// scheduler at the point determined by misched-cutoff. unsigned NumInstrsScheduled; #endif - public: - ScheduleDAGMI(MachineSchedContext *C, MachineSchedStrategy *S): - ScheduleDAGInstrs(*C->MF, *C->MLI, *C->MDT, /*IsPostRA=*/false, C->LIS), - AA(C->AA), RegClassInfo(C->RegClassInfo), SchedImpl(S), DFSResult(0), - Topo(SUnits, &ExitSU), ShouldTrackPressure(false), - RPTracker(RegPressure), CurrentTop(), TopRPTracker(TopPressure), - CurrentBottom(), BotRPTracker(BotPressure), + ScheduleDAGMI(MachineSchedContext *C, MachineSchedStrategy *S, + bool IsPostRA): + ScheduleDAGInstrs(*C->MF, *C->MLI, *C->MDT, IsPostRA, C->LIS), AA(C->AA), + SchedImpl(S), Topo(SUnits, &ExitSU), CurrentTop(), CurrentBottom(), NextClusterPred(NULL), NextClusterSucc(NULL) { #ifndef NDEBUG NumInstrsScheduled = 0; @@ -286,8 +258,8 @@ public: virtual ~ScheduleDAGMI(); - /// \brief Return true if register pressure tracking is enabled. - bool isTrackingPressure() const { return ShouldTrackPressure; } + /// Return true if this DAG supports VReg liveness and RegPressure. + virtual bool hasVRegLiveness() const { return false; } /// Add a postprocessing step to the DAG builder. /// Mutations are applied in the order that they are added after normal DAG @@ -328,6 +300,95 @@ public: /// live ranges and region boundary iterators. void moveInstruction(MachineInstr *MI, MachineBasicBlock::iterator InsertPos); + const SUnit *getNextClusterPred() const { return NextClusterPred; } + + const SUnit *getNextClusterSucc() const { return NextClusterSucc; } + + void viewGraph(const Twine &Name, const Twine &Title) LLVM_OVERRIDE; + void viewGraph() LLVM_OVERRIDE; + +protected: + // Top-Level entry points for the schedule() driver... + + /// Apply each ScheduleDAGMutation step in order. This allows different + /// instances of ScheduleDAGMI to perform custom DAG postprocessing. + void postprocessDAG(); + + /// Release ExitSU predecessors and setup scheduler queues. + void initQueues(ArrayRef TopRoots, ArrayRef BotRoots); + + /// Update scheduler DAG and queues after scheduling an instruction. + void updateQueues(SUnit *SU, bool IsTopNode); + + /// Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues. + void placeDebugValues(); + + /// \brief dump the scheduled Sequence. + void dumpSchedule() const; + + // Lesser helpers... + bool checkSchedLimit(); + + void findRootsAndBiasEdges(SmallVectorImpl &TopRoots, + SmallVectorImpl &BotRoots); + + void releaseSucc(SUnit *SU, SDep *SuccEdge); + void releaseSuccessors(SUnit *SU); + void releasePred(SUnit *SU, SDep *PredEdge); + void releasePredecessors(SUnit *SU); +}; + +/// ScheduleDAGMILive is an implementation of ScheduleDAGInstrs that schedules +/// machine instructions while updating LiveIntervals and tracking regpressure. +class ScheduleDAGMILive : public ScheduleDAGMI { +protected: + RegisterClassInfo *RegClassInfo; + + /// Information about DAG subtrees. If DFSResult is NULL, then SchedulerTrees + /// will be empty. + SchedDFSResult *DFSResult; + BitVector ScheduledTrees; + + MachineBasicBlock::iterator LiveRegionEnd; + + // Map each SU to its summary of pressure changes. This array is updated for + // liveness during bottom-up scheduling. Top-down scheduling may proceed but + // has no affect on the pressure diffs. + PressureDiffs SUPressureDiffs; + + /// Register pressure in this region computed by initRegPressure. + bool ShouldTrackPressure; + IntervalPressure RegPressure; + RegPressureTracker RPTracker; + + /// List of pressure sets that exceed the target's pressure limit before + /// scheduling, listed in increasing set ID order. Each pressure set is paired + /// with its max pressure in the currently scheduled regions. + std::vector RegionCriticalPSets; + + /// The top of the unscheduled zone. + IntervalPressure TopPressure; + RegPressureTracker TopRPTracker; + + /// The bottom of the unscheduled zone. + IntervalPressure BotPressure; + RegPressureTracker BotRPTracker; + +public: + ScheduleDAGMILive(MachineSchedContext *C, MachineSchedStrategy *S): + ScheduleDAGMI(C, S, /*IsPostRA=*/false), RegClassInfo(C->RegClassInfo), + DFSResult(0), ShouldTrackPressure(false), RPTracker(RegPressure), + TopRPTracker(TopPressure), BotRPTracker(BotPressure) + {} + + virtual ~ScheduleDAGMILive(); + + /// Return true if this DAG supports VReg liveness and RegPressure. + virtual bool hasVRegLiveness() const { return true; } + + /// \brief Return true if register pressure tracking is enabled. + bool isTrackingPressure() const { return ShouldTrackPressure; } + /// Get current register pressure for the top scheduled instructions. const IntervalPressure &getTopPressure() const { return TopPressure; } const RegPressureTracker &getTopRPTracker() const { return TopRPTracker; } @@ -347,10 +408,6 @@ public: return SUPressureDiffs[SU->NodeNum]; } - const SUnit *getNextClusterPred() const { return NextClusterPred; } - - const SUnit *getNextClusterSucc() const { return NextClusterSucc; } - /// Compute a DFSResult after DAG building is complete, and before any /// queue comparisons. void computeDFSResult(); @@ -360,12 +417,21 @@ public: BitVector &getScheduledTrees() { return ScheduledTrees; } + /// Implement the ScheduleDAGInstrs interface for handling the next scheduling + /// region. This covers all instructions in a block, while schedule() may only + /// cover a subset. + void enterRegion(MachineBasicBlock *bb, + MachineBasicBlock::iterator begin, + MachineBasicBlock::iterator end, + unsigned regioninstrs) LLVM_OVERRIDE; + + /// Implement ScheduleDAGInstrs interface for scheduling a sequence of + /// reorderable instructions. + virtual void schedule(); + /// Compute the cyclic critical path through the DAG. unsigned computeCyclicCriticalPath(); - void viewGraph(const Twine &Name, const Twine &Title) LLVM_OVERRIDE; - void viewGraph() LLVM_OVERRIDE; - protected: // Top-Level entry points for the schedule() driver... @@ -375,25 +441,9 @@ protected: /// bottom of the DAG region without covereing any unscheduled instruction. void buildDAGWithRegPressure(); - /// Apply each ScheduleDAGMutation step in order. This allows different - /// instances of ScheduleDAGMI to perform custom DAG postprocessing. - void postprocessDAG(); - - /// Release ExitSU predecessors and setup scheduler queues. - void initQueues(ArrayRef TopRoots, ArrayRef BotRoots); - /// Move an instruction and update register pressure. void scheduleMI(SUnit *SU, bool IsTopNode); - /// Update scheduler DAG and queues after scheduling an instruction. - void updateQueues(SUnit *SU, bool IsTopNode); - - /// Reinsert debug_values recorded in ScheduleDAGInstrs::DbgValues. - void placeDebugValues(); - - /// \brief dump the scheduled Sequence. - void dumpSchedule() const; - // Lesser helpers... void initRegPressure(); @@ -402,16 +452,6 @@ protected: void updateScheduledPressure(const SUnit *SU, const std::vector &NewMaxPressure); - - bool checkSchedLimit(); - - void findRootsAndBiasEdges(SmallVectorImpl &TopRoots, - SmallVectorImpl &BotRoots); - - void releaseSucc(SUnit *SU, SDep *SuccEdge); - void releaseSuccessors(SUnit *SU); - void releasePred(SUnit *SU, SDep *PredEdge); - void releasePredecessors(SUnit *SU); }; //===----------------------------------------------------------------------===// diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp index c228de2b5fb8..dd0b4c8a83cb 100644 --- a/llvm/lib/CodeGen/MachineScheduler.cpp +++ b/llvm/lib/CodeGen/MachineScheduler.cpp @@ -90,20 +90,27 @@ MachineSchedContext::~MachineSchedContext() { } namespace { +/// Base class for a machine scheduler class that can run at any point. +class MachineSchedulerBase : public MachineSchedContext, + public MachineFunctionPass { +public: + MachineSchedulerBase(char &ID): MachineFunctionPass(ID) {} + + virtual void print(raw_ostream &O, const Module* = 0) const; + +protected: + void scheduleRegions(ScheduleDAGInstrs &Scheduler); +}; + /// MachineScheduler runs after coalescing and before register allocation. -class MachineScheduler : public MachineSchedContext, - public MachineFunctionPass { +class MachineScheduler : public MachineSchedulerBase { public: MachineScheduler(); virtual void getAnalysisUsage(AnalysisUsage &AU) const; - virtual void releaseMemory() {} - virtual bool runOnMachineFunction(MachineFunction&); - virtual void print(raw_ostream &O, const Module* = 0) const; - static char ID; // Class identification, replacement for typeinfo protected: @@ -124,7 +131,7 @@ INITIALIZE_PASS_END(MachineScheduler, "misched", "Machine Instruction Scheduler", false, false) MachineScheduler::MachineScheduler() -: MachineFunctionPass(ID) { +: MachineSchedulerBase(ID) { initializeMachineSchedulerPass(*PassRegistry::getPassRegistry()); } @@ -163,7 +170,7 @@ DefaultSchedRegistry("default", "Use the target's default scheduler choice.", /// Forward declare the standard machine scheduler. This will be used as the /// default scheduler if the target does not set a default. static ScheduleDAGInstrs *createGenericSched(MachineSchedContext *C); - +static ScheduleDAGInstrs *createRawGenericSched(MachineSchedContext *C); /// Decrement this iterator until reaching the top or a non-debug instr. static MachineBasicBlock::const_iterator @@ -252,7 +259,6 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) { AA = &getAnalysis(); LIS = &getAnalysis(); - const TargetInstrInfo *TII = MF->getTarget().getInstrInfo(); if (VerifyScheduling) { DEBUG(LIS->dump()); @@ -263,6 +269,17 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) { // Instantiate the selected scheduler for this target, function, and // optimization level. OwningPtr Scheduler(createMachineScheduler()); + scheduleRegions(*Scheduler); + + DEBUG(LIS->dump()); + if (VerifyScheduling) + MF->verify(this, "After machine scheduling."); + return true; +} + +/// Main driver for both MachineScheduler and PostMachineScheduler. +void MachineSchedulerBase::scheduleRegions(ScheduleDAGInstrs &Scheduler) { + const TargetInstrInfo *TII = MF->getTarget().getInstrInfo(); // Visit all machine basic blocks. // @@ -271,7 +288,7 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) { for (MachineFunction::iterator MBB = MF->begin(), MBBEnd = MF->end(); MBB != MBBEnd; ++MBB) { - Scheduler->startBlock(MBB); + Scheduler.startBlock(MBB); // Break the block into scheduling regions [I, RegionEnd), and schedule each // region as soon as it is discovered. RegionEnd points the scheduling @@ -285,7 +302,7 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) { // 'RegionEnd' are invalid across these calls. unsigned RemainingInstrs = MBB->size(); for(MachineBasicBlock::iterator RegionEnd = MBB->end(); - RegionEnd != MBB->begin(); RegionEnd = Scheduler->begin()) { + RegionEnd != MBB->begin(); RegionEnd = Scheduler.begin()) { // Avoid decrementing RegionEnd for blocks with no terminator. if (RegionEnd != MBB->end() @@ -305,13 +322,13 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) { } // Notify the scheduler of the region, even if we may skip scheduling // it. Perhaps it still needs to be bundled. - Scheduler->enterRegion(MBB, I, RegionEnd, NumRegionInstrs); + Scheduler.enterRegion(MBB, I, RegionEnd, NumRegionInstrs); // Skip empty scheduling regions (0 or 1 schedulable instructions). if (I == RegionEnd || I == llvm::prior(RegionEnd)) { // Close the current region. Bundle the terminator if needed. // This invalidates 'RegionEnd' and 'I'. - Scheduler->exitRegion(); + Scheduler.exitRegion(); continue; } DEBUG(dbgs() << "********** MI Scheduling **********\n"); @@ -325,26 +342,22 @@ bool MachineScheduler::runOnMachineFunction(MachineFunction &mf) { // Schedule a region: possibly reorder instructions. // This invalidates 'RegionEnd' and 'I'. - Scheduler->schedule(); + Scheduler.schedule(); // Close the current region. - Scheduler->exitRegion(); + Scheduler.exitRegion(); // Scheduling has invalidated the current iterator 'I'. Ask the // scheduler for the top of it's scheduled region. - RegionEnd = Scheduler->begin(); + RegionEnd = Scheduler.begin(); } assert(RemainingInstrs == 0 && "Instruction count mismatch!"); - Scheduler->finishBlock(); + Scheduler.finishBlock(); } - Scheduler->finalizeSchedule(); - DEBUG(LIS->dump()); - if (VerifyScheduling) - MF->verify(this, "After machine scheduling."); - return true; + Scheduler.finalizeSchedule(); } -void MachineScheduler::print(raw_ostream &O, const Module* m) const { +void MachineSchedulerBase::print(raw_ostream &O, const Module* m) const { // unimplemented } @@ -358,12 +371,12 @@ void ReadyQueue::dump() { #endif //===----------------------------------------------------------------------===// -// ScheduleDAGMI - Base class for MachineInstr scheduling with LiveIntervals -// preservation. -//===----------------------------------------------------------------------===// +// ScheduleDAGMI - Basic machine instruction scheduling. This is +// independent of PreRA/PostRA scheduling and involves no extra book-keeping for +// virtual registers. +// ===----------------------------------------------------------------------===/ ScheduleDAGMI::~ScheduleDAGMI() { - delete DFSResult; DeleteContainerPointers(Mutations); delete SchedImpl; } @@ -453,10 +466,24 @@ void ScheduleDAGMI::releasePredecessors(SUnit *SU) { } } +/// enterRegion - Called back from MachineScheduler::runOnMachineFunction after +/// crossing a scheduling boundary. [begin, end) includes all instructions in +/// the region, including the boundary itself and single-instruction regions +/// that don't get scheduled. +void ScheduleDAGMI::enterRegion(MachineBasicBlock *bb, + MachineBasicBlock::iterator begin, + MachineBasicBlock::iterator end, + unsigned regioninstrs) +{ + ScheduleDAGInstrs::enterRegion(bb, begin, end, regioninstrs); + + SchedImpl->initPolicy(begin, end, regioninstrs); +} + /// This is normally called from the main scheduler loop but may also be invoked /// by the scheduling strategy to perform additional code motion. -void ScheduleDAGMI::moveInstruction(MachineInstr *MI, - MachineBasicBlock::iterator InsertPos) { +void ScheduleDAGMI::moveInstruction( + MachineInstr *MI, MachineBasicBlock::iterator InsertPos) { // Advance RegionBegin if the first instruction moves down. if (&*RegionBegin == MI) ++RegionBegin; @@ -465,7 +492,8 @@ void ScheduleDAGMI::moveInstruction(MachineInstr *MI, BB->splice(InsertPos, BB, MI); // Update LiveIntervals - LIS->handleMove(MI, /*UpdateFlags=*/true); + if (LIS) + LIS->handleMove(MI, /*UpdateFlags=*/true); // Recede RegionBegin if an instruction moves above the first. if (RegionBegin == InsertPos) @@ -483,16 +511,200 @@ bool ScheduleDAGMI::checkSchedLimit() { return true; } +/// Per-region scheduling driver, called back from +/// MachineScheduler::runOnMachineFunction. This is a simplified driver that +/// does not consider liveness or register pressure. It is useful for PostRA +/// scheduling and potentially other custom schedulers. +void ScheduleDAGMI::schedule() { + // Build the DAG. + buildSchedGraph(AA); + + Topo.InitDAGTopologicalSorting(); + + postprocessDAG(); + + SmallVector TopRoots, BotRoots; + findRootsAndBiasEdges(TopRoots, BotRoots); + + // Initialize the strategy before modifying the DAG. + // This may initialize a DFSResult to be used for queue priority. + SchedImpl->initialize(this); + + DEBUG(for (unsigned su = 0, e = SUnits.size(); su != e; ++su) + SUnits[su].dumpAll(this)); + if (ViewMISchedDAGs) viewGraph(); + + // Initialize ready queues now that the DAG and priority data are finalized. + initQueues(TopRoots, BotRoots); + + bool IsTopNode = false; + while (SUnit *SU = SchedImpl->pickNode(IsTopNode)) { + assert(!SU->isScheduled && "Node already scheduled"); + if (!checkSchedLimit()) + break; + + MachineInstr *MI = SU->getInstr(); + if (IsTopNode) { + assert(SU->isTopReady() && "node still has unscheduled dependencies"); + if (&*CurrentTop == MI) + CurrentTop = nextIfDebug(++CurrentTop, CurrentBottom); + else + moveInstruction(MI, CurrentTop); + } + else { + assert(SU->isBottomReady() && "node still has unscheduled dependencies"); + MachineBasicBlock::iterator priorII = + priorNonDebug(CurrentBottom, CurrentTop); + if (&*priorII == MI) + CurrentBottom = priorII; + else { + if (&*CurrentTop == MI) + CurrentTop = nextIfDebug(++CurrentTop, priorII); + moveInstruction(MI, CurrentBottom); + CurrentBottom = MI; + } + } + updateQueues(SU, IsTopNode); + + // Notify the scheduling strategy after updating the DAG. + SchedImpl->schedNode(SU, IsTopNode); + } + assert(CurrentTop == CurrentBottom && "Nonempty unscheduled zone."); + + placeDebugValues(); + + DEBUG({ + unsigned BBNum = begin()->getParent()->getNumber(); + dbgs() << "*** Final schedule for BB#" << BBNum << " ***\n"; + dumpSchedule(); + dbgs() << '\n'; + }); +} + +/// Apply each ScheduleDAGMutation step in order. +void ScheduleDAGMI::postprocessDAG() { + for (unsigned i = 0, e = Mutations.size(); i < e; ++i) { + Mutations[i]->apply(this); + } +} + +void ScheduleDAGMI:: +findRootsAndBiasEdges(SmallVectorImpl &TopRoots, + SmallVectorImpl &BotRoots) { + for (std::vector::iterator + I = SUnits.begin(), E = SUnits.end(); I != E; ++I) { + SUnit *SU = &(*I); + assert(!SU->isBoundaryNode() && "Boundary node should not be in SUnits"); + + // Order predecessors so DFSResult follows the critical path. + SU->biasCriticalPath(); + + // A SUnit is ready to top schedule if it has no predecessors. + if (!I->NumPredsLeft) + TopRoots.push_back(SU); + // A SUnit is ready to bottom schedule if it has no successors. + if (!I->NumSuccsLeft) + BotRoots.push_back(SU); + } + ExitSU.biasCriticalPath(); +} + +/// Identify DAG roots and setup scheduler queues. +void ScheduleDAGMI::initQueues(ArrayRef TopRoots, + ArrayRef BotRoots) { + NextClusterSucc = NULL; + NextClusterPred = NULL; + + // Release all DAG roots for scheduling, not including EntrySU/ExitSU. + // + // Nodes with unreleased weak edges can still be roots. + // Release top roots in forward order. + for (SmallVectorImpl::const_iterator + I = TopRoots.begin(), E = TopRoots.end(); I != E; ++I) { + SchedImpl->releaseTopNode(*I); + } + // Release bottom roots in reverse order so the higher priority nodes appear + // first. This is more natural and slightly more efficient. + for (SmallVectorImpl::const_reverse_iterator + I = BotRoots.rbegin(), E = BotRoots.rend(); I != E; ++I) { + SchedImpl->releaseBottomNode(*I); + } + + releaseSuccessors(&EntrySU); + releasePredecessors(&ExitSU); + + SchedImpl->registerRoots(); + + // Advance past initial DebugValues. + CurrentTop = nextIfDebug(RegionBegin, RegionEnd); + CurrentBottom = RegionEnd; +} + +/// Update scheduler queues after scheduling an instruction. +void ScheduleDAGMI::updateQueues(SUnit *SU, bool IsTopNode) { + // Release dependent instructions for scheduling. + if (IsTopNode) + releaseSuccessors(SU); + else + releasePredecessors(SU); + + SU->isScheduled = true; +} + +/// Reinsert any remaining debug_values, just like the PostRA scheduler. +void ScheduleDAGMI::placeDebugValues() { + // If first instruction was a DBG_VALUE then put it back. + if (FirstDbgValue) { + BB->splice(RegionBegin, BB, FirstDbgValue); + RegionBegin = FirstDbgValue; + } + + for (std::vector >::iterator + DI = DbgValues.end(), DE = DbgValues.begin(); DI != DE; --DI) { + std::pair P = *prior(DI); + MachineInstr *DbgValue = P.first; + MachineBasicBlock::iterator OrigPrevMI = P.second; + if (&*RegionBegin == DbgValue) + ++RegionBegin; + BB->splice(++OrigPrevMI, BB, DbgValue); + if (OrigPrevMI == llvm::prior(RegionEnd)) + RegionEnd = DbgValue; + } + DbgValues.clear(); + FirstDbgValue = NULL; +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +void ScheduleDAGMI::dumpSchedule() const { + for (MachineBasicBlock::iterator MI = begin(), ME = end(); MI != ME; ++MI) { + if (SUnit *SU = getSUnit(&(*MI))) + SU->dump(this); + else + dbgs() << "Missing SUnit\n"; + } +} +#endif + +//===----------------------------------------------------------------------===// +// ScheduleDAGMILive - Base class for MachineInstr scheduling with LiveIntervals +// preservation. +//===----------------------------------------------------------------------===// + +ScheduleDAGMILive::~ScheduleDAGMILive() { + delete DFSResult; +} + /// enterRegion - Called back from MachineScheduler::runOnMachineFunction after /// crossing a scheduling boundary. [begin, end) includes all instructions in /// the region, including the boundary itself and single-instruction regions /// that don't get scheduled. -void ScheduleDAGMI::enterRegion(MachineBasicBlock *bb, +void ScheduleDAGMILive::enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, MachineBasicBlock::iterator end, unsigned regioninstrs) { - ScheduleDAGInstrs::enterRegion(bb, begin, end, regioninstrs); + // ScheduleDAGMI initializes SchedImpl's per-region policy. + ScheduleDAGMI::enterRegion(bb, begin, end, regioninstrs); // For convenience remember the end of the liveness region. LiveRegionEnd = @@ -500,14 +712,12 @@ void ScheduleDAGMI::enterRegion(MachineBasicBlock *bb, SUPressureDiffs.clear(); - SchedImpl->initPolicy(begin, end, regioninstrs); - ShouldTrackPressure = SchedImpl->shouldTrackPressure(); } // Setup the register pressure trackers for the top scheduled top and bottom // scheduled regions. -void ScheduleDAGMI::initRegPressure() { +void ScheduleDAGMILive::initRegPressure() { TopRPTracker.init(&MF, RegClassInfo, LIS, BB, RegionBegin); BotRPTracker.init(&MF, RegClassInfo, LIS, BB, LiveRegionEnd); @@ -567,7 +777,7 @@ void ScheduleDAGMI::initRegPressure() { dbgs() << "\n"); } -void ScheduleDAGMI:: +void ScheduleDAGMILive:: updateScheduledPressure(const SUnit *SU, const std::vector &NewMaxPressure) { const PressureDiff &PDiff = getPressureDiff(SU); @@ -595,7 +805,7 @@ updateScheduledPressure(const SUnit *SU, /// Update the PressureDiff array for liveness after scheduling this /// instruction. -void ScheduleDAGMI::updatePressureDiffs(ArrayRef LiveUses) { +void ScheduleDAGMILive::updatePressureDiffs(ArrayRef LiveUses) { for (unsigned LUIdx = 0, LUEnd = LiveUses.size(); LUIdx != LUEnd; ++LUIdx) { /// FIXME: Currently assuming single-use physregs. unsigned Reg = LiveUses[LUIdx]; @@ -644,9 +854,9 @@ void ScheduleDAGMI::updatePressureDiffs(ArrayRef LiveUses) { /// so that it can be easilly extended by experimental schedulers. Generally, /// implementing MachineSchedStrategy should be sufficient to implement a new /// scheduling algorithm. However, if a scheduler further subclasses -/// ScheduleDAGMI then it will want to override this virtual method in order to -/// update any specialized state. -void ScheduleDAGMI::schedule() { +/// ScheduleDAGMILive then it will want to override this virtual method in order +/// to update any specialized state. +void ScheduleDAGMILive::schedule() { buildDAGWithRegPressure(); Topo.InitDAGTopologicalSorting(); @@ -667,6 +877,11 @@ void ScheduleDAGMI::schedule() { // Initialize ready queues now that the DAG and priority data are finalized. initQueues(TopRoots, BotRoots); + if (ShouldTrackPressure) { + assert(TopRPTracker.getPos() == RegionBegin && "bad initial Top tracker"); + TopRPTracker.setPos(CurrentTop); + } + bool IsTopNode = false; while (SUnit *SU = SchedImpl->pickNode(IsTopNode)) { assert(!SU->isScheduled && "Node already scheduled"); @@ -676,6 +891,18 @@ void ScheduleDAGMI::schedule() { scheduleMI(SU, IsTopNode); updateQueues(SU, IsTopNode); + + if (DFSResult) { + unsigned SubtreeID = DFSResult->getSubtreeID(SU); + if (!ScheduledTrees.test(SubtreeID)) { + ScheduledTrees.set(SubtreeID); + DFSResult->scheduleTree(SubtreeID); + SchedImpl->scheduleTree(SubtreeID); + } + } + + // Notify the scheduling strategy after updating the DAG. + SchedImpl->schedNode(SU, IsTopNode); } assert(CurrentTop == CurrentBottom && "Nonempty unscheduled zone."); @@ -690,7 +917,7 @@ void ScheduleDAGMI::schedule() { } /// Build the DAG and setup three register pressure trackers. -void ScheduleDAGMI::buildDAGWithRegPressure() { +void ScheduleDAGMILive::buildDAGWithRegPressure() { if (!ShouldTrackPressure) { RPTracker.reset(); RegionCriticalPSets.clear(); @@ -713,14 +940,7 @@ void ScheduleDAGMI::buildDAGWithRegPressure() { initRegPressure(); } -/// Apply each ScheduleDAGMutation step in order. -void ScheduleDAGMI::postprocessDAG() { - for (unsigned i = 0, e = Mutations.size(); i < e; ++i) { - Mutations[i]->apply(this); - } -} - -void ScheduleDAGMI::computeDFSResult() { +void ScheduleDAGMILive::computeDFSResult() { if (!DFSResult) DFSResult = new SchedDFSResult(/*BottomU*/true, MinSubtreeSize); DFSResult->clear(); @@ -730,26 +950,6 @@ void ScheduleDAGMI::computeDFSResult() { ScheduledTrees.resize(DFSResult->getNumSubtrees()); } -void ScheduleDAGMI::findRootsAndBiasEdges(SmallVectorImpl &TopRoots, - SmallVectorImpl &BotRoots) { - for (std::vector::iterator - I = SUnits.begin(), E = SUnits.end(); I != E; ++I) { - SUnit *SU = &(*I); - assert(!SU->isBoundaryNode() && "Boundary node should not be in SUnits"); - - // Order predecessors so DFSResult follows the critical path. - SU->biasCriticalPath(); - - // A SUnit is ready to top schedule if it has no predecessors. - if (!I->NumPredsLeft) - TopRoots.push_back(SU); - // A SUnit is ready to bottom schedule if it has no successors. - if (!I->NumSuccsLeft) - BotRoots.push_back(SU); - } - ExitSU.biasCriticalPath(); -} - /// Compute the max cyclic critical path through the DAG. The scheduling DAG /// only provides the critical path for single block loops. To handle loops that /// span blocks, we could use the vreg path latencies provided by @@ -773,7 +973,10 @@ void ScheduleDAGMI::findRootsAndBiasEdges(SmallVectorImpl &TopRoots, /// LiveOutDepth - LiveInDepth = 3 - 1 = 2 /// LiveInHeight - LiveOutHeight = 4 - 2 = 2 /// CyclicCriticalPath = min(2, 2) = 2 -unsigned ScheduleDAGMI::computeCyclicCriticalPath() { +/// +/// This could be relevant to PostRA scheduling, but is currently implemented +/// assuming LiveIntervals. +unsigned ScheduleDAGMILive::computeCyclicCriticalPath() { // This only applies to single block loop. if (!BB->isSuccessor(BB)) return 0; @@ -835,44 +1038,8 @@ unsigned ScheduleDAGMI::computeCyclicCriticalPath() { return MaxCyclicLatency; } -/// Identify DAG roots and setup scheduler queues. -void ScheduleDAGMI::initQueues(ArrayRef TopRoots, - ArrayRef BotRoots) { - NextClusterSucc = NULL; - NextClusterPred = NULL; - - // Release all DAG roots for scheduling, not including EntrySU/ExitSU. - // - // Nodes with unreleased weak edges can still be roots. - // Release top roots in forward order. - for (SmallVectorImpl::const_iterator - I = TopRoots.begin(), E = TopRoots.end(); I != E; ++I) { - SchedImpl->releaseTopNode(*I); - } - // Release bottom roots in reverse order so the higher priority nodes appear - // first. This is more natural and slightly more efficient. - for (SmallVectorImpl::const_reverse_iterator - I = BotRoots.rbegin(), E = BotRoots.rend(); I != E; ++I) { - SchedImpl->releaseBottomNode(*I); - } - - releaseSuccessors(&EntrySU); - releasePredecessors(&ExitSU); - - SchedImpl->registerRoots(); - - // Advance past initial DebugValues. - CurrentTop = nextIfDebug(RegionBegin, RegionEnd); - CurrentBottom = RegionEnd; - - if (ShouldTrackPressure) { - assert(TopRPTracker.getPos() == RegionBegin && "bad initial Top tracker"); - TopRPTracker.setPos(CurrentTop); - } -} - /// Move an instruction and update register pressure. -void ScheduleDAGMI::scheduleMI(SUnit *SU, bool IsTopNode) { +void ScheduleDAGMILive::scheduleMI(SUnit *SU, bool IsTopNode) { // Move the instruction to its new location in the instruction stream. MachineInstr *MI = SU->getInstr(); @@ -917,63 +1084,6 @@ void ScheduleDAGMI::scheduleMI(SUnit *SU, bool IsTopNode) { } } -/// Update scheduler queues after scheduling an instruction. -void ScheduleDAGMI::updateQueues(SUnit *SU, bool IsTopNode) { - // Release dependent instructions for scheduling. - if (IsTopNode) - releaseSuccessors(SU); - else - releasePredecessors(SU); - - SU->isScheduled = true; - - if (DFSResult) { - unsigned SubtreeID = DFSResult->getSubtreeID(SU); - if (!ScheduledTrees.test(SubtreeID)) { - ScheduledTrees.set(SubtreeID); - DFSResult->scheduleTree(SubtreeID); - SchedImpl->scheduleTree(SubtreeID); - } - } - - // Notify the scheduling strategy after updating the DAG. - SchedImpl->schedNode(SU, IsTopNode); -} - -/// Reinsert any remaining debug_values, just like the PostRA scheduler. -void ScheduleDAGMI::placeDebugValues() { - // If first instruction was a DBG_VALUE then put it back. - if (FirstDbgValue) { - BB->splice(RegionBegin, BB, FirstDbgValue); - RegionBegin = FirstDbgValue; - } - - for (std::vector >::iterator - DI = DbgValues.end(), DE = DbgValues.begin(); DI != DE; --DI) { - std::pair P = *prior(DI); - MachineInstr *DbgValue = P.first; - MachineBasicBlock::iterator OrigPrevMI = P.second; - if (&*RegionBegin == DbgValue) - ++RegionBegin; - BB->splice(++OrigPrevMI, BB, DbgValue); - if (OrigPrevMI == llvm::prior(RegionEnd)) - RegionEnd = DbgValue; - } - DbgValues.clear(); - FirstDbgValue = NULL; -} - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -void ScheduleDAGMI::dumpSchedule() const { - for (MachineBasicBlock::iterator MI = begin(), ME = end(); MI != ME; ++MI) { - if (SUnit *SU = getSUnit(&(*MI))) - SU->dump(this); - else - dbgs() << "Missing SUnit\n"; - } -} -#endif - //===----------------------------------------------------------------------===// // LoadClusterMutation - DAG post-processing to cluster loads. //===----------------------------------------------------------------------===// @@ -1154,7 +1264,7 @@ public: virtual void apply(ScheduleDAGMI *DAG); protected: - void constrainLocalCopy(SUnit *CopySU, ScheduleDAGMI *DAG); + void constrainLocalCopy(SUnit *CopySU, ScheduleDAGMILive *DAG); }; } // anonymous @@ -1177,7 +1287,7 @@ protected: /// this algorithm should handle extended blocks. An EBB is a set of /// contiguously numbered blocks such that the previous block in the EBB is /// always the single predecessor. -void CopyConstrain::constrainLocalCopy(SUnit *CopySU, ScheduleDAGMI *DAG) { +void CopyConstrain::constrainLocalCopy(SUnit *CopySU, ScheduleDAGMILive *DAG) { LiveIntervals *LIS = DAG->getLIS(); MachineInstr *Copy = CopySU->getInstr(); @@ -1302,6 +1412,8 @@ void CopyConstrain::constrainLocalCopy(SUnit *CopySU, ScheduleDAGMI *DAG) { /// \brief Callback from DAG postProcessing to create weak edges to encourage /// copy elimination. void CopyConstrain::apply(ScheduleDAGMI *DAG) { + assert(DAG->hasVRegLiveness() && "Expect VRegs with LiveIntervals"); + MachineBasicBlock::iterator FirstPos = nextIfDebug(DAG->begin(), DAG->end()); if (FirstPos == DAG->end()) return; @@ -1314,7 +1426,7 @@ void CopyConstrain::apply(ScheduleDAGMI *DAG) { if (!SU->getInstr()->isCopy()) continue; - constrainLocalCopy(SU, DAG); + constrainLocalCopy(SU, static_cast(DAG)); } } @@ -1948,13 +2060,13 @@ public: bool isRepeat(CandReason R) { return RepeatReasonSet & (1 << R); } void setRepeat(CandReason R) { RepeatReasonSet |= (1 << R); } - void initResourceDelta(const ScheduleDAGMI *DAG, + void initResourceDelta(const ScheduleDAGMILive *DAG, const TargetSchedModel *SchedModel); }; private: const MachineSchedContext *Context; - ScheduleDAGMI *DAG; + ScheduleDAGMILive *DAG; const TargetSchedModel *SchedModel; const TargetRegisterInfo *TRI; @@ -2014,7 +2126,9 @@ protected: } // namespace void GenericScheduler::initialize(ScheduleDAGMI *dag) { - DAG = dag; + assert(dag->hasVRegLiveness() && + "(PreRA)GenericScheduler needs vreg liveness"); + DAG = static_cast(dag); SchedModel = DAG->getSchedModel(); TRI = DAG->TRI; @@ -2201,7 +2315,7 @@ void GenericScheduler::setPolicy(CandPolicy &Policy, SchedBoundary &CurrZone, } void GenericScheduler::SchedCandidate:: -initResourceDelta(const ScheduleDAGMI *DAG, +initResourceDelta(const ScheduleDAGMILive *DAG, const TargetSchedModel *SchedModel) { if (!Policy.ReduceResIdx && !Policy.DemandResIdx) return; @@ -2720,7 +2834,7 @@ void GenericScheduler::reschedulePhysRegCopies(SUnit *SU, bool isTop) { } /// Update the scheduler's state after scheduling a node. This is the same node -/// that was just returned by pickNode(). However, ScheduleDAGMI needs to update +/// that was just returned by pickNode(). However, ScheduleDAGMILive needs to update /// it's state based on the current cycle before MachineSchedStrategy does. /// /// FIXME: Eventually, we may bundle physreg copies rather than rescheduling @@ -2740,10 +2854,16 @@ void GenericScheduler::schedNode(SUnit *SU, bool IsTopNode) { } } +/// Create a generic scheduler with no DAG mutation passes. +static ScheduleDAGInstrs *createRawGenericSched(MachineSchedContext *C) { + return new ScheduleDAGMILive(C, new GenericScheduler(C)); +} + /// Create the standard converging machine scheduler. This will be used as the /// default scheduler if the target does not set a default. static ScheduleDAGInstrs *createGenericSched(MachineSchedContext *C) { - ScheduleDAGMI *DAG = new ScheduleDAGMI(C, new GenericScheduler(C)); + ScheduleDAGMILive *DAG = + static_cast(createRawGenericSched(C)); // Register DAG post-processors. // // FIXME: extend the mutation API to allow earlier mutations to instantiate @@ -2800,7 +2920,7 @@ struct ILPOrder { /// \brief Schedule based on the ILP metric. class ILPScheduler : public MachineSchedStrategy { - ScheduleDAGMI *DAG; + ScheduleDAGMILive *DAG; ILPOrder Cmp; std::vector ReadyQ; @@ -2808,7 +2928,8 @@ public: ILPScheduler(bool MaximizeILP): DAG(0), Cmp(MaximizeILP) {} virtual void initialize(ScheduleDAGMI *dag) { - DAG = dag; + assert(dag->hasVRegLiveness() && "ILPScheduler needs vreg liveness"); + DAG = static_cast(dag); DAG->computeDFSResult(); Cmp.DFSResult = DAG->getDFSResult(); Cmp.ScheduledTrees = &DAG->getScheduledTrees(); @@ -2860,10 +2981,10 @@ public: } // namespace static ScheduleDAGInstrs *createILPMaxScheduler(MachineSchedContext *C) { - return new ScheduleDAGMI(C, new ILPScheduler(true)); + return new ScheduleDAGMILive(C, new ILPScheduler(true)); } static ScheduleDAGInstrs *createILPMinScheduler(MachineSchedContext *C) { - return new ScheduleDAGMI(C, new ILPScheduler(false)); + return new ScheduleDAGMILive(C, new ILPScheduler(false)); } static MachineSchedRegistry ILPMaxRegistry( "ilpmax", "Schedule bottom-up for max ILP", createILPMaxScheduler); @@ -2905,7 +3026,7 @@ public: InstructionShuffler(bool alternate, bool topdown) : IsAlternating(alternate), IsTopDown(topdown) {} - virtual void initialize(ScheduleDAGMI *) { + virtual void initialize(ScheduleDAGMI*) { TopQ.clear(); BottomQ.clear(); } @@ -2952,7 +3073,7 @@ static ScheduleDAGInstrs *createInstructionShuffler(MachineSchedContext *C) { bool TopDown = !ForceBottomUp; assert((TopDown || !ForceTopDown) && "-misched-topdown incompatible with -misched-bottomup"); - return new ScheduleDAGMI(C, new InstructionShuffler(Alternate, TopDown)); + return new ScheduleDAGMILive(C, new InstructionShuffler(Alternate, TopDown)); } static MachineSchedRegistry ShufflerRegistry( "shuffle", "Shuffle machine instructions alternating directions", @@ -2960,7 +3081,7 @@ static MachineSchedRegistry ShufflerRegistry( #endif // !NDEBUG //===----------------------------------------------------------------------===// -// GraphWriter support for ScheduleDAGMI. +// GraphWriter support for ScheduleDAGMILive. //===----------------------------------------------------------------------===// #ifndef NDEBUG @@ -3006,8 +3127,9 @@ struct DOTGraphTraits : public DefaultDOTGraphTraits { static std::string getNodeLabel(const SUnit *SU, const ScheduleDAG *G) { std::string Str; raw_string_ostream SS(Str); - const SchedDFSResult *DFS = - static_cast(G)->getDFSResult(); + const ScheduleDAGMI *DAG = static_cast(G); + const SchedDFSResult *DFS = DAG->hasVRegLiveness() ? + static_cast(G)->getDFSResult() : 0; SS << "SU:" << SU->NodeNum; if (DFS) SS << " I:" << DFS->getNumInstrs(SU); @@ -3017,11 +3139,11 @@ struct DOTGraphTraits : public DefaultDOTGraphTraits { return G->getGraphNodeLabel(SU); } - static std::string getNodeAttributes(const SUnit *N, - const ScheduleDAG *Graph) { + static std::string getNodeAttributes(const SUnit *N, const ScheduleDAG *G) { std::string Str("shape=Mrecord"); - const SchedDFSResult *DFS = - static_cast(Graph)->getDFSResult(); + const ScheduleDAGMI *DAG = static_cast(G); + const SchedDFSResult *DFS = DAG->hasVRegLiveness() ? + static_cast(G)->getDFSResult() : 0; if (DFS) { Str += ",style=filled,fillcolor=\"#"; Str += DOT::getColorString(DFS->getSubtreeID(N)); diff --git a/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp b/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp index c94f081ab13b..98aeabb800be 100644 --- a/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp +++ b/llvm/lib/Target/Hexagon/HexagonMachineScheduler.cpp @@ -186,6 +186,9 @@ void VLIWMachineScheduler::schedule() { scheduleMI(SU, IsTopNode); updateQueues(SU, IsTopNode); + + // Notify the scheduling strategy after updating the DAG. + SchedImpl->schedNode(SU, IsTopNode); } assert(CurrentTop == CurrentBottom && "Nonempty unscheduled zone."); @@ -266,7 +269,7 @@ void ConvergingVLIWScheduler::releaseBottomNode(SUnit *SU) { /// can dispatch per cycle. /// /// TODO: Also check whether the SU must start a new group. -bool ConvergingVLIWScheduler::SchedBoundary::checkHazard(SUnit *SU) { +bool ConvergingVLIWScheduler::VLIWSchedBoundary::checkHazard(SUnit *SU) { if (HazardRec->isEnabled()) return HazardRec->getHazardType(SU) != ScheduleHazardRecognizer::NoHazard; @@ -277,7 +280,7 @@ bool ConvergingVLIWScheduler::SchedBoundary::checkHazard(SUnit *SU) { return false; } -void ConvergingVLIWScheduler::SchedBoundary::releaseNode(SUnit *SU, +void ConvergingVLIWScheduler::VLIWSchedBoundary::releaseNode(SUnit *SU, unsigned ReadyCycle) { if (ReadyCycle < MinReadyCycle) MinReadyCycle = ReadyCycle; @@ -292,7 +295,7 @@ void ConvergingVLIWScheduler::SchedBoundary::releaseNode(SUnit *SU, } /// Move the boundary of scheduled code by one cycle. -void ConvergingVLIWScheduler::SchedBoundary::bumpCycle() { +void ConvergingVLIWScheduler::VLIWSchedBoundary::bumpCycle() { unsigned Width = SchedModel->getIssueWidth(); IssueCount = (IssueCount <= Width) ? 0 : IssueCount - Width; @@ -318,7 +321,7 @@ void ConvergingVLIWScheduler::SchedBoundary::bumpCycle() { } /// Move the boundary of scheduled code by one SUnit. -void ConvergingVLIWScheduler::SchedBoundary::bumpNode(SUnit *SU) { +void ConvergingVLIWScheduler::VLIWSchedBoundary::bumpNode(SUnit *SU) { bool startNewCycle = false; // Update the reservation table. @@ -348,7 +351,7 @@ void ConvergingVLIWScheduler::SchedBoundary::bumpNode(SUnit *SU) { /// Release pending ready nodes in to the available queue. This makes them /// visible to heuristics. -void ConvergingVLIWScheduler::SchedBoundary::releasePending() { +void ConvergingVLIWScheduler::VLIWSchedBoundary::releasePending() { // If the available queue is empty, it is safe to reset MinReadyCycle. if (Available.empty()) MinReadyCycle = UINT_MAX; @@ -376,7 +379,7 @@ void ConvergingVLIWScheduler::SchedBoundary::releasePending() { } /// Remove SU from the ready set for this boundary. -void ConvergingVLIWScheduler::SchedBoundary::removeReady(SUnit *SU) { +void ConvergingVLIWScheduler::VLIWSchedBoundary::removeReady(SUnit *SU) { if (Available.isInQueue(SU)) Available.remove(Available.find(SU)); else { @@ -388,7 +391,7 @@ void ConvergingVLIWScheduler::SchedBoundary::removeReady(SUnit *SU) { /// If this queue only has one ready candidate, return it. As a side effect, /// advance the cycle until at least one node is ready. If multiple instructions /// are ready, return NULL. -SUnit *ConvergingVLIWScheduler::SchedBoundary::pickOnlyChoice() { +SUnit *ConvergingVLIWScheduler::VLIWSchedBoundary::pickOnlyChoice() { if (CheckPending) releasePending(); diff --git a/llvm/lib/Target/Hexagon/HexagonMachineScheduler.h b/llvm/lib/Target/Hexagon/HexagonMachineScheduler.h index 8ac333fa7db3..8106a205a490 100644 --- a/llvm/lib/Target/Hexagon/HexagonMachineScheduler.h +++ b/llvm/lib/Target/Hexagon/HexagonMachineScheduler.h @@ -92,14 +92,14 @@ VLIWResourceModel(const TargetMachine &TM, const TargetSchedModel *SM) : /// Extend the standard ScheduleDAGMI to provide more context and override the /// top-level schedule() driver. -class VLIWMachineScheduler : public ScheduleDAGMI { +class VLIWMachineScheduler : public ScheduleDAGMILive { public: VLIWMachineScheduler(MachineSchedContext *C, MachineSchedStrategy *S): - ScheduleDAGMI(C, S) {} + ScheduleDAGMILive(C, S) {} /// Schedule - This is called back from ScheduleDAGInstrs::Run() when it's /// time to do some work. - virtual void schedule(); + virtual void schedule() LLVM_OVERRIDE; /// Perform platform specific DAG postprocessing. void postprocessDAG(); }; @@ -130,7 +130,7 @@ class ConvergingVLIWScheduler : public MachineSchedStrategy { /// Each Scheduling boundary is associated with ready queues. It tracks the /// current cycle in whichever direction at has moved, and maintains the state /// of "hazards" and other interlocks at the current cycle. - struct SchedBoundary { + struct VLIWSchedBoundary { VLIWMachineScheduler *DAG; const TargetSchedModel *SchedModel; @@ -152,14 +152,14 @@ class ConvergingVLIWScheduler : public MachineSchedStrategy { /// Pending queues extend the ready queues with the same ID and the /// PendingFlag set. - SchedBoundary(unsigned ID, const Twine &Name): + VLIWSchedBoundary(unsigned ID, const Twine &Name): DAG(0), SchedModel(0), Available(ID, Name+".A"), Pending(ID << ConvergingVLIWScheduler::LogMaxQID, Name+".P"), CheckPending(false), HazardRec(0), ResourceModel(0), CurrCycle(0), IssueCount(0), MinReadyCycle(UINT_MAX), MaxMinLatency(0) {} - ~SchedBoundary() { + ~VLIWSchedBoundary() { delete ResourceModel; delete HazardRec; } @@ -192,8 +192,8 @@ class ConvergingVLIWScheduler : public MachineSchedStrategy { const TargetSchedModel *SchedModel; // State of the top and bottom scheduled instruction boundaries. - SchedBoundary Top; - SchedBoundary Bot; + VLIWSchedBoundary Top; + VLIWSchedBoundary Bot; public: /// SUnit::NodeQueueId: 0 (none), 1 (top), 2 (bot), 3 (both) @@ -206,15 +206,15 @@ public: ConvergingVLIWScheduler(): DAG(0), SchedModel(0), Top(TopQID, "TopQ"), Bot(BotQID, "BotQ") {} - virtual void initialize(ScheduleDAGMI *dag); + virtual void initialize(ScheduleDAGMI *dag) LLVM_OVERRIDE; - virtual SUnit *pickNode(bool &IsTopNode); + virtual SUnit *pickNode(bool &IsTopNode) LLVM_OVERRIDE; - virtual void schedNode(SUnit *SU, bool IsTopNode); + virtual void schedNode(SUnit *SU, bool IsTopNode) LLVM_OVERRIDE; - virtual void releaseTopNode(SUnit *SU); + virtual void releaseTopNode(SUnit *SU) LLVM_OVERRIDE; - virtual void releaseBottomNode(SUnit *SU); + virtual void releaseBottomNode(SUnit *SU) LLVM_OVERRIDE; unsigned ReportPackets() { return Top.ResourceModel->getTotalPackets() + diff --git a/llvm/lib/Target/R600/AMDGPUTargetMachine.cpp b/llvm/lib/Target/R600/AMDGPUTargetMachine.cpp index e20e92014deb..1279665f16a8 100644 --- a/llvm/lib/Target/R600/AMDGPUTargetMachine.cpp +++ b/llvm/lib/Target/R600/AMDGPUTargetMachine.cpp @@ -42,7 +42,7 @@ extern "C" void LLVMInitializeR600Target() { } static ScheduleDAGInstrs *createR600MachineScheduler(MachineSchedContext *C) { - return new ScheduleDAGMI(C, new R600SchedStrategy()); + return new ScheduleDAGMILive(C, new R600SchedStrategy()); } static MachineSchedRegistry diff --git a/llvm/lib/Target/R600/R600MachineScheduler.cpp b/llvm/lib/Target/R600/R600MachineScheduler.cpp index da2a4d862e7d..d3ffb506f1ba 100644 --- a/llvm/lib/Target/R600/R600MachineScheduler.cpp +++ b/llvm/lib/Target/R600/R600MachineScheduler.cpp @@ -24,8 +24,8 @@ using namespace llvm; void R600SchedStrategy::initialize(ScheduleDAGMI *dag) { - - DAG = dag; + assert(dag->hasVRegLiveness() && "R600SchedStrategy needs vreg liveness"); + DAG = static_cast(dag); TII = static_cast(DAG->TII); TRI = static_cast(DAG->TRI); VLIW5 = !DAG->MF.getTarget().getSubtarget().hasCaymanISA(); @@ -72,7 +72,7 @@ SUnit* R600SchedStrategy::pickNode(bool &IsTopNode) { // OpenCL Programming Guide : // The approx. number of WF that allows TEX inst to hide ALU inst is : // 500 (cycles for TEX) / (AluFetchRatio * 8 (cycles for ALU)) - float ALUFetchRationEstimate = + float ALUFetchRationEstimate = (AluInstCount + AvailablesAluCount() + Pending[IDAlu].size()) / (FetchInstCount + Available[IDFetch].size()); unsigned NeededWF = 62.5f / ALUFetchRationEstimate; @@ -464,4 +464,3 @@ SUnit* R600SchedStrategy::pickOther(int QID) { } return SU; } - diff --git a/llvm/lib/Target/R600/R600MachineScheduler.h b/llvm/lib/Target/R600/R600MachineScheduler.h index 97c8cdec0aae..b909ff71a692 100644 --- a/llvm/lib/Target/R600/R600MachineScheduler.h +++ b/llvm/lib/Target/R600/R600MachineScheduler.h @@ -26,7 +26,7 @@ namespace llvm { class R600SchedStrategy : public MachineSchedStrategy { - const ScheduleDAGMI *DAG; + const ScheduleDAGMILive *DAG; const R600InstrInfo *TII; const R600RegisterInfo *TRI; MachineRegisterInfo *MRI;