diff --git a/llvm/tools/llvm-mca/DispatchStage.cpp b/llvm/tools/llvm-mca/DispatchStage.cpp index d397f72f4ab5..4f2dbeece7e3 100644 --- a/llvm/tools/llvm-mca/DispatchStage.cpp +++ b/llvm/tools/llvm-mca/DispatchStage.cpp @@ -33,7 +33,7 @@ void DispatchStage::notifyInstructionDispatched(const InstRef &IR, notifyEvent(HWInstructionDispatchedEvent(IR, UsedRegs)); } -bool DispatchStage::checkPRF(const InstRef &IR) { +bool DispatchStage::checkPRF(const InstRef &IR) const { SmallVector RegDefs; for (const std::unique_ptr &RegDef : IR.getInstruction()->getDefs()) @@ -50,7 +50,7 @@ bool DispatchStage::checkPRF(const InstRef &IR) { return true; } -bool DispatchStage::checkRCU(const InstRef &IR) { +bool DispatchStage::checkRCU(const InstRef &IR) const { const unsigned NumMicroOps = IR.getInstruction()->getDesc().NumMicroOps; if (RCU.isAvailable(NumMicroOps)) return true; @@ -59,7 +59,7 @@ bool DispatchStage::checkRCU(const InstRef &IR) { return false; } -bool DispatchStage::checkScheduler(const InstRef &IR) { +bool DispatchStage::checkScheduler(const InstRef &IR) const { HWStallEvent::GenericEventType Event; const bool Ready = SC.canBeDispatched(IR, Event); if (!Ready) @@ -131,9 +131,10 @@ void DispatchStage::dispatch(InstRef IR) { notifyInstructionDispatched(IR, RegisterFiles); } -void DispatchStage::cycleStart() { +llvm::Error DispatchStage::cycleStart() { AvailableEntries = CarryOver >= DispatchWidth ? 0 : DispatchWidth - CarryOver; CarryOver = CarryOver >= DispatchWidth ? CarryOver - DispatchWidth : 0U; + return llvm::ErrorSuccess(); } Stage::Status DispatchStage::execute(InstRef &IR) { diff --git a/llvm/tools/llvm-mca/DispatchStage.h b/llvm/tools/llvm-mca/DispatchStage.h index 06619a83133b..2312f8675f89 100644 --- a/llvm/tools/llvm-mca/DispatchStage.h +++ b/llvm/tools/llvm-mca/DispatchStage.h @@ -58,9 +58,9 @@ class DispatchStage final : public Stage { RegisterFile &PRF; Scheduler &SC; - bool checkRCU(const InstRef &IR); - bool checkPRF(const InstRef &IR); - bool checkScheduler(const InstRef &IR); + bool checkRCU(const InstRef &IR) const; + bool checkPRF(const InstRef &IR) const; + bool checkScheduler(const InstRef &IR) const; void dispatch(InstRef IR); void updateRAWDependencies(ReadState &RS, const llvm::MCSubtargetInfo &STI); @@ -93,7 +93,7 @@ public: // The retire stage, which controls the RCU, might have items to complete but // RetireStage::hasWorkToComplete will check for that case. bool hasWorkToComplete() const override { return false; } - void cycleStart() override; + llvm::Error cycleStart() override; Status execute(InstRef &IR) override; void notifyDispatchStall(const InstRef &IR, unsigned EventType); diff --git a/llvm/tools/llvm-mca/ExecuteStage.cpp b/llvm/tools/llvm-mca/ExecuteStage.cpp index 25f90b469644..f9abf9f90f43 100644 --- a/llvm/tools/llvm-mca/ExecuteStage.cpp +++ b/llvm/tools/llvm-mca/ExecuteStage.cpp @@ -35,7 +35,7 @@ void ExecuteStage::reclaimSchedulerResources() { } // Update the scheduler's instruction queues. -void ExecuteStage::updateSchedulerQueues() { +Error ExecuteStage::updateSchedulerQueues() { SmallVector InstructionIDs; HWS.updateIssuedSet(InstructionIDs); for (const InstRef &IR : InstructionIDs) @@ -45,10 +45,11 @@ void ExecuteStage::updateSchedulerQueues() { HWS.updatePendingQueue(InstructionIDs); for (const InstRef &IR : InstructionIDs) notifyInstructionReady(IR); + return ErrorSuccess(); } // Issue instructions that are waiting in the scheduler's ready queue. -void ExecuteStage::issueReadyInstructions() { +Error ExecuteStage::issueReadyInstructions() { SmallVector InstructionIDs; InstRef IR = HWS.select(); while (IR.isValid()) { @@ -75,6 +76,8 @@ void ExecuteStage::issueReadyInstructions() { // Select the next instruction to issue. IR = HWS.select(); } + + return ErrorSuccess(); } // The following routine is the maintenance routine of the ExecuteStage. @@ -89,10 +92,11 @@ void ExecuteStage::issueReadyInstructions() { // Notifications are issued to this stage's listeners when instructions are // moved between the HWS's queues. In particular, when an instruction becomes // ready or executed. -void ExecuteStage::cycleStart() { +Error ExecuteStage::cycleStart() { reclaimSchedulerResources(); - updateSchedulerQueues(); - issueReadyInstructions(); + if (Error S = updateSchedulerQueues()) + return S; + return issueReadyInstructions(); } // Schedule the instruction for execution on the hardware. diff --git a/llvm/tools/llvm-mca/ExecuteStage.h b/llvm/tools/llvm-mca/ExecuteStage.h index d40057a56892..d22cca026aab 100644 --- a/llvm/tools/llvm-mca/ExecuteStage.h +++ b/llvm/tools/llvm-mca/ExecuteStage.h @@ -33,8 +33,8 @@ class ExecuteStage final : public Stage { // The following routines are used to maintain the HWS. void reclaimSchedulerResources(); - void updateSchedulerQueues(); - void issueReadyInstructions(); + llvm::Error updateSchedulerQueues(); + llvm::Error issueReadyInstructions(); ExecuteStage(const ExecuteStage &Other) = delete; ExecuteStage &operator=(const ExecuteStage &Other) = delete; @@ -42,11 +42,15 @@ class ExecuteStage final : public Stage { public: ExecuteStage(RetireControlUnit &R, Scheduler &S) : Stage(), RCU(R), HWS(S) {} - // The ExecuteStage will always complete all of its work per call to - // execute(), so it is never left in a 'to-be-processed' state. + // This stage works under the assumption that the Pipeline will eventually + // execute a retire stage. We don't need to check if pipelines and/or + // schedulers have instructions to process, because those instructions are + // also tracked by the retire control unit. That means, + // RetireControlUnit::hasWorkToComplete() is responsible for checking if there + // are still instructions in-flight in the out-of-order backend. bool hasWorkToComplete() const override { return false; } - void cycleStart() override; + llvm::Error cycleStart() override; Status execute(InstRef &IR) override; void diff --git a/llvm/tools/llvm-mca/FetchStage.cpp b/llvm/tools/llvm-mca/FetchStage.cpp index acb5b05f347e..1a6b939a1f04 100644 --- a/llvm/tools/llvm-mca/FetchStage.cpp +++ b/llvm/tools/llvm-mca/FetchStage.cpp @@ -35,7 +35,7 @@ Stage::Status FetchStage::execute(InstRef &IR) { void FetchStage::postExecute() { SM.updateNext(); } -void FetchStage::cycleEnd() { +llvm::Error FetchStage::cycleEnd() { // Find the first instruction which hasn't been retired. const InstMap::iterator It = llvm::find_if(Instructions, [](const InstMap::value_type &KeyValuePair) { @@ -45,6 +45,8 @@ void FetchStage::cycleEnd() { // Erase instructions up to the first that hasn't been retired. if (It != Instructions.begin()) Instructions.erase(Instructions.begin(), It); + + return llvm::ErrorSuccess(); } } // namespace mca diff --git a/llvm/tools/llvm-mca/FetchStage.h b/llvm/tools/llvm-mca/FetchStage.h index 447ec3bf7c0b..e028cd02f277 100644 --- a/llvm/tools/llvm-mca/FetchStage.h +++ b/llvm/tools/llvm-mca/FetchStage.h @@ -38,7 +38,7 @@ public: bool hasWorkToComplete() const override; Status execute(InstRef &IR) override; void postExecute() override; - void cycleEnd() override; + llvm::Error cycleEnd() override; }; } // namespace mca diff --git a/llvm/tools/llvm-mca/Pipeline.cpp b/llvm/tools/llvm-mca/Pipeline.cpp index f099ec629b9d..91c7a5d25ea7 100644 --- a/llvm/tools/llvm-mca/Pipeline.cpp +++ b/llvm/tools/llvm-mca/Pipeline.cpp @@ -32,10 +32,9 @@ void Pipeline::addEventListener(HWEventListener *Listener) { } bool Pipeline::hasWorkToProcess() { - const auto It = llvm::find_if(Stages, [](const std::unique_ptr &S) { + return llvm::any_of(Stages, [](const std::unique_ptr &S) { return S->hasWorkToComplete(); }); - return It != Stages.end(); } // This routine returns early if any stage returns 'false' after execute() is @@ -62,6 +61,8 @@ void Pipeline::postExecuteStages() { } llvm::Error Pipeline::run() { + assert(!Stages.empty() && "Unexpected empty pipeline found!"); + while (hasWorkToProcess()) { notifyCycleBegin(); if (llvm::Error Err = runCycle()) @@ -73,13 +74,18 @@ llvm::Error Pipeline::run() { } llvm::Error Pipeline::runCycle() { - // Update the stages before we do any processing for this cycle. - InstRef IR; - for (auto &S : Stages) - S->cycleStart(); + // Update stages before we start processing new instructions. + llvm::Error Err = llvm::ErrorSuccess(); + for (auto I = Stages.begin(), E = Stages.end(); I != E && !Err; ++I) { + const std::unique_ptr &S = *I; + Err = S->cycleStart(); + } - // Continue executing this cycle until any stage claims it cannot make - // progress. + if (Err) + return Err; + + // Now fetch and execute new instructions. + InstRef IR; while (true) { preExecuteStages(); Stage::Status Val = executeStages(IR); @@ -90,9 +96,12 @@ llvm::Error Pipeline::runCycle() { postExecuteStages(); } - for (auto &S : Stages) - S->cycleEnd(); - return llvm::ErrorSuccess(); + // Update stages in preparation for a new cycle. + for (auto I = Stages.begin(), E = Stages.end(); I != E && !Err; ++I) { + const std::unique_ptr &S = *I; + Err = S->cycleEnd(); + } + return Err; } void Pipeline::notifyCycleBegin() { diff --git a/llvm/tools/llvm-mca/RetireStage.cpp b/llvm/tools/llvm-mca/RetireStage.cpp index 55c3b887e478..43e3364028f9 100644 --- a/llvm/tools/llvm-mca/RetireStage.cpp +++ b/llvm/tools/llvm-mca/RetireStage.cpp @@ -18,15 +18,13 @@ #include "HWEventListener.h" #include "llvm/Support/Debug.h" -using namespace llvm; - #define DEBUG_TYPE "llvm-mca" namespace mca { -void RetireStage::cycleStart() { +llvm::Error RetireStage::cycleStart() { if (RCU.isEmpty()) - return; + return llvm::ErrorSuccess(); const unsigned MaxRetirePerCycle = RCU.getMaxRetirePerCycle(); unsigned NumRetired = 0; @@ -40,11 +38,13 @@ void RetireStage::cycleStart() { notifyInstructionRetired(Current.IR); NumRetired++; } + + return llvm::ErrorSuccess(); } void RetireStage::notifyInstructionRetired(const InstRef &IR) { - LLVM_DEBUG(dbgs() << "[E] Instruction Retired: #" << IR << '\n'); - SmallVector FreedRegs(PRF.getNumRegisterFiles()); + LLVM_DEBUG(llvm::dbgs() << "[E] Instruction Retired: #" << IR << '\n'); + llvm::SmallVector FreedRegs(PRF.getNumRegisterFiles()); const Instruction &Inst = *IR.getInstruction(); const InstrDesc &Desc = Inst.getDesc(); diff --git a/llvm/tools/llvm-mca/RetireStage.h b/llvm/tools/llvm-mca/RetireStage.h index f10188d1e6e5..36af4478618b 100644 --- a/llvm/tools/llvm-mca/RetireStage.h +++ b/llvm/tools/llvm-mca/RetireStage.h @@ -36,7 +36,7 @@ public: : Stage(), RCU(R), PRF(F) {} bool hasWorkToComplete() const override { return !RCU.isEmpty(); } - void cycleStart() override; + llvm::Error cycleStart() override; Status execute(InstRef &IR) override { return Stage::Continue; } void notifyInstructionRetired(const InstRef &IR); void onInstructionExecuted(unsigned TokenID); diff --git a/llvm/tools/llvm-mca/SourceMgr.h b/llvm/tools/llvm-mca/SourceMgr.h index 15a85a69569f..573ca7a9a003 100644 --- a/llvm/tools/llvm-mca/SourceMgr.h +++ b/llvm/tools/llvm-mca/SourceMgr.h @@ -44,6 +44,7 @@ public: void updateNext() { Current++; } const SourceRef peekNext() const { + assert(hasNext() && "Already at end of sequence!"); unsigned Index = getCurrentInstructionIndex(); return SourceRef(Current, Sequence[Index].get()); } diff --git a/llvm/tools/llvm-mca/Stage.cpp b/llvm/tools/llvm-mca/Stage.cpp index 7ead940e63c1..5cf76ab0a412 100644 --- a/llvm/tools/llvm-mca/Stage.cpp +++ b/llvm/tools/llvm-mca/Stage.cpp @@ -18,7 +18,7 @@ namespace mca { // Pin the vtable here in the implementation file. -Stage::Stage() {} +Stage::~Stage() = default; void Stage::addListener(HWEventListener *Listener) { Listeners.insert(Listener); diff --git a/llvm/tools/llvm-mca/Stage.h b/llvm/tools/llvm-mca/Stage.h index 151671291491..c9ca1b295b3b 100644 --- a/llvm/tools/llvm-mca/Stage.h +++ b/llvm/tools/llvm-mca/Stage.h @@ -25,9 +25,10 @@ namespace mca { class InstRef; class Stage { + std::set Listeners; + Stage(const Stage &Other) = delete; Stage &operator=(const Stage &Other) = delete; - std::set Listeners; public: /// A Stage's execute() returns Continue, Stop, or an error. Returning @@ -46,8 +47,8 @@ protected: const std::set &getListeners() const { return Listeners; } public: - Stage(); - virtual ~Stage() = default; + Stage() {} + virtual ~Stage(); /// Called prior to preExecute to ensure that the stage has items that it /// is to process. For example, a FetchStage might have more instructions @@ -57,10 +58,10 @@ public: /// Called once at the start of each cycle. This can be used as a setup /// phase to prepare for the executions during the cycle. - virtual void cycleStart() {} + virtual llvm::Error cycleStart() { return llvm::ErrorSuccess(); } /// Called once at the end of each cycle. - virtual void cycleEnd() {} + virtual llvm::Error cycleEnd() { return llvm::ErrorSuccess(); } /// Called prior to executing the list of stages. /// This can be called multiple times per cycle. @@ -80,7 +81,7 @@ public: void addListener(HWEventListener *Listener); /// Notify listeners of a particular hardware event. - template void notifyEvent(const EventT &Event) { + template void notifyEvent(const EventT &Event) const { for (HWEventListener *Listener : Listeners) Listener->onEvent(Event); }