[NewPM] Provide method to run all pipeline callbacks, used for -O0

Some targets may add required passes via
TargetMachine::registerPassBuilderCallbacks(). We need to run those even
under -O0. As an example, BPFTargetMachine adds
BPFAbstractMemberAccessPass, a required pass.

This also allows us to clean up BackendUtil.cpp (and out-of-tree Rust
usage of the NPM) by allowing us to share added passes like coroutines
and sanitizers between -O0 and other optimization levels.

Since callbacks may end up not adding passes, we need to check if the
pass managers are empty before adding them, so PassManager now has an
isEmpty() function. For example, polly adds callbacks but doesn't always
add passes in those callbacks, so this is necessary to keep
-debug-pass-manager tests' output from changing depending on if polly is
enabled or not.

Tests are a continuation of those added in
https://reviews.llvm.org/D89083.

Reviewed By: asbirlea, Meinersbur

Differential Revision: https://reviews.llvm.org/D89158

clang/lib/CodeGen/BackendUtil.cpp

@@ -1020,6 +1020,9 @@ static PassBuilder::OptimizationLevel mapToLevel(const CodeGenOptions &Opts) {
   default:
     llvm_unreachable("Invalid optimization level!");
 
+  case 0:
+    return PassBuilder::OptimizationLevel::O0;
+
   case 1:
     return PassBuilder::OptimizationLevel::O1;
 
@@ -1249,6 +1252,10 @@ void EmitAssemblyHelper::EmitAssemblyWithNewPassManager(
   ModulePassManager MPM(CodeGenOpts.DebugPassManager);
 
   if (!CodeGenOpts.DisableLLVMPasses) {
+    // Map our optimization levels into one of the distinct levels used to
+    // configure the pipeline.
+    PassBuilder::OptimizationLevel Level = mapToLevel(CodeGenOpts);
+
     bool IsThinLTO = CodeGenOpts.PrepareForThinLTO;
     bool IsLTO = CodeGenOpts.PrepareForLTO;
 
@@ -1297,10 +1304,6 @@ void EmitAssemblyHelper::EmitAssemblyWithNewPassManager(
         MPM.addPass(NameAnonGlobalPass());
       }
     } else {
-      // Map our optimization levels into one of the distinct levels used to
-      // configure the pipeline.
-      PassBuilder::OptimizationLevel Level = mapToLevel(CodeGenOpts);
-
       // If we reached here with a non-empty index file name, then the index
       // file was empty and we are not performing ThinLTO backend compilation
       // (used in testing in a distributed build environment). Drop any the type
@@ -1438,6 +1441,8 @@ void EmitAssemblyHelper::EmitAssemblyWithNewPassManager(
     }
 
     if (CodeGenOpts.OptimizationLevel == 0) {
+      PB.runRegisteredEPCallbacks(MPM, Level, CodeGenOpts.DebugPassManager);
+
       // FIXME: the backends do not handle matrix intrinsics currently. Make
       // sure they are also lowered in O0. A lightweight version of the pass
       // should run in the backend pipeline on demand.

clang/test/CodeGen/bpf-O0.c

@@ -0,0 +1,7 @@
+// RUN: %clang -O0 %s -target bpf -g -c -o /dev/null -fexperimental-new-pass-manager
+// REQUIRES: bpf-registered-target
+
+struct ss {
+  int a;
+};
+int foo() { return __builtin_btf_type_id(0, 0) + __builtin_preserve_type_info(*(struct ss *)0, 0); }

llvm/include/llvm/IR/PassManager.h

@@ -570,6 +570,9 @@ public:
       Passes.emplace_back(std::move(P));
   }
 
+  /// Returns if the pass manager contains any passes.
+  bool isEmpty() { return Passes.empty(); }
+
   static bool isRequired() { return true; }
 
 protected:

llvm/include/llvm/Passes/PassBuilder.h

@@ -594,15 +594,20 @@ public:
   /// Register a callback for a default optimizer pipeline extension point
   ///
   /// This extension point allows adding optimizations at the very end of the
-  /// function optimization pipeline. A key difference between this and the
-  /// legacy PassManager's OptimizerLast callback is that this extension point
-  /// is not triggered at O0. Extensions to the O0 pipeline should append their
-  /// passes to the end of the overall pipeline.
+  /// function optimization pipeline.
   void registerOptimizerLastEPCallback(
       const std::function<void(ModulePassManager &, OptimizationLevel)> &C) {
     OptimizerLastEPCallbacks.push_back(C);
   }
 
+  /// Run all registered extension point callbacks
+  ///
+  /// This runs the registered callbacks in the order they would be run in a
+  /// typical build*Pipeline(). This allows for reusing register*EPCallback()
+  /// between O0 and O[123] pipelines.
+  void runRegisteredEPCallbacks(ModulePassManager &MPM, OptimizationLevel Level,
+                                bool DebugLogging);
+
   /// Register a callback for parsing an AliasAnalysis Name to populate
   /// the given AAManager \p AA
   void registerParseAACallback(

llvm/lib/Passes/PassBuilder.cpp

@@ -1662,6 +1662,56 @@ PassBuilder::buildLTODefaultPipeline(OptimizationLevel Level,
   return MPM;
 }
 
+void PassBuilder::runRegisteredEPCallbacks(ModulePassManager &MPM,
+                                           OptimizationLevel Level,
+                                           bool DebugLogging) {
+  assert(Level == OptimizationLevel::O0 &&
+         "runRegisteredEPCallbacks should only be used with O0");
+  for (auto &C : PipelineStartEPCallbacks)
+    C(MPM, Level);
+  if (!LateLoopOptimizationsEPCallbacks.empty()) {
+    LoopPassManager LPM(DebugLogging);
+    for (auto &C : LateLoopOptimizationsEPCallbacks)
+      C(LPM, Level);
+    if (!LPM.isEmpty()) {
+      MPM.addPass(createModuleToFunctionPassAdaptor(
+          createFunctionToLoopPassAdaptor(std::move(LPM))));
+    }
+  }
+  if (!LoopOptimizerEndEPCallbacks.empty()) {
+    LoopPassManager LPM(DebugLogging);
+    for (auto &C : LoopOptimizerEndEPCallbacks)
+      C(LPM, Level);
+    if (!LPM.isEmpty()) {
+      MPM.addPass(createModuleToFunctionPassAdaptor(
+          createFunctionToLoopPassAdaptor(std::move(LPM))));
+    }
+  }
+  if (!ScalarOptimizerLateEPCallbacks.empty()) {
+    FunctionPassManager FPM(DebugLogging);
+    for (auto &C : ScalarOptimizerLateEPCallbacks)
+      C(FPM, Level);
+    if (!FPM.isEmpty())
+      MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+  }
+  if (!CGSCCOptimizerLateEPCallbacks.empty()) {
+    CGSCCPassManager CGPM(DebugLogging);
+    for (auto &C : CGSCCOptimizerLateEPCallbacks)
+      C(CGPM, Level);
+    if (!CGPM.isEmpty())
+      MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
+  }
+  if (!VectorizerStartEPCallbacks.empty()) {
+    FunctionPassManager FPM(DebugLogging);
+    for (auto &C : VectorizerStartEPCallbacks)
+      C(FPM, Level);
+    if (!FPM.isEmpty())
+      MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
+  }
+  for (auto &C : OptimizerLastEPCallbacks)
+    C(MPM, Level);
+}
+
 AAManager PassBuilder::buildDefaultAAPipeline() {
   AAManager AA;
 
@@ -2244,6 +2294,8 @@ Error PassBuilder::parseModulePass(ModulePassManager &MPM,
         MPM.addPass(createModuleToFunctionPassAdaptor(CoroCleanupPass()));
       }
 
+      runRegisteredEPCallbacks(MPM, L, DebugLogging);
+
       // Do nothing else at all!
       return Error::success();
     }

llvm/test/CodeGen/BPF/optnone-2.ll

@@ -1,5 +1,5 @@
 ; RUN: opt < %s -passes='default<O2>' | llc -march=bpfel -filetype=asm -o /dev/null -
-; TODO: add -O0 once that's supported
+; RUN: opt < %s -passes='default<O0>' | llc -march=bpfel -filetype=asm -o /dev/null -
 
 ; IR generated by
 ; $ cat /tmp/a.c

llvm/test/Feature/optnone-opt.ll

@@ -4,7 +4,7 @@
 ; RUN: opt -O3 -S -debug -enable-new-pm=0 %s 2>&1 | FileCheck %s --check-prefix=O1 --check-prefix=O2O3
 ; RUN: opt -dce -gvn-hoist -loweratomic -S -debug -enable-new-pm=0 %s 2>&1 | FileCheck %s --check-prefix=MORE
 ; RUN: opt -indvars -licm -loop-deletion -loop-extract -loop-idiom -loop-instsimplify -loop-reduce -loop-reroll -loop-rotate -loop-unroll -loop-unswitch -enable-new-pm=0 -S -debug %s 2>&1 | FileCheck %s --check-prefix=LOOP
-; RUN: opt -enable-npm-optnone -passes='default<O0>' -S -debug-pass-manager %s 2>&1 | FileCheck %s --check-prefix=NPM-O0
+; RUN: opt -enable-npm-optnone -passes='default<O0>' -S -debug-pass-manager %s 2>&1 | FileCheck %s --check-prefix=%llvmcheckext-NPM-O0
 ; RUN: opt -enable-npm-optnone -passes='default<O1>' -S -debug-pass-manager %s 2>&1 | FileCheck %s --check-prefix=NPM-O1
 ; RUN: opt -enable-npm-optnone -passes='default<O2>' -S -debug-pass-manager %s 2>&1 | FileCheck %s --check-prefix=NPM-O1 --check-prefix=NPM-O2O3
 ; RUN: opt -enable-npm-optnone -passes='default<O3>' -S -debug-pass-manager %s 2>&1 | FileCheck %s --check-prefix=NPM-O1 --check-prefix=NPM-O2O3
@@ -40,9 +40,10 @@ while.end:                                        ; preds = %while.cond
 
 attributes #0 = { optnone noinline }
 
-; Nothing that runs at -O0 gets skipped.
+; Nothing that runs at -O0 gets skipped (except when the Bye extension is present).
 ; O0-NOT: Skipping pass
-; NPM-O0-NOT: Skipping pass
+; CHECK-EXT-NPM-O0: Skipping pass {{.*}}Bye
+; CHECK-NOEXT-NPM-O0-NOT: Skipping pass
 
 ; IR passes run at -O1 and higher.
 ; O1-DAG: Skipping pass 'Aggressive Dead Code Elimination'

llvm/test/Other/new-pass-manager.ll

@@ -358,8 +358,12 @@
 
 ; RUN: opt -disable-output -disable-verify -debug-pass-manager \
 ; RUN:     -passes='default<O0>' %s 2>&1 \
-; RUN:     | FileCheck %s --check-prefix=CHECK-O0
+; RUN:     | FileCheck %s --check-prefix=CHECK-O0 --check-prefix=%llvmcheckext
 ; CHECK-O0: Starting llvm::Module pass manager run
+; CHECK-EXT-NEXT: Running analysis: InnerAnalysisManagerProxy<{{.*}}>
+; CHECK-EXT-NEXT: Starting llvm::Function pass manager run.
+; CHECK-EXT-NEXT: Running pass: {{.*}}Bye
+; CHECK-EXT-NEXT: Finished llvm::Function pass manager run.
 ; CHECK-O0-NEXT: Finished llvm::Module pass manager run
 
 ; RUN: opt -disable-output -disable-verify -debug-pass-manager \

llvm/test/Other/new-pm-O0-ep-callbacks.ll

@@ -0,0 +1,24 @@
+; RUN: opt -disable-output -debug-pass-manager -passes-ep-late-loop-optimizations=no-op-loop -passes='default<O0>' 2>&1 < %s | FileCheck %s
+; RUN: opt -disable-output -debug-pass-manager -passes-ep-loop-optimizer-end=no-op-loop -passes='default<O0>' 2>&1 < %s | FileCheck %s
+; RUN: opt -disable-output -debug-pass-manager -passes-ep-scalar-optimizer-late=no-op-function -passes='default<O0>' 2>&1 < %s | FileCheck %s
+; RUN: opt -disable-output -debug-pass-manager -passes-ep-cgscc-optimizer-late=no-op-cgscc -passes='default<O0>' 2>&1 < %s | FileCheck %s
+; RUN: opt -disable-output -debug-pass-manager -passes-ep-vectorizer-start=no-op-function -passes='default<O0>' 2>&1 < %s | FileCheck %s
+; RUN: opt -disable-output -debug-pass-manager -passes-ep-pipeline-start=no-op-module -passes='default<O0>' 2>&1 < %s | FileCheck %s
+; RUN: opt -disable-output -debug-pass-manager -passes-ep-optimizer-last=no-op-function -passes='default<O0>' 2>&1 < %s | FileCheck %s
+
+; CHECK: Running pass: NoOp
+
+declare void @bar() local_unnamed_addr
+
+define void @foo(i32 %n) local_unnamed_addr {
+entry:
+  br label %loop
+loop:
+  %iv = phi i32 [ 0, %entry ], [ %iv.next, %loop ]
+  %iv.next = add i32 %iv, 1
+  tail call void @bar()
+  %cmp = icmp eq i32 %iv, %n
+  br i1 %cmp, label %exit, label %loop
+exit:
+  ret void
+}