[PGO] Indirect-call profile annotation in IR level profiling

This patch reads the indirect-call value records in the profile and makes the
annotation in the indirect-call instruction. This is for IR level profile
instrumentation.

Differential Revision: http://reviews.llvm.org/D16935

llvm-svn: 260400

llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp

@@ -25,9 +25,12 @@
 //
 // This file contains two passes:
 // (1) Pass PGOInstrumentationGen which instruments the IR to generate edge
-// count profile, and
+// count profile, and generates the instrumentation for indirect call
+// profiling.
 // (2) Pass PGOInstrumentationUse which reads the edge count profile and
-// annotates the branch weights.
+// annotates the branch weights. It also reads the indirect call value
+// profiling records and annotate the indirect call instructions.
+//
 // To get the precise counter information, These two passes need to invoke at
 // the same compilation point (so they see the same IR). For pass
 // PGOInstrumentationGen, the real work is done in instrumentOneFunc(). For
@@ -84,7 +87,7 @@ STATISTIC(NumOfPGOSplit, "Number of critical edge splits.");
 STATISTIC(NumOfPGOFunc, "Number of functions having valid profile counts.");
 STATISTIC(NumOfPGOMismatch, "Number of functions having mismatch profile.");
 STATISTIC(NumOfPGOMissing, "Number of functions without profile.");
-STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentation.");
+STATISTIC(NumOfPGOICall, "Number of indirect call value instrumentations.");
 
 // Command line option to specify the file to read profile from. This is
 // mainly used for testing.
@@ -95,7 +98,7 @@ static cl::opt<std::string>
                                 "mainly for test purpose."));
 
 // Command line options to disable value profiling. The default is false:
-// i.e. vaule profiling is enabled by default. This is for debug purpose.
+// i.e. value profiling is enabled by default. This is for debug purpose.
 static cl::opt<bool>
 DisableValueProfiling("disable-vp", cl::init(false),
                                     cl::Hidden,
@@ -463,6 +466,9 @@ private:
   // compilation.
   uint64_t ProgramMaxCount;
 
+  // ProfileRecord for this function.
+  InstrProfRecord ProfileRecord;
+
   // Find the Instrumented BB and set the value.
   void setInstrumentedCounts(const std::vector<uint64_t> &CountFromProfile);
 
@@ -502,6 +508,9 @@ public:
 
   // Set the branch weights based on the count values.
   void setBranchWeights();
+
+  // Annotate the indirect call sites.
+  void annotateIndirectCallSites();
 };
 
 // Visit all the edges and assign the count value for the instrumented
@@ -574,7 +583,8 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader) {
         DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
     return false;
   }
-  std::vector<uint64_t> &CountFromProfile = Result.get().Counts;
+  ProfileRecord = std::move(Result.get());
+  std::vector<uint64_t> &CountFromProfile = ProfileRecord.Counts;
 
   NumOfPGOFunc++;
   DEBUG(dbgs() << CountFromProfile.size() << " counts\n");
@@ -712,6 +722,36 @@ void PGOUseFunc::setBranchWeights() {
           dbgs() << "\n";);
   }
 }
+
+// Traverse all the indirect callsites and annotate the instructions.
+void PGOUseFunc::annotateIndirectCallSites() {
+  if (DisableValueProfiling)
+    return;
+
+  unsigned IndirectCallSiteIndex = 0;
+  PGOIndirectCallSiteVisitor ICV;
+  ICV.visit(F);
+  unsigned NumValueSites=
+      ProfileRecord.getNumValueSites(IPVK_IndirectCallTarget);
+  if (NumValueSites != ICV.IndirectCallInsts.size()) {
+    std::string Msg =
+        std::string("Inconsistent number of indirect call sites: ") +
+            F.getName().str();
+    auto &Ctx = M->getContext();
+    Ctx.diagnose(
+        DiagnosticInfoPGOProfile(M->getName().data(), Msg, DS_Warning));
+    return;
+  }
+
+  for (auto &I : ICV.IndirectCallInsts) {
+    DEBUG(dbgs() << "Read one indirect call instrumentation: Index="
+                 << IndirectCallSiteIndex << " out of "
+                 << NumValueSites<< "\n");
+    annotateValueSite(*M, *I, ProfileRecord, IPVK_IndirectCallTarget,
+                      IndirectCallSiteIndex);
+    IndirectCallSiteIndex++;
+  }
+}
 } // end anonymous namespace
 
 // Create a COMDAT variable IR_LEVEL_PROF_VARNAME to make the runtime
@@ -751,6 +791,7 @@ static void setPGOCountOnFunc(PGOUseFunc &Func,
   if (Func.readCounters(PGOReader)) {
     Func.populateCounters();
     Func.setBranchWeights();
+    Func.annotateIndirectCallSites();
   }
 }
 

llvm/test/Transforms/PGOProfile/Inputs/indirect_call.proftext

@@ -0,0 +1,43 @@
+:ir
+bar
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+140
+# Num Value Kinds:
+1
+# ValueKind = IPVK_IndirectCallTarget:
+0
+# NumValueSites:
+1
+3
+func2:80
+func1:40
+func3:20
+
+func1
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+40
+
+func2
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+80
+
+func3
+# Func Hash:
+12884901887
+# Num Counters:
+1
+# Counter Values:
+20
+

llvm/test/Transforms/PGOProfile/indirect_call_annotation.ll

@@ -0,0 +1,35 @@
+; RUN: llvm-profdata merge %S/Inputs/indirect_call.proftext -o %t.profdata
+; RUN: opt < %s -pgo-instr-use -pgo-test-profile-file=%t.profdata -S | FileCheck %s --check-prefix=VP-ANNOTATION
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+@foo = common global i32 (i32)* null, align 8
+
+define i32 @func1(i32 %x) {
+entry:
+  ret i32 %x
+}
+
+define i32 @func2(i32 %x) {
+entry:
+  %add = add nsw i32 %x, 1
+  ret i32 %add
+}
+
+define i32 @func3(i32 %x) {
+entry:
+  %add = add nsw i32 %x, 3
+  ret i32 %add
+}
+
+define i32 @bar(i32 %i) {
+entry:
+  %tmp = load i32 (i32)*, i32 (i32)** @foo, align 8
+  %call = call i32 %tmp(i32 %i)
+; VP-ANNOTATION: %call = call i32 %tmp(i32 %i)
+; VP-ANNOTATION-SAME: !prof ![[VP:[0-9]+]]
+; VP-ANNOTATION: ![[VP]] = !{!"VP", i32 0, i64 140, i64 -4377547752858689819, i64 80, i64 -2545542355363006406, i64 40, i64 -6929281286627296573, i64 20}
+  ret i32 %call
+}
+
+