[Index] Add an option to collect macros from preprocesor.

Summary: Also added unit tests for the index library; lit+c-index-test is painful...

Reviewers: ilya-biryukov

Reviewed By: ilya-biryukov

Subscribers: mgorny, cfe-commits

Differential Revision: https://reviews.llvm.org/D52098

llvm-svn: 342451

clang/include/clang/Index/IndexingAction.h

@@ -12,6 +12,7 @@
 
 #include "clang/Basic/LLVM.h"
 #include "clang/Lex/PPCallbacks.h"
+#include "clang/Lex/Preprocessor.h"
 #include "llvm/ADT/ArrayRef.h"
 #include <memory>
 
@@ -40,6 +41,10 @@ struct IndexingOptions {
     = SystemSymbolFilterKind::DeclarationsOnly;
   bool IndexFunctionLocals = false;
   bool IndexImplicitInstantiation = false;
+  // Whether to index macro definitions in the Preprocesor when preprocessor
+  // callback is not available (e.g. after parsing has finished). Note that
+  // macro references are not available in Proprocessor.
+  bool IndexMacrosInPreprocessor = false;
 };
 
 /// Creates a frontend action that indexes all symbols (macros and AST decls).
@@ -50,13 +55,12 @@ createIndexingAction(std::shared_ptr<IndexDataConsumer> DataConsumer,
                      std::unique_ptr<FrontendAction> WrappedAction);
 
 /// Recursively indexes all decls in the AST.
-/// Note that this does not index macros.
 void indexASTUnit(ASTUnit &Unit, IndexDataConsumer &DataConsumer,
                   IndexingOptions Opts);
 
 /// Recursively indexes \p Decls.
-/// Note that this does not index macros.
-void indexTopLevelDecls(ASTContext &Ctx, ArrayRef<const Decl *> Decls,
+void indexTopLevelDecls(ASTContext &Ctx, Preprocessor &PP,
+                        ArrayRef<const Decl *> Decls,
                         IndexDataConsumer &DataConsumer, IndexingOptions Opts);
 
 /// Creates a PPCallbacks that indexes macros and feeds macros to \p Consumer.
@@ -65,7 +69,6 @@ std::unique_ptr<PPCallbacks> indexMacrosCallback(IndexDataConsumer &Consumer,
                                                  IndexingOptions Opts);
 
 /// Recursively indexes all top-level decls in the module.
-/// FIXME: make this index macros as well.
 void indexModuleFile(serialization::ModuleFile &Mod, ASTReader &Reader,
                      IndexDataConsumer &DataConsumer, IndexingOptions Opts);
 

clang/lib/Index/IndexingAction.cpp

@@ -215,23 +215,41 @@ static void indexTranslationUnit(ASTUnit &Unit, IndexingContext &IndexCtx) {
   Unit.visitLocalTopLevelDecls(&IndexCtx, topLevelDeclVisitor);
 }
 
+static void indexPreprocessorMacros(const Preprocessor &PP,
+                                    IndexDataConsumer &DataConsumer) {
+  for (const auto &M : PP.macros())
+    if (MacroDirective *MD = M.second.getLatest())
+      DataConsumer.handleMacroOccurence(
+          M.first, MD->getMacroInfo(),
+          static_cast<unsigned>(index::SymbolRole::Definition),
+          MD->getLocation());
+}
+
 void index::indexASTUnit(ASTUnit &Unit, IndexDataConsumer &DataConsumer,
                          IndexingOptions Opts) {
   IndexingContext IndexCtx(Opts, DataConsumer);
   IndexCtx.setASTContext(Unit.getASTContext());
   DataConsumer.initialize(Unit.getASTContext());
   DataConsumer.setPreprocessor(Unit.getPreprocessorPtr());
+
+  if (Opts.IndexMacrosInPreprocessor)
+    indexPreprocessorMacros(Unit.getPreprocessor(), DataConsumer);
   indexTranslationUnit(Unit, IndexCtx);
   DataConsumer.finish();
 }
 
-void index::indexTopLevelDecls(ASTContext &Ctx, ArrayRef<const Decl *> Decls,
+void index::indexTopLevelDecls(ASTContext &Ctx, Preprocessor &PP,
+                               ArrayRef<const Decl *> Decls,
                                IndexDataConsumer &DataConsumer,
                                IndexingOptions Opts) {
   IndexingContext IndexCtx(Opts, DataConsumer);
   IndexCtx.setASTContext(Ctx);
 
   DataConsumer.initialize(Ctx);
+
+  if (Opts.IndexMacrosInPreprocessor)
+    indexPreprocessorMacros(PP, DataConsumer);
+
   for (const Decl *D : Decls)
     IndexCtx.indexTopLevelDecl(D);
   DataConsumer.finish();
@@ -251,6 +269,9 @@ void index::indexModuleFile(serialization::ModuleFile &Mod, ASTReader &Reader,
   IndexCtx.setASTContext(Ctx);
   DataConsumer.initialize(Ctx);
 
+  if (Opts.IndexMacrosInPreprocessor)
+    indexPreprocessorMacros(Reader.getPreprocessor(), DataConsumer);
+
   for (const Decl *D : Reader.getModuleFileLevelDecls(Mod)) {
     IndexCtx.indexTopLevelDecl(D);
   }

clang/unittests/CMakeLists.txt

@@ -31,3 +31,4 @@ if(NOT WIN32 AND CLANG_TOOL_LIBCLANG_BUILD)
   add_subdirectory(libclang)
 endif()
 add_subdirectory(Rename)
+add_subdirectory(Index)

clang/unittests/Index/CMakeLists.txt

@@ -0,0 +1,18 @@
+set(LLVM_LINK_COMPONENTS
+  ${LLVM_TARGETS_TO_BUILD}
+  Support
+  )
+
+add_clang_unittest(IndexTests
+  IndexTests.cpp
+  )
+
+target_link_libraries(IndexTests
+  PRIVATE
+  clangAST
+  clangBasic
+  clangFrontend
+  clangIndex
+  clangLex
+  clangTooling
+  )

clang/unittests/Index/IndexTests.cpp

@@ -0,0 +1,125 @@
+//===--- IndexTests.cpp - Test indexing actions -----------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/AST/ASTConsumer.h"
+#include "clang/AST/Decl.h"
+#include "clang/Basic/VirtualFileSystem.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/FrontendAction.h"
+#include "clang/Index/IndexDataConsumer.h"
+#include "clang/Index/IndexSymbol.h"
+#include "clang/Index/IndexingAction.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/ADT/StringRef.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include <memory>
+
+namespace clang {
+namespace index {
+
+struct TestSymbol {
+  std::string QName;
+  // FIXME: add more information.
+};
+
+llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const TestSymbol &S) {
+  return OS << S.QName;
+}
+
+namespace {
+class Indexer : public IndexDataConsumer {
+public:
+  bool handleDeclOccurence(const Decl *D, SymbolRoleSet Roles,
+                           ArrayRef<SymbolRelation>, SourceLocation,
+                           ASTNodeInfo) override {
+    const auto *ND = llvm::dyn_cast<NamedDecl>(D);
+    if (!ND)
+      return true;
+    TestSymbol S;
+    S.QName = ND->getQualifiedNameAsString();
+    Symbols.push_back(std::move(S));
+    return true;
+  }
+
+  bool handleMacroOccurence(const IdentifierInfo *Name, const MacroInfo *,
+                            SymbolRoleSet, SourceLocation) override {
+    TestSymbol S;
+    S.QName = Name->getName();
+    Symbols.push_back(std::move(S));
+    return true;
+  }
+
+  std::vector<TestSymbol> Symbols;
+};
+
+class IndexAction : public ASTFrontendAction {
+public:
+  IndexAction(std::shared_ptr<Indexer> Index,
+              IndexingOptions Opts = IndexingOptions())
+      : Index(std::move(Index)), Opts(Opts) {}
+
+protected:
+  std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
+                                                 StringRef InFile) override {
+    class Consumer : public ASTConsumer {
+      std::shared_ptr<Indexer> Index;
+      std::shared_ptr<Preprocessor> PP;
+      IndexingOptions Opts;
+
+    public:
+      Consumer(std::shared_ptr<Indexer> Index, std::shared_ptr<Preprocessor> PP,
+               IndexingOptions Opts)
+          : Index(std::move(Index)), PP(std::move(PP)), Opts(Opts) {}
+
+      void HandleTranslationUnit(ASTContext &Ctx) override {
+        std::vector<Decl *> DeclsToIndex(
+            Ctx.getTranslationUnitDecl()->decls().begin(),
+            Ctx.getTranslationUnitDecl()->decls().end());
+        indexTopLevelDecls(Ctx, *PP, DeclsToIndex, *Index, Opts);
+      }
+    };
+    return llvm::make_unique<Consumer>(Index, CI.getPreprocessorPtr(), Opts);
+  }
+
+private:
+  std::shared_ptr<Indexer> Index;
+  IndexingOptions Opts;
+};
+
+using testing::Contains;
+using testing::Not;
+using testing::UnorderedElementsAre;
+
+MATCHER_P(QName, Name, "") { return arg.QName == Name; }
+
+TEST(IndexTest, Simple) {
+  auto Index = std::make_shared<Indexer>();
+  tooling::runToolOnCode(new IndexAction(Index), "class X {}; void f() {}");
+  EXPECT_THAT(Index->Symbols, UnorderedElementsAre(QName("X"), QName("f")));
+}
+
+TEST(IndexTest, IndexPreprocessorMacros) {
+  std::string Code = "#define INDEX_MAC 1";
+  auto Index = std::make_shared<Indexer>();
+  IndexingOptions Opts;
+  Opts.IndexMacrosInPreprocessor = true;
+  tooling::runToolOnCode(new IndexAction(Index, Opts), Code);
+  EXPECT_THAT(Index->Symbols, Contains(QName("INDEX_MAC")));
+
+  Opts.IndexMacrosInPreprocessor = false;
+  Index->Symbols.clear();
+  tooling::runToolOnCode(new IndexAction(Index, Opts), Code);
+  EXPECT_THAT(Index->Symbols, UnorderedElementsAre());
+}
+
+} // namespace
+} // namespace index
+} // namespace clang