[clangd] Allow observation of changes to global CDBs.

Summary:
Currently, changes *within* CDBs are not tracked (CDB has no facility to do so).
However, discovery of new CDBs are tracked (all files are marked as modified).
Also, files whose compilation commands are explicitly set are marked modified.

The intent is to use this for auto-index. Newly discovered files will be indexed
with low priority.

Reviewers: ilya-biryukov

Subscribers: ioeric, MaskRay, jkorous, arphaman, kadircet, cfe-commits

Differential Revision: https://reviews.llvm.org/D54475

llvm-svn: 347297

clang-tools-extra/clangd/Function.h

@@ -16,6 +16,7 @@
 
 #include "llvm/ADT/FunctionExtras.h"
 #include "llvm/Support/Error.h"
+#include <mutex>
 #include <tuple>
 #include <utility>
 
@@ -83,6 +84,82 @@ ForwardBinder<Func, Args...> Bind(Func F, Args &&... As) {
       std::make_tuple(std::forward<Func>(F), std::forward<Args>(As)...));
 }
 
+/// An Event<T> allows events of type T to be broadcast to listeners.
+template <typename T> class Event {
+public:
+  // A Listener is the callback through which events are delivered.
+  using Listener = std::function<void(const T &)>;
+
+  // A subscription defines the scope of when a listener should receive events.
+  // After destroying the subscription, no more events are received.
+  class LLVM_NODISCARD Subscription {
+    Event *Parent;
+    unsigned ListenerID;
+
+    Subscription(Event *Parent, unsigned ListenerID)
+        : Parent(Parent), ListenerID(ListenerID) {}
+    friend Event;
+
+  public:
+    Subscription() : Parent(nullptr) {}
+    Subscription(Subscription &&Other) : Parent(nullptr) {
+      *this = std::move(Other);
+    }
+    Subscription &operator=(Subscription &&Other) {
+      // If *this is active, unsubscribe.
+      if (Parent) {
+        std::lock_guard<std::recursive_mutex>(Parent->ListenersMu);
+        llvm::erase_if(Parent->Listeners,
+                       [&](const std::pair<Listener, unsigned> &P) {
+                         return P.second == ListenerID;
+                       });
+      }
+      // Take over the other subscription, and mark it inactive.
+      std::tie(Parent, ListenerID) = std::tie(Other.Parent, Other.ListenerID);
+      Other.Parent = nullptr;
+      return *this;
+    }
+    // Destroying a subscription may block if an event is being broadcast.
+    ~Subscription() {
+      if (Parent)
+        *this = Subscription(); // Unsubscribe.
+    }
+  };
+
+  // Adds a listener that will observe all future events until the returned
+  // subscription is destroyed.
+  // May block if an event is currently being broadcast.
+  Subscription observe(Listener L) {
+    std::lock_guard<std::recursive_mutex> Lock(ListenersMu);
+    Listeners.push_back({std::move(L), ++ListenerCount});
+    return Subscription(this, ListenerCount);
+    ;
+  }
+
+  // Synchronously sends an event to all registered listeners.
+  // Must not be called from a listener to this event.
+  void broadcast(const T &V) {
+    // FIXME: it would be nice to dynamically check non-reentrancy here.
+    std::lock_guard<std::recursive_mutex> Lock(ListenersMu);
+    for (const auto &L : Listeners)
+      L.first(V);
+  }
+
+  ~Event() {
+    std::lock_guard<std::recursive_mutex> Lock(ListenersMu);
+    assert(Listeners.empty());
+  }
+
+private:
+  static_assert(std::is_same<typename std::decay<T>::type, T>::value,
+                "use a plain type: event values are always passed by const&");
+
+  std::recursive_mutex ListenersMu;
+  bool IsBroadcasting = false;
+  std::vector<std::pair<Listener, unsigned>> Listeners;
+  unsigned ListenerCount = 0;
+};
+
 } // namespace clangd
 } // namespace clang
 

clang-tools-extra/clangd/GlobalCompilationDatabase.cpp

@@ -49,17 +49,17 @@ DirectoryBasedGlobalCompilationDatabase::getCompileCommand(PathRef File) const {
   return None;
 }
 
-tooling::CompilationDatabase *
+std::pair<tooling::CompilationDatabase *, /*Cached*/ bool>
 DirectoryBasedGlobalCompilationDatabase::getCDBInDirLocked(PathRef Dir) const {
   // FIXME(ibiryukov): Invalidate cached compilation databases on changes
   auto CachedIt = CompilationDatabases.find(Dir);
   if (CachedIt != CompilationDatabases.end())
-    return CachedIt->second.get();
+    return {CachedIt->second.get(), true};
   std::string Error = "";
   auto CDB = tooling::CompilationDatabase::loadFromDirectory(Dir, Error);
   auto Result = CDB.get();
   CompilationDatabases.insert(std::make_pair(Dir, std::move(CDB)));
-  return Result;
+  return {Result, false};
 }
 
 tooling::CompilationDatabase *
@@ -69,14 +69,29 @@ DirectoryBasedGlobalCompilationDatabase::getCDBForFile(PathRef File) const {
           path::is_absolute(File, path::Style::windows)) &&
          "path must be absolute");
 
+  tooling::CompilationDatabase *CDB = nullptr;
+  bool Cached = false;
   std::lock_guard<std::mutex> Lock(Mutex);
-  if (CompileCommandsDir)
-    return getCDBInDirLocked(*CompileCommandsDir);
-  for (auto Path = path::parent_path(File); !Path.empty();
-       Path = path::parent_path(Path))
-    if (auto CDB = getCDBInDirLocked(Path))
-      return CDB;
-  return nullptr;
+  if (CompileCommandsDir) {
+    std::tie(CDB, Cached) = getCDBInDirLocked(*CompileCommandsDir);
+  } else {
+    for (auto Path = path::parent_path(File); !CDB && !Path.empty();
+         Path = path::parent_path(Path)) {
+      std::tie(CDB, Cached) = getCDBInDirLocked(Path);
+    }
+  }
+  if (CDB && !Cached)
+    OnCommandChanged.broadcast(CDB->getAllFiles());
+  return CDB;
+}
+
+OverlayCDB::OverlayCDB(const GlobalCompilationDatabase *Base,
+                       std::vector<std::string> FallbackFlags)
+    : Base(Base), FallbackFlags(std::move(FallbackFlags)) {
+  if (Base)
+    BaseChanged = Base->watch([this](const std::vector<std::string> Changes) {
+      OnCommandChanged.broadcast(Changes);
+    });
 }
 
 Optional<tooling::CompileCommand>
@@ -101,11 +116,14 @@ tooling::CompileCommand OverlayCDB::getFallbackCommand(PathRef File) const {
 
 void OverlayCDB::setCompileCommand(
     PathRef File, llvm::Optional<tooling::CompileCommand> Cmd) {
-  std::unique_lock<std::mutex> Lock(Mutex);
-  if (Cmd)
-    Commands[File] = std::move(*Cmd);
-  else
-    Commands.erase(File);
+  {
+    std::unique_lock<std::mutex> Lock(Mutex);
+    if (Cmd)
+      Commands[File] = std::move(*Cmd);
+    else
+      Commands.erase(File);
+  }
+  OnCommandChanged.broadcast({File});
 }
 
 } // namespace clangd

clang-tools-extra/clangd/GlobalCompilationDatabase.h

@@ -10,6 +10,7 @@
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_GLOBALCOMPILATIONDATABASE_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_GLOBALCOMPILATIONDATABASE_H
 
+#include "Function.h"
 #include "Path.h"
 #include "llvm/ADT/StringMap.h"
 #include <memory>
@@ -41,8 +42,15 @@ public:
   /// Clangd should treat the results as unreliable.
   virtual tooling::CompileCommand getFallbackCommand(PathRef File) const;
 
-  /// FIXME(ibiryukov): add facilities to track changes to compilation flags of
-  /// existing targets.
+  using CommandChanged = Event<std::vector<std::string>>;
+  /// The callback is notified when files may have new compile commands.
+  /// The argument is a list of full file paths.
+  CommandChanged::Subscription watch(CommandChanged::Listener L) const {
+    return OnCommandChanged.observe(std::move(L));
+  }
+
+protected:
+  mutable CommandChanged OnCommandChanged;
 };
 
 /// Gets compile args from tooling::CompilationDatabases built for parent
@@ -61,7 +69,8 @@ public:
 
 private:
   tooling::CompilationDatabase *getCDBForFile(PathRef File) const;
-  tooling::CompilationDatabase *getCDBInDirLocked(PathRef File) const;
+  std::pair<tooling::CompilationDatabase *, /*Cached*/ bool>
+  getCDBInDirLocked(PathRef File) const;
 
   mutable std::mutex Mutex;
   /// Caches compilation databases loaded from directories(keys are
@@ -81,8 +90,7 @@ public:
   // Base may be null, in which case no entries are inherited.
   // FallbackFlags are added to the fallback compile command.
   OverlayCDB(const GlobalCompilationDatabase *Base,
-             std::vector<std::string> FallbackFlags = {})
-      : Base(Base), FallbackFlags(std::move(FallbackFlags)) {}
+             std::vector<std::string> FallbackFlags = {});
 
   llvm::Optional<tooling::CompileCommand>
   getCompileCommand(PathRef File) const override;
@@ -98,6 +106,7 @@ private:
   llvm::StringMap<tooling::CompileCommand> Commands; /* GUARDED_BY(Mut) */
   const GlobalCompilationDatabase *Base;
   std::vector<std::string> FallbackFlags;
+  CommandChanged::Subscription BaseChanged;
 };
 
 } // namespace clangd

clang-tools-extra/unittests/clangd/CMakeLists.txt

@@ -23,6 +23,7 @@ add_extra_unittest(ClangdTests
   FileIndexTests.cpp
   FindSymbolsTests.cpp
   FSTests.cpp
+  FunctionTests.cpp
   FuzzyMatchTests.cpp
   GlobalCompilationDatabaseTests.cpp
   HeadersTests.cpp

clang-tools-extra/unittests/clangd/FunctionTests.cpp

@@ -0,0 +1,53 @@
+//===-- FunctionsTests.cpp ------------------------------------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "Function.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+namespace clang {
+namespace clangd {
+namespace {
+
+TEST(EventTest, Subscriptions) {
+  Event<int> E;
+  int N = 0;
+  {
+    Event<int>::Subscription SubA;
+    // No subscriptions are active.
+    E.broadcast(42);
+    EXPECT_EQ(0, N);
+
+    Event<int>::Subscription SubB = E.observe([&](int) { ++N; });
+    // Now one is active.
+    E.broadcast(42);
+    EXPECT_EQ(1, N);
+
+    SubA = E.observe([&](int) { ++N; });
+    // Both are active.
+    EXPECT_EQ(1, N);
+    E.broadcast(42);
+    EXPECT_EQ(3, N);
+
+    SubA = std::move(SubB);
+    // One is active.
+    EXPECT_EQ(3, N);
+    E.broadcast(42);
+    EXPECT_EQ(4, N);
+  }
+  // None are active.
+  EXPECT_EQ(4, N);
+  E.broadcast(42);
+  EXPECT_EQ(4, N);
+}
+
+} // namespace
+} // namespace clangd
+} // namespace clang

clang-tools-extra/unittests/clangd/GlobalCompilationDatabaseTests.cpp

@@ -88,6 +88,23 @@ TEST_F(OverlayCDBTest, NoBase) {
               ElementsAre("clang", testPath("foo.cc"), "-DA=6"));
 }
 
+TEST_F(OverlayCDBTest, Watch) {
+  OverlayCDB Inner(nullptr);
+  OverlayCDB Outer(&Inner);
+
+  std::vector<std::vector<std::string>> Changes;
+  auto Sub = Outer.watch([&](const std::vector<std::string> &ChangedFiles) {
+    Changes.push_back(ChangedFiles);
+  });
+
+  Inner.setCompileCommand("A.cpp", tooling::CompileCommand());
+  Outer.setCompileCommand("B.cpp", tooling::CompileCommand());
+  Inner.setCompileCommand("A.cpp", llvm::None);
+  Outer.setCompileCommand("C.cpp", llvm::None);
+  EXPECT_THAT(Changes, ElementsAre(ElementsAre("A.cpp"), ElementsAre("B.cpp"),
+                                   ElementsAre("A.cpp"), ElementsAre("C.cpp")));
+}
+
 } // namespace
 } // namespace clangd
 } // namespace clang