[VFS] Add an in-memory file system implementation.

This is a simple file system tree of memory buffers that can be filled by a
client. In conjunction with an OverlayFS it can be used to make virtual
files accessible right next to physical files. This can be used as a
replacement for the virtual file handling in FileManager and which I intend
to remove eventually.

llvm-svn: 249315

clang/include/clang/Basic/VirtualFileSystem.h

@@ -241,6 +241,27 @@ public:
   iterator overlays_end() { return FSList.rend(); }
 };
 
+namespace detail {
+class InMemoryDirectory;
+} // end namespace detail
+
+/// An in-memory file system.
+class InMemoryFileSystem : public FileSystem {
+  std::unique_ptr<detail::InMemoryDirectory> Root;
+
+public:
+  InMemoryFileSystem();
+  ~InMemoryFileSystem() override;
+  void addFile(const Twine &Path, time_t ModificationTime,
+               std::unique_ptr<llvm::MemoryBuffer> Buffer);
+  StringRef toString() const;
+
+  llvm::ErrorOr<Status> status(const Twine &Path) override;
+  llvm::ErrorOr<std::unique_ptr<File>>
+  openFileForRead(const Twine &Path) override;
+  directory_iterator dir_begin(const Twine &Dir, std::error_code &EC) override;
+};
+
 /// \brief Get a globally unique ID for a virtual file or directory.
 llvm::sys::fs::UniqueID getNextVirtualUniqueID();
 

clang/lib/Basic/VirtualFileSystem.cpp

@@ -320,6 +320,252 @@ directory_iterator OverlayFileSystem::dir_begin(const Twine &Dir,
       std::make_shared<OverlayFSDirIterImpl>(Dir, *this, EC));
 }
 
+namespace clang {
+namespace vfs {
+namespace detail {
+
+enum InMemoryNodeKind { IME_File, IME_Directory };
+
+/// The in memory file system is a tree of Nodes. Every node can either be a
+/// file or a directory.
+class InMemoryNode {
+  Status Stat;
+  InMemoryNodeKind Kind;
+
+public:
+  InMemoryNode(Status Stat, InMemoryNodeKind Kind)
+      : Stat(std::move(Stat)), Kind(Kind) {}
+  virtual ~InMemoryNode() {}
+  const Status &getStatus() const { return Stat; }
+  InMemoryNodeKind getKind() const { return Kind; }
+  virtual std::string toString(unsigned Indent) const = 0;
+};
+
+namespace {
+class InMemoryFile : public InMemoryNode {
+  std::unique_ptr<llvm::MemoryBuffer> Buffer;
+
+public:
+  InMemoryFile(Status Stat, std::unique_ptr<llvm::MemoryBuffer> Buffer)
+      : InMemoryNode(std::move(Stat), IME_File), Buffer(std::move(Buffer)) {}
+
+  llvm::MemoryBuffer *getBuffer() { return Buffer.get(); }
+  std::string toString(unsigned Indent) const override {
+    return (std::string(Indent, ' ') + getStatus().getName() + "\n").str();
+  }
+  static bool classof(const InMemoryNode *N) {
+    return N->getKind() == IME_File;
+  }
+};
+
+/// Adapt a InMemoryFile for VFS' File interface.
+class InMemoryFileAdaptor : public File {
+  InMemoryFile &Node;
+
+public:
+  explicit InMemoryFileAdaptor(InMemoryFile &Node) : Node(Node) {}
+
+  llvm::ErrorOr<Status> status() override { return Node.getStatus(); }
+  llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>>
+  getBuffer(const Twine &Name, int64_t FileSize = -1,
+            bool RequiresNullTerminator = true,
+            bool IsVolatile = false) override {
+    llvm::MemoryBuffer *Buf = Node.getBuffer();
+    return llvm::MemoryBuffer::getMemBuffer(
+        Buf->getBuffer(), Buf->getBufferIdentifier(), RequiresNullTerminator);
+  }
+  std::error_code close() override { return std::error_code(); }
+};
+} // end anonymous namespace
+
+class InMemoryDirectory : public InMemoryNode {
+  std::map<std::string, std::unique_ptr<InMemoryNode>> Entries;
+
+public:
+  InMemoryDirectory(Status Stat)
+      : InMemoryNode(std::move(Stat), IME_Directory) {}
+  InMemoryNode *getChild(StringRef Name) {
+    auto I = Entries.find(Name);
+    if (I != Entries.end())
+      return I->second.get();
+    return nullptr;
+  }
+  InMemoryNode *addChild(StringRef Name, std::unique_ptr<InMemoryNode> Child) {
+    return Entries.insert(make_pair(Name, std::move(Child)))
+        .first->second.get();
+  }
+
+  typedef decltype(Entries)::const_iterator const_iterator;
+  const_iterator begin() const { return Entries.begin(); }
+  const_iterator end() const { return Entries.end(); }
+
+  std::string toString(unsigned Indent) const override {
+    std::string Result =
+        (std::string(Indent, ' ') + getStatus().getName() + "\n").str();
+    for (const auto &Entry : Entries) {
+      Result += Entry.second->toString(Indent + 2);
+    }
+    return Result;
+  }
+  static bool classof(const InMemoryNode *N) {
+    return N->getKind() == IME_Directory;
+  }
+};
+}
+
+InMemoryFileSystem::InMemoryFileSystem()
+    : Root(new detail::InMemoryDirectory(
+          Status("", getNextVirtualUniqueID(), llvm::sys::TimeValue::MinTime(),
+                 0, 0, 0, llvm::sys::fs::file_type::directory_file,
+                 llvm::sys::fs::perms::all_all))) {}
+
+InMemoryFileSystem::~InMemoryFileSystem() {}
+
+StringRef InMemoryFileSystem::toString() const {
+  return Root->toString(/*Indent=*/0);
+}
+
+void InMemoryFileSystem::addFile(const Twine &P, time_t ModificationTime,
+                                 std::unique_ptr<llvm::MemoryBuffer> Buffer) {
+  SmallString<128> Path;
+  P.toVector(Path);
+
+  // Fix up relative paths. This just prepends the current working directory.
+  std::error_code EC = sys::fs::make_absolute(Path);
+  assert(!EC);
+  (void)EC;
+
+  detail::InMemoryDirectory *Dir = Root.get();
+  auto I = llvm::sys::path::begin(Path), E = llvm::sys::path::end(Path);
+  while (true) {
+    StringRef Name = *I;
+    detail::InMemoryNode *Node = Dir->getChild(Name);
+    ++I;
+    if (!Node) {
+      if (I == E) {
+        // End of the path, create a new file.
+        // FIXME: expose the status details in the interface.
+        Status Stat(Path, getNextVirtualUniqueID(),
+                    llvm::sys::TimeValue(ModificationTime), 0, 0,
+                    Buffer->getBufferSize(),
+                    llvm::sys::fs::file_type::regular_file,
+                    llvm::sys::fs::all_all);
+        Dir->addChild(Name, llvm::make_unique<detail::InMemoryFile>(
+                                std::move(Stat), std::move(Buffer)));
+        return;
+      }
+
+      // Create a new directory. Use the path up to here.
+      // FIXME: expose the status details in the interface.
+      Status Stat(
+          StringRef(Path.str().begin(), Name.end() - Path.str().begin()),
+          getNextVirtualUniqueID(), llvm::sys::TimeValue(ModificationTime), 0,
+          0, Buffer->getBufferSize(), llvm::sys::fs::file_type::directory_file,
+          llvm::sys::fs::all_all);
+      Dir = cast<detail::InMemoryDirectory>(Dir->addChild(
+          Name, llvm::make_unique<detail::InMemoryDirectory>(std::move(Stat))));
+      continue;
+    }
+
+    if (auto *NewDir = dyn_cast<detail::InMemoryDirectory>(Node))
+      Dir = NewDir;
+  }
+}
+
+static ErrorOr<detail::InMemoryNode *>
+lookupInMemoryNode(detail::InMemoryDirectory *Dir, const Twine &P) {
+  SmallString<128> Path;
+  P.toVector(Path);
+
+  // Fix up relative paths. This just prepends the current working directory.
+  std::error_code EC = sys::fs::make_absolute(Path);
+  assert(!EC);
+  (void)EC;
+
+  auto I = llvm::sys::path::begin(Path), E = llvm::sys::path::end(Path);
+  while (true) {
+    detail::InMemoryNode *Node = Dir->getChild(*I);
+    ++I;
+    if (!Node)
+      return errc::no_such_file_or_directory;
+
+    // Return the file if it's at the end of the path.
+    if (auto File = dyn_cast<detail::InMemoryFile>(Node)) {
+      if (I == E)
+        return File;
+      return errc::no_such_file_or_directory;
+    }
+
+    // Traverse directories.
+    Dir = cast<detail::InMemoryDirectory>(Node);
+    if (I == E)
+      return Dir;
+  }
+}
+
+llvm::ErrorOr<Status> InMemoryFileSystem::status(const Twine &Path) {
+  auto Node = lookupInMemoryNode(Root.get(), Path);
+  if (Node)
+    return (*Node)->getStatus();
+  return Node.getError();
+}
+
+llvm::ErrorOr<std::unique_ptr<File>>
+InMemoryFileSystem::openFileForRead(const Twine &Path) {
+  auto Node = lookupInMemoryNode(Root.get(), Path);
+  if (!Node)
+    return Node.getError();
+
+  // When we have a file provide a heap-allocated wrapper for the memory buffer
+  // to match the ownership semantics for File.
+  if (auto *F = dyn_cast<detail::InMemoryFile>(*Node))
+    return std::unique_ptr<File>(new detail::InMemoryFileAdaptor(*F));
+
+  // FIXME: errc::not_a_file?
+  return make_error_code(llvm::errc::invalid_argument);
+}
+
+namespace {
+/// Adaptor from InMemoryDir::iterator to directory_iterator.
+class InMemoryDirIterator : public clang::vfs::detail::DirIterImpl {
+  detail::InMemoryDirectory::const_iterator I;
+  detail::InMemoryDirectory::const_iterator E;
+
+public:
+  InMemoryDirIterator() {}
+  explicit InMemoryDirIterator(detail::InMemoryDirectory &Dir)
+      : I(Dir.begin()), E(Dir.end()) {
+    if (I != E)
+      CurrentEntry = I->second->getStatus();
+  }
+
+  std::error_code increment() override {
+    ++I;
+    // When we're at the end, make CurrentEntry invalid and DirIterImpl will do
+    // the rest.
+    CurrentEntry = I != E ? I->second->getStatus() : Status();
+    return std::error_code();
+  }
+};
+} // end anonymous namespace
+
+directory_iterator InMemoryFileSystem::dir_begin(const Twine &Dir,
+                                                 std::error_code &EC) {
+  auto Node = lookupInMemoryNode(Root.get(), Dir);
+  if (!Node) {
+    EC = Node.getError();
+    return directory_iterator(std::make_shared<InMemoryDirIterator>());
+  }
+
+  if (auto *DirNode = dyn_cast<detail::InMemoryDirectory>(*Node))
+    return directory_iterator(std::make_shared<InMemoryDirIterator>(*DirNode));
+
+  EC = make_error_code(llvm::errc::not_a_directory);
+  return directory_iterator(std::make_shared<InMemoryDirIterator>());
+}
+}
+}
+
 //===-----------------------------------------------------------------------===/
 // VFSFromYAML implementation
 //===-----------------------------------------------------------------------===/

clang/unittests/Basic/VirtualFileSystemTest.cpp

@@ -515,6 +515,73 @@ TEST(VirtualFileSystemTest, HiddenInIteration) {
   }
 }
 
+class InMemoryFileSystemTest : public ::testing::Test {
+protected:
+  clang::vfs::InMemoryFileSystem FS;
+};
+
+TEST_F(InMemoryFileSystemTest, IsEmpty) {
+  auto Stat = FS.status("/a");
+  ASSERT_EQ(errc::no_such_file_or_directory, Stat.getError()) << FS.toString();
+  Stat = FS.status("/");
+  ASSERT_EQ(errc::no_such_file_or_directory, Stat.getError()) << FS.toString();
+}
+
+TEST_F(InMemoryFileSystemTest, WindowsPath) {
+  FS.addFile("c:/windows/system128/foo.cpp", 0, MemoryBuffer::getMemBuffer(""));
+  auto Stat = FS.status("c:");
+  ASSERT_FALSE(Stat.getError()) << Stat.getError() << FS.toString();
+  Stat = FS.status("c:/windows/system128/foo.cpp");
+  ASSERT_FALSE(Stat.getError()) << Stat.getError() << FS.toString();
+  FS.addFile("d:/windows/foo.cpp", 0, MemoryBuffer::getMemBuffer(""));
+  Stat = FS.status("d:/windows/foo.cpp");
+  ASSERT_FALSE(Stat.getError()) << Stat.getError() << FS.toString();
+}
+
+TEST_F(InMemoryFileSystemTest, OverlayFile) {
+  FS.addFile("/a", 0, MemoryBuffer::getMemBuffer("a"));
+  auto Stat = FS.status("/");
+  ASSERT_FALSE(Stat.getError()) << Stat.getError() << FS.toString();
+  Stat = FS.status("/a");
+  ASSERT_FALSE(Stat.getError()) << Stat.getError() << "\n" << FS.toString();
+  ASSERT_EQ("/a", Stat->getName());
+}
+
+TEST_F(InMemoryFileSystemTest, OpenFileForRead) {
+  FS.addFile("/a", 0, MemoryBuffer::getMemBuffer("a"));
+  auto File = FS.openFileForRead("/a");
+  ASSERT_EQ("a", (*(*File)->getBuffer("ignored"))->getBuffer());
+  File = FS.openFileForRead("/a"); // Open again.
+  ASSERT_EQ("a", (*(*File)->getBuffer("ignored"))->getBuffer());
+  File = FS.openFileForRead("/");
+  ASSERT_EQ(errc::invalid_argument, File.getError()) << FS.toString();
+  File = FS.openFileForRead("/b");
+  ASSERT_EQ(errc::no_such_file_or_directory, File.getError()) << FS.toString();
+}
+
+TEST_F(InMemoryFileSystemTest, DirectoryIteration) {
+  FS.addFile("/a", 0, MemoryBuffer::getMemBuffer(""));
+  FS.addFile("/b/c", 0, MemoryBuffer::getMemBuffer(""));
+
+  std::error_code EC;
+  vfs::directory_iterator I = FS.dir_begin("/", EC);
+  ASSERT_FALSE(EC);
+  ASSERT_EQ("/a", I->getName());
+  I.increment(EC);
+  ASSERT_FALSE(EC);
+  ASSERT_EQ("/b", I->getName());
+  I.increment(EC);
+  ASSERT_FALSE(EC);
+  ASSERT_EQ(vfs::directory_iterator(), I);
+
+  I = FS.dir_begin("/b", EC);
+  ASSERT_FALSE(EC);
+  ASSERT_EQ("/b/c", I->getName());
+  I.increment(EC);
+  ASSERT_FALSE(EC);
+  ASSERT_EQ(vfs::directory_iterator(), I);
+}
+
 // NOTE: in the tests below, we use '//root/' as our root directory, since it is
 // a legal *absolute* path on Windows as well as *nix.
 class VFSFromYAMLTest : public ::testing::Test {

clang/unittests/Tooling/RewriterTestContext.h

@@ -34,15 +34,20 @@ namespace clang {
 /// methods, which help with writing tests that change files.
 class RewriterTestContext {
  public:
-  RewriterTestContext()
-      : DiagOpts(new DiagnosticOptions()),
-        Diagnostics(IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs),
-                    &*DiagOpts),
-        DiagnosticPrinter(llvm::outs(), &*DiagOpts),
-        Files((FileSystemOptions())),
-        Sources(Diagnostics, Files),
-        Rewrite(Sources, Options) {
+   RewriterTestContext()
+       : DiagOpts(new DiagnosticOptions()),
+         Diagnostics(IntrusiveRefCntPtr<DiagnosticIDs>(new DiagnosticIDs),
+                     &*DiagOpts),
+         DiagnosticPrinter(llvm::outs(), &*DiagOpts),
+         InMemoryFileSystem(new vfs::InMemoryFileSystem),
+         OverlayFileSystem(
+             new vfs::OverlayFileSystem(vfs::getRealFileSystem())),
+         Files(FileSystemOptions(), OverlayFileSystem),
+         Sources(Diagnostics, Files), Rewrite(Sources, Options) {
     Diagnostics.setClient(&DiagnosticPrinter, false);
+    // FIXME: To make these tests truly in-memory, we need to overlay the
+    // builtin headers.
+    OverlayFileSystem->pushOverlay(InMemoryFileSystem);
   }
 
   ~RewriterTestContext() {}
@@ -50,9 +55,9 @@ class RewriterTestContext {
   FileID createInMemoryFile(StringRef Name, StringRef Content) {
     std::unique_ptr<llvm::MemoryBuffer> Source =
         llvm::MemoryBuffer::getMemBuffer(Content);
-    const FileEntry *Entry =
-      Files.getVirtualFile(Name, Source->getBufferSize(), 0);
-    Sources.overrideFileContents(Entry, std::move(Source));
+    InMemoryFileSystem->addFile(Name, 0, std::move(Source));
+
+    const FileEntry *Entry = Files.getFile(Name);
     assert(Entry != nullptr);
     return Sources.createFileID(Entry, SourceLocation(), SrcMgr::C_User);
   }
@@ -109,6 +114,8 @@ class RewriterTestContext {
   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
   DiagnosticsEngine Diagnostics;
   TextDiagnosticPrinter DiagnosticPrinter;
+  IntrusiveRefCntPtr<vfs::InMemoryFileSystem> InMemoryFileSystem;
+  IntrusiveRefCntPtr<vfs::OverlayFileSystem> OverlayFileSystem;
   FileManager Files;
   SourceManager Sources;
   LangOptions Options;