Introduce '-chain-include' option to specify headers that will be converted to chained PCHs in memory

without having to use multiple runs and intermediate files.

Intended for testing & debugging of chained PCH.

llvm-svn: 127339

clang/include/clang/Driver/CC1Options.td

@@ -560,6 +560,8 @@ def include_pch : Separate<"-include-pch">, MetaVarName<"<file>">,
   HelpText<"Include precompiled header file">;
 def include_pth : Separate<"-include-pth">, MetaVarName<"<file>">,
   HelpText<"Include file before parsing">;
+def chain_include : Separate<"-chain-include">, MetaVarName<"<file>">,
+  HelpText<"Include and chain a header file after turning it into PCH">;
 def preamble_bytes_EQ : Joined<"-preamble-bytes=">,
   HelpText<"Assume that the precompiled header is a precompiled preamble "
            "covering the first N bytes of the main file">;

clang/include/clang/Frontend/ASTUnit.h

@@ -540,7 +540,11 @@ public:
   /// \brief A mapping from a file name to the memory buffer that stores the
   /// remapped contents of that file.
   typedef std::pair<std::string, FilenameOrMemBuf> RemappedFile;
-  
+
+  /// \brief Create a ASTUnit. Gets ownership of the passed CompilerInvocation. 
+  static ASTUnit *create(CompilerInvocation *CI,
+                         llvm::IntrusiveRefCntPtr<Diagnostic> Diags);
+
   /// \brief Create a ASTUnit from an AST file.
   ///
   /// \param Filename - The AST file to load.
@@ -656,6 +660,11 @@ public:
   ///
   /// \returns True if an error occurred, false otherwise.
   bool Save(llvm::StringRef File);
+
+  /// \brief Serialize this translation unit with the given output stream.
+  ///
+  /// \returns True if an error occurred, false otherwise.
+  bool serialize(llvm::raw_ostream &OS);
 };
 
 } // namespace clang

clang/include/clang/Frontend/PreprocessorOptions.h

@@ -44,6 +44,9 @@ public:
   /// The implicit PCH included at the start of the translation unit, or empty.
   std::string ImplicitPCHInclude;
 
+  /// \brief Headers that will be converted to chained PCHs in memory.
+  std::vector<std::string> ChainedIncludes;
+
   /// \brief When true, disables most of the normal validation performed on
   /// precompiled headers.
   bool DisablePCHValidation;

clang/include/clang/Serialization/ASTReader.h

@@ -213,6 +213,10 @@ private:
   /// \brief The AST consumer.
   ASTConsumer *Consumer;
 
+  /// \brief AST buffers for chained PCHs created and stored in memory.
+  /// First (not depending on another) PCH in chain is in front.
+  std::deque<llvm::MemoryBuffer *> ASTBuffers;
+
   /// \brief Information that is needed for every module.
   struct PerFileData {
     PerFileData(ASTFileType Ty);
@@ -886,6 +890,13 @@ public:
   /// \brief Sets and initializes the given Context.
   void InitializeContext(ASTContext &Context);
 
+  /// \brief Set AST buffers for chained PCHs created and stored in memory.
+  /// First (not depending on another) PCH in chain is first in array.
+  void setASTMemoryBuffers(llvm::MemoryBuffer **bufs, unsigned numBufs) {
+    ASTBuffers.clear();
+    ASTBuffers.insert(ASTBuffers.begin(), bufs, bufs + numBufs);
+  }
+
   /// \brief Retrieve the name of the named (primary) AST file
   const std::string &getFileName() const { return Chain[0]->FileName; }
 

clang/include/clang/Serialization/ChainedIncludesSource.h

@@ -0,0 +1,63 @@
+//===- ChainedIncludesSource.h - Chained PCHs in Memory ---------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the ChainedIncludesSource class, which converts headers
+//  to chained PCHs in memory, mainly used for testing.
+//
+//===----------------------------------------------------------------------===//
+#ifndef LLVM_CLANG_SERIALIZATION_CHAINEDINCLUDESSOURCE_H
+#define LLVM_CLANG_SERIALIZATION_CHAINEDINCLUDESSOURCE_H
+
+#include "clang/AST/ExternalASTSource.h"
+#include <vector>
+
+namespace clang {
+  class CompilerInstance;
+
+class ChainedIncludesSource : public ExternalASTSource {
+public:
+  virtual ~ChainedIncludesSource();
+
+  static ChainedIncludesSource *create(CompilerInstance &CI);
+
+private:
+  ExternalASTSource &getFinalReader() const { return *FinalReader; }
+
+  std::vector<CompilerInstance *> CIs;
+  llvm::OwningPtr<ExternalASTSource> FinalReader;
+
+  
+protected:
+
+//===----------------------------------------------------------------------===//
+// ExternalASTSource interface.
+//===----------------------------------------------------------------------===//
+
+  virtual Decl *GetExternalDecl(uint32_t ID);
+  virtual Selector GetExternalSelector(uint32_t ID);
+  virtual uint32_t GetNumExternalSelectors();
+  virtual Stmt *GetExternalDeclStmt(uint64_t Offset);
+  virtual CXXBaseSpecifier *GetExternalCXXBaseSpecifiers(uint64_t Offset);
+  virtual DeclContextLookupResult
+  FindExternalVisibleDeclsByName(const DeclContext *DC, DeclarationName Name);
+  virtual void MaterializeVisibleDecls(const DeclContext *DC);
+  virtual bool FindExternalLexicalDecls(const DeclContext *DC,
+                                        bool (*isKindWeWant)(Decl::Kind),
+                                        llvm::SmallVectorImpl<Decl*> &Result);
+  virtual void CompleteType(TagDecl *Tag);
+  virtual void CompleteType(ObjCInterfaceDecl *Class);
+  virtual void StartedDeserializing();
+  virtual void FinishedDeserializing();
+  virtual void StartTranslationUnit(ASTConsumer *Consumer);
+  virtual void PrintStats();
+};
+
+}
+
+#endif

clang/lib/Frontend/ASTUnit.cpp

@@ -92,7 +92,8 @@ const unsigned DefaultPreambleRebuildInterval = 5;
 static llvm::sys::cas_flag ActiveASTUnitObjects;
 
 ASTUnit::ASTUnit(bool _MainFileIsAST)
-  : CaptureDiagnostics(false), MainFileIsAST(_MainFileIsAST), 
+  : OnlyLocalDecls(false), CaptureDiagnostics(false),
+    MainFileIsAST(_MainFileIsAST), 
     CompleteTranslationUnit(true), WantTiming(getenv("LIBCLANG_TIMING")),
     OwnsRemappedFileBuffers(true),
     NumStoredDiagnosticsFromDriver(0),
@@ -1522,6 +1523,19 @@ llvm::StringRef ASTUnit::getMainFileName() const {
   return Invocation->getFrontendOpts().Inputs[0].second;
 }
 
+ASTUnit *ASTUnit::create(CompilerInvocation *CI,
+                         llvm::IntrusiveRefCntPtr<Diagnostic> Diags) {
+  llvm::OwningPtr<ASTUnit> AST;
+  AST.reset(new ASTUnit(false));
+  ConfigureDiags(Diags, 0, 0, *AST, /*CaptureDiagnostics=*/false);
+  AST->Diagnostics = Diags;
+  AST->Invocation.reset(CI);
+  AST->FileMgr.reset(new FileManager(CI->getFileSystemOpts()));
+  AST->SourceMgr.reset(new SourceManager(*Diags, *AST->FileMgr));
+
+  return AST.take();
+}
+
 bool ASTUnit::LoadFromCompilerInvocation(bool PrecompilePreamble) {
   if (!Invocation)
     return true;
@@ -2148,7 +2162,16 @@ bool ASTUnit::Save(llvm::StringRef File) {
                            llvm::raw_fd_ostream::F_Binary);
   if (!ErrorInfo.empty() || Out.has_error())
     return true;
-  
+
+  serialize(Out);
+  Out.close();
+  return Out.has_error();
+}
+
+bool ASTUnit::serialize(llvm::raw_ostream &OS) {
+  if (getDiagnostics().hasErrorOccurred())
+    return true;
+
   std::vector<unsigned char> Buffer;
   llvm::BitstreamWriter Stream(Buffer);
   ASTWriter Writer(Stream);
@@ -2156,7 +2179,7 @@ bool ASTUnit::Save(llvm::StringRef File) {
   
   // Write the generated bitstream to "Out".
   if (!Buffer.empty())
-    Out.write((char *)&Buffer.front(), Buffer.size());  
-  Out.close();
-  return Out.has_error();
+    OS.write((char *)&Buffer.front(), Buffer.size());
+
+  return false;
 }

clang/lib/Frontend/CompilerInvocation.cpp

@@ -704,6 +704,10 @@ static void PreprocessorOptsToArgs(const PreprocessorOptions &Opts,
       assert(Opts.ImplicitPTHInclude == Opts.TokenCache &&
              "Unsupported option combination!");
   }
+  for (unsigned i = 0, e = Opts.ChainedIncludes.size(); i != e; ++i) {
+    Res.push_back("-chain-include");
+    Res.push_back(Opts.ChainedIncludes[i]);
+  }
   for (unsigned i = 0, e = Opts.RemappedFiles.size(); i != e; ++i) {
     Res.push_back("-remap-file");
     Res.push_back(Opts.RemappedFiles[i].first + ";" +
@@ -1562,6 +1566,12 @@ static void ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args,
       Opts.Includes.push_back(A->getValue(Args));
   }
 
+  for (arg_iterator it = Args.filtered_begin(OPT_chain_include),
+         ie = Args.filtered_end(); it != ie; ++it) {
+    const Arg *A = *it;
+    Opts.ChainedIncludes.push_back(A->getValue(Args));
+  }
+
   // Include 'altivec.h' if -faltivec option present
   if (Args.hasArg(OPT_faltivec))
     Opts.Includes.push_back("altivec.h");

clang/lib/Frontend/FrontendAction.cpp

@@ -20,6 +20,7 @@
 #include "clang/Frontend/MultiplexConsumer.h"
 #include "clang/Parse/ParseAST.h"
 #include "clang/Serialization/ASTDeserializationListener.h"
+#include "clang/Serialization/ChainedIncludesSource.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Timer.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -209,8 +210,16 @@ bool FrontendAction::BeginSourceFile(CompilerInstance &CI,
 
     CI.getASTContext().setASTMutationListener(Consumer->GetASTMutationListener());
 
-    /// Use PCH?
-    if (!CI.getPreprocessorOpts().ImplicitPCHInclude.empty()) {
+    if (!CI.getPreprocessorOpts().ChainedIncludes.empty()) {
+      // Convert headers to PCH and chain them.
+      llvm::OwningPtr<ExternalASTSource> source;
+      source.reset(ChainedIncludesSource::create(CI));
+      if (!source)
+        goto failure;
+      CI.getASTContext().setExternalSource(source);
+
+    } else if (!CI.getPreprocessorOpts().ImplicitPCHInclude.empty()) {
+      // Use PCH.
       assert(hasPCHSupport() && "This action does not have PCH support!");
       ASTDeserializationListener *DeserialListener
           = CI.getInvocation().getFrontendOpts().ChainedPCH ?

clang/lib/Serialization/ASTReader.cpp

@@ -2461,20 +2461,26 @@ ASTReader::ASTReadResult ASTReader::ReadASTCore(llvm::StringRef FileName,
     if (CurrentDir.empty()) CurrentDir = ".";
   }
 
-  // Open the AST file.
-  //
-  // FIXME: This shouldn't be here, we should just take a raw_ostream.
-  std::string ErrStr;
-  llvm::error_code ec;
-  if (FileName == "-") {
-    ec = llvm::MemoryBuffer::getSTDIN(F.Buffer);
-    if (ec)
-      ErrStr = ec.message();
-  } else
-    F.Buffer.reset(FileMgr.getBufferForFile(FileName, &ErrStr));
-  if (!F.Buffer) {
-    Error(ErrStr.c_str());
-    return IgnorePCH;
+  if (!ASTBuffers.empty()) {
+    F.Buffer.reset(ASTBuffers.front());
+    ASTBuffers.pop_front();
+    assert(F.Buffer && "Passed null buffer");
+  } else {
+    // Open the AST file.
+    //
+    // FIXME: This shouldn't be here, we should just take a raw_ostream.
+    std::string ErrStr;
+    llvm::error_code ec;
+    if (FileName == "-") {
+      ec = llvm::MemoryBuffer::getSTDIN(F.Buffer);
+      if (ec)
+        ErrStr = ec.message();
+    } else
+      F.Buffer.reset(FileMgr.getBufferForFile(FileName, &ErrStr));
+    if (!F.Buffer) {
+      Error(ErrStr.c_str());
+      return IgnorePCH;
+    }
   }
 
   // Initialize the stream

clang/lib/Serialization/ChainedIncludesSource.cpp

@@ -0,0 +1,207 @@
+//===- ChainedIncludesSource.cpp - Chained PCHs in Memory -------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines the ChainedIncludesSource class, which converts headers
+//  to chained PCHs in memory, mainly used for testing.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Serialization/ChainedIncludesSource.h"
+#include "clang/Serialization/ASTReader.h"
+#include "clang/Serialization/ASTWriter.h"
+#include "clang/Frontend/TextDiagnosticPrinter.h"
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/ASTUnit.h"
+#include "clang/Parse/ParseAST.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Basic/TargetInfo.h"
+#include "llvm/Support/MemoryBuffer.h"
+
+using namespace clang;
+
+static ASTReader *createASTReader(CompilerInstance &CI,
+                                  llvm::StringRef pchFile,
+                                  llvm::MemoryBuffer **memBufs,
+                                  unsigned numBufs,
+                             ASTDeserializationListener *deserialListener = 0) {
+  Preprocessor &PP = CI.getPreprocessor();
+  llvm::OwningPtr<ASTReader> Reader;
+  Reader.reset(new ASTReader(PP, &CI.getASTContext(), /*isysroot=*/0,
+                             /*DisableValidation=*/true));
+  Reader->setASTMemoryBuffers(memBufs, numBufs);
+  Reader->setDeserializationListener(deserialListener);
+  switch (Reader->ReadAST(pchFile, ASTReader::PCH)) {
+  case ASTReader::Success:
+    // Set the predefines buffer as suggested by the PCH reader.
+    PP.setPredefines(Reader->getSuggestedPredefines());
+    return Reader.take();
+
+  case ASTReader::Failure:
+  case ASTReader::IgnorePCH:
+    break;
+  }
+  return 0;
+}
+
+ChainedIncludesSource::~ChainedIncludesSource() {
+  for (unsigned i = 0, e = CIs.size(); i != e; ++i)
+    delete CIs[i];
+}
+
+ChainedIncludesSource *ChainedIncludesSource::create(CompilerInstance &CI) {
+
+  std::vector<std::string> &includes = CI.getPreprocessorOpts().ChainedIncludes;
+  assert(!includes.empty() && "No '-chain-include' in options!");
+
+  llvm::OwningPtr<ChainedIncludesSource> source(new ChainedIncludesSource());
+  InputKind IK = CI.getFrontendOpts().Inputs[0].first;
+
+  llvm::SmallVector<llvm::MemoryBuffer *, 4> serialBufs;
+
+  for (unsigned i = 0, e = includes.size(); i != e; ++i) {
+    bool firstInclude = (i == 0);
+    llvm::OwningPtr<CompilerInvocation> CInvok;
+    CInvok.reset(new CompilerInvocation(CI.getInvocation()));
+    
+    CInvok->getPreprocessorOpts().ChainedIncludes.clear();
+    CInvok->getPreprocessorOpts().ImplicitPCHInclude.clear();
+    CInvok->getPreprocessorOpts().ImplicitPTHInclude.clear();
+    CInvok->getPreprocessorOpts().DisablePCHValidation = true;
+    CInvok->getPreprocessorOpts().Includes.clear();
+    CInvok->getPreprocessorOpts().MacroIncludes.clear();
+    CInvok->getPreprocessorOpts().Macros.clear();
+    
+    CInvok->getFrontendOpts().Inputs.clear();
+    CInvok->getFrontendOpts().Inputs.push_back(std::make_pair(IK, includes[i]));
+
+    TextDiagnosticPrinter *DiagClient =
+      new TextDiagnosticPrinter(llvm::errs(), DiagnosticOptions());
+    llvm::IntrusiveRefCntPtr<DiagnosticIDs> DiagID(new DiagnosticIDs());
+    llvm::IntrusiveRefCntPtr<Diagnostic> Diags(new Diagnostic(DiagID,
+                                                              DiagClient));
+
+    llvm::OwningPtr<CompilerInstance> Clang(new CompilerInstance());
+    Clang->setInvocation(CInvok.take());
+    Clang->setDiagnostics(Diags.getPtr());
+    Clang->setTarget(TargetInfo::CreateTargetInfo(Clang->getDiagnostics(),
+                                                  Clang->getTargetOpts()));
+    Clang->createFileManager();
+    Clang->createSourceManager(Clang->getFileManager());
+    Clang->createPreprocessor();
+    Clang->createASTContext();
+
+    llvm::SmallVector<char, 256> serialAST;
+    llvm::raw_svector_ostream OS(serialAST);
+    llvm::OwningPtr<ASTConsumer> consumer;
+    consumer.reset(new PCHGenerator(Clang->getPreprocessor(), "-",
+                                    /*Chaining=*/!firstInclude,
+                                    /*isysroot=*/0, &OS));
+    Clang->getASTContext().setASTMutationListener(
+                                            consumer->GetASTMutationListener());
+    Clang->setASTConsumer(consumer.take());
+    Clang->createSema(/*CompleteTranslationUnit=*/false, 0);
+
+    if (firstInclude) {
+      Preprocessor &PP = Clang->getPreprocessor();
+      PP.getBuiltinInfo().InitializeBuiltins(PP.getIdentifierTable(),
+                                             PP.getLangOptions());
+    } else {
+      assert(!serialBufs.empty());
+      llvm::SmallVector<llvm::MemoryBuffer *, 4> bufs;
+      for (unsigned i = 0, e = serialBufs.size(); i != e; ++i) {
+        bufs.push_back(llvm::MemoryBuffer::getMemBufferCopy(
+                              llvm::StringRef(serialBufs[i]->getBufferStart(),
+                                              serialBufs[i]->getBufferSize())));
+      }
+      std::string pchName = includes[i-1];
+      llvm::raw_string_ostream os(pchName);
+      os << ".pch" << i-1;
+      os.flush();
+      llvm::OwningPtr<ExternalASTSource> Reader;
+      Reader.reset(createASTReader(*Clang, pchName, bufs.data(), bufs.size(),
+                      Clang->getASTConsumer().GetASTDeserializationListener()));
+      if (!Reader)
+        return 0;
+      Clang->getASTContext().setExternalSource(Reader);
+    }
+    
+    if (!Clang->InitializeSourceManager(includes[i]))
+      return 0;
+
+    ParseAST(Clang->getSema());
+    OS.flush();
+    serialBufs.push_back(
+      llvm::MemoryBuffer::getMemBufferCopy(llvm::StringRef(serialAST.data(),
+                                                           serialAST.size())));
+    source->CIs.push_back(Clang.take());
+  }
+
+  assert(!serialBufs.empty());
+  std::string pchName = includes.back() + ".pch-final";
+  llvm::OwningPtr<ASTReader> Reader;
+  Reader.reset(createASTReader(CI, pchName,
+                               serialBufs.data(), serialBufs.size()));
+  if (!Reader)
+    return 0;
+
+  source->FinalReader.reset(Reader.take());
+  return source.take();
+}
+
+//===----------------------------------------------------------------------===//
+// ExternalASTSource interface.
+//===----------------------------------------------------------------------===//
+
+Decl *ChainedIncludesSource::GetExternalDecl(uint32_t ID) {
+  return getFinalReader().GetExternalDecl(ID);
+}
+Selector ChainedIncludesSource::GetExternalSelector(uint32_t ID) {
+  return getFinalReader().GetExternalSelector(ID);
+}
+uint32_t ChainedIncludesSource::GetNumExternalSelectors() {
+  return getFinalReader().GetNumExternalSelectors();
+}
+Stmt *ChainedIncludesSource::GetExternalDeclStmt(uint64_t Offset) {
+  return getFinalReader().GetExternalDeclStmt(Offset);
+}
+CXXBaseSpecifier *
+ChainedIncludesSource::GetExternalCXXBaseSpecifiers(uint64_t Offset) {
+  return getFinalReader().GetExternalCXXBaseSpecifiers(Offset);
+}
+DeclContextLookupResult
+ChainedIncludesSource::FindExternalVisibleDeclsByName(const DeclContext *DC,
+                                                      DeclarationName Name) {
+  return getFinalReader().FindExternalVisibleDeclsByName(DC, Name);
+}
+void ChainedIncludesSource::MaterializeVisibleDecls(const DeclContext *DC) {
+  return getFinalReader().MaterializeVisibleDecls(DC);
+}
+bool ChainedIncludesSource::FindExternalLexicalDecls(const DeclContext *DC,
+                                      bool (*isKindWeWant)(Decl::Kind),
+                                      llvm::SmallVectorImpl<Decl*> &Result) {
+  return getFinalReader().FindExternalLexicalDecls(DC, isKindWeWant, Result);
+}
+void ChainedIncludesSource::CompleteType(TagDecl *Tag) {
+  return getFinalReader().CompleteType(Tag);
+}
+void ChainedIncludesSource::CompleteType(ObjCInterfaceDecl *Class) {
+  return getFinalReader().CompleteType(Class);
+}
+void ChainedIncludesSource::StartedDeserializing() {
+  return getFinalReader().StartedDeserializing();
+}
+void ChainedIncludesSource::FinishedDeserializing() {
+  return getFinalReader().FinishedDeserializing();
+}
+void ChainedIncludesSource::StartTranslationUnit(ASTConsumer *Consumer) {
+  return getFinalReader().StartTranslationUnit(Consumer);
+}
+void ChainedIncludesSource::PrintStats() {
+  return getFinalReader().PrintStats();
+}

clang/test/PCH/chain-cxx.cpp

@@ -4,9 +4,7 @@
 // RUN: %clang_cc1 -fsyntax-only -verify -include %s -include %s %s
 
 // With PCH
-// RUN: %clang_cc1 -x c++-header -emit-pch -o %t1 %s
-// RUN: %clang_cc1 -x c++-header -emit-pch -o %t2 %s -include-pch %t1 -chained-pch
-// RUN: %clang_cc1 -fsyntax-only -verify -include-pch %t2 %s
+// RUN: %clang_cc1 -fsyntax-only -verify %s -chain-include %s -chain-include %s
 
 #ifndef HEADER1
 #define HEADER1