[modules] Add an experimental -cc1 feature to embed the contents of an input

file in the .pcm files. This allows a smaller set of files to be sent to a
remote build worker when building with explicit modules (for instance, module
map files need not be sent along with the corresponding precompiled modules).

This doesn't actually make the embedded files visible to header search, so
it's not useful as a packaging format for public header files.

llvm-svn: 245028

clang/include/clang/Basic/DiagnosticFrontendKinds.td

@@ -200,6 +200,9 @@ def remark_module_build : Remark<"building module '%0' as '%1'">,
   InGroup<ModuleBuild>;
 def remark_module_build_done : Remark<"finished building module '%0'">,
   InGroup<ModuleBuild>;
+def err_modules_embed_file_not_found :
+  Error<"file '%0' specified by '-fmodules-embed-file=' not found">,
+  DefaultFatal;
 
 def err_conflicting_module_names : Error<
   "conflicting module names specified: '-fmodule-name=%0' and "

clang/include/clang/Basic/SourceManager.h

@@ -851,6 +851,13 @@ public:
   /// This should be called before parsing has begun.
   void disableFileContentsOverride(const FileEntry *File);
 
+  /// \brief Request that the contents of the given source file are written
+  /// to a created module file if they are used in this compilation. This
+  /// removes the requirement that the file still exist when the module is used
+  /// (but does not make the file visible to header search and the like when
+  /// the module is used).
+  void embedFileContentsInModule(const FileEntry *SourceFile);
+
   //===--------------------------------------------------------------------===//
   // FileID manipulation methods.
   //===--------------------------------------------------------------------===//

clang/include/clang/Driver/CC1Options.td

@@ -368,6 +368,10 @@ def fmodule_map_file_home_is_cwd : Flag<["-"], "fmodule-map-file-home-is-cwd">,
 def fmodule_feature : Separate<["-"], "fmodule-feature">,
   MetaVarName<"<feature>">,
   HelpText<"Enable <feature> in module map requires declarations">;
+def fmodules_embed_file_EQ : Joined<["-"], "fmodules-embed-file=">,
+  MetaVarName<"<file>">,
+  HelpText<"Embed the contents of the specified file into the module file "
+           "being compiled.">;
 def fmodules_local_submodule_visibility :
   Flag<["-"], "fmodules-local-submodule-visibility">,
   HelpText<"Enforce name visibility rules across submodules of the same "

clang/include/clang/Frontend/FrontendOptions.h

@@ -241,6 +241,9 @@ public:
   /// processing the input.
   std::vector<std::string> ModuleFiles;
 
+  /// \brief The list of files to embed into the compiled module file.
+  std::vector<std::string> ModulesEmbedFiles;
+
   /// \brief The list of AST files to merge.
   std::vector<std::string> ASTMergeFiles;
 

clang/lib/Basic/SourceManager.cpp

@@ -678,6 +678,13 @@ void SourceManager::disableFileContentsOverride(const FileEntry *File) {
   OverriddenFilesInfo->OverriddenFilesWithBuffer.erase(File);
 }
 
+void SourceManager::embedFileContentsInModule(const FileEntry *File) {
+  // We model an embedded file as a file whose buffer has been overridden
+  // by its contents as they are now.
+  const SrcMgr::ContentCache *CC = getOrCreateContentCache(File);
+  const_cast<SrcMgr::ContentCache *>(CC)->BufferOverridden = true;
+}
+
 StringRef SourceManager::getBufferData(FileID FID, bool *Invalid) const {
   bool MyInvalid = false;
   const SLocEntry &SLoc = getSLocEntry(FID, &MyInvalid);

clang/lib/Frontend/CompilerInvocation.cpp

@@ -960,6 +960,7 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
   Opts.GenerateGlobalModuleIndex = Opts.UseGlobalModuleIndex;
   Opts.ModuleMapFiles = Args.getAllArgValues(OPT_fmodule_map_file);
   Opts.ModuleFiles = Args.getAllArgValues(OPT_fmodule_file);
+  Opts.ModulesEmbedFiles = Args.getAllArgValues(OPT_fmodules_embed_file_EQ);
 
   Opts.CodeCompleteOpts.IncludeMacros
     = Args.hasArg(OPT_code_completion_macros);

clang/lib/Frontend/FrontendActions.cpp

@@ -268,8 +268,9 @@ collectModuleHeaderIncludes(const LangOptions &LangOpts, FileManager &FileMgr,
 
 bool GenerateModuleAction::BeginSourceFileAction(CompilerInstance &CI, 
                                                  StringRef Filename) {
-  // Find the module map file.  
-  const FileEntry *ModuleMap = CI.getFileManager().getFile(Filename);
+  // Find the module map file.
+  const FileEntry *ModuleMap =
+      CI.getFileManager().getFile(Filename, /*openFile*/true);
   if (!ModuleMap)  {
     CI.getDiagnostics().Report(diag::err_module_map_not_found)
       << Filename;
@@ -291,6 +292,14 @@ bool GenerateModuleAction::BeginSourceFileAction(CompilerInstance &CI,
     return false;
   }
 
+  // Set up embedding for any specified files.
+  for (const auto &F : CI.getFrontendOpts().ModulesEmbedFiles) {
+    if (const auto *FE = CI.getFileManager().getFile(F, /*openFile*/true))
+      CI.getSourceManager().embedFileContentsInModule(FE);
+    else
+      CI.getDiagnostics().Report(diag::err_modules_embed_file_not_found) << F;
+  }
+
   // If we're being run from the command-line, the module build stack will not
   // have been filled in yet, so complete it now in order to allow us to detect
   // module cycles.

clang/lib/Serialization/ASTReader.cpp

@@ -1023,7 +1023,8 @@ bool ASTReader::ReadVisibleDeclContextStorage(ModuleFile &M,
 
 void ASTReader::Error(StringRef Msg) {
   Error(diag::err_fe_pch_malformed, Msg);
-  if (Context.getLangOpts().Modules && !Diags.isDiagnosticInFlight()) {
+  if (Context.getLangOpts().Modules && !Diags.isDiagnosticInFlight() &&
+      !PP.getHeaderSearchInfo().getModuleCachePath().empty()) {
     Diag(diag::note_module_cache_path)
       << PP.getHeaderSearchInfo().getModuleCachePath();
   }

clang/test/Modules/explicit-build-missing-files.cpp

@@ -1,19 +1,26 @@
 // RUN: rm -rf %t
 // RUN: mkdir %t
-// RUN: echo 'extern int a;' > %t/a.h
-// RUN: echo 'extern int b; template<typename T> int b2 = T::error;' > %t/b.h
-// RUN: echo 'module a { header "a.h" header "b.h" }' > %t/modulemap
+// RUN: echo 'extern int a; template<typename T> int a2 = T::error;' > %t/a.h
+// RUN: echo 'extern int b;' > %t/b.h
+// RUN: echo 'extern int c = 0;' > %t/c.h
+// RUN: echo 'module a { header "a.h" header "b.h" header "c.h" }' > %t/modulemap
+// RUN: echo 'module other {}' > %t/other.modulemap
 
 // We lazily check that the files referenced by an explicitly-specified .pcm
 // file exist. Test this by removing files and ensuring that the compilation
 // still succeeds.
 //
-// RUN: %clang_cc1 -fmodules -I %t -emit-module -fmodule-name=a -x c++ %t/modulemap -o %t/a.pcm
+// RUN: %clang_cc1 -fmodules -I %t -emit-module -fmodule-name=a -x c++ %t/modulemap -o %t/a.pcm \
+// RUN:            -fmodule-map-file=%t/other.modulemap \
+// RUN:            -fmodules-embed-file=%t/modulemap -fmodules-embed-file=%t/other.modulemap
 // RUN: %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s
 // RUN: not %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s -DERRORS 2>&1 | FileCheck %s
 // RUN: rm %t/modulemap
 // RUN: %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s
 // RUN: not %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s -DERRORS 2>&1 | FileCheck %s
+// RUN: rm %t/other.modulemap
+// RUN: %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s
+// RUN: not %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s -DERRORS 2>&1 | FileCheck %s
 // RUN: rm %t/b.h
 // RUN: %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s
 // RUN: not %clang_cc1 -fmodules -I %t -fmodule-file=%t/a.pcm %s -DERRORS 2>&1 | FileCheck %s --check-prefix=MISSING-B
@@ -24,10 +31,14 @@
 int x = b;
 
 #ifdef ERRORS
-int y = b2<int>;
+int y = a2<int>;
 // CHECK: In module 'a':
-// CHECK-NEXT: b.h:1:45: error:
+// CHECK-NEXT: a.h:1:45: error:
 
 // MISSING-B: could not find file '{{.*}}b.h'
 // MISSING-B-NOT: please delete the module cache
 #endif
+
+// RUN: not %clang_cc1 -fmodules -I %t -emit-module -fmodule-name=a -x c++ /dev/null -o %t/a.pcm \
+// RUN:                -fmodules-embed-file=%t/does-not-exist 2>&1 | FileCheck %s --check-prefix=MISSING-EMBED
+// MISSING-EMBED: fatal error: file '{{.*}}does-not-exist' specified by '-fmodules-embed-file=' not found