diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td index 282dd711719b..7a71285482f6 100644 --- a/clang/include/clang/Basic/DiagnosticDriverKinds.td +++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td @@ -193,6 +193,10 @@ def err_drv_modules_validate_once_requires_timestamp : Error< "option '-fmodules-validate-once-per-build-session' requires " "'-fbuild-session-timestamp=' or '-fbuild-session-file='">; +def err_test_module_file_extension_format : Error< + "-ftest-module-file-extension argument '%0' is not of the required form " + "'blockname:major:minor:hashed:user info'">; + def warn_drv_invoking_fallback : Warning<"falling back to %0">, InGroup; diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index 28fc3059b258..033834bc505d 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -204,6 +204,10 @@ def err_modules_embed_file_not_found : Error<"file '%0' specified by '-fmodules-embed-file=' not found">, DefaultFatal; +def err_test_module_file_extension_version : Error< + "test module file extension '%0' has different version (%1.%2) than expected " + "(%3.%4)">; + def err_conflicting_module_names : Error< "conflicting module names specified: '-fmodule-name=%0' and " "'-fmodule-implementation-of %1'">; diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index 042244125b8d..4b94b186cd6a 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -253,6 +253,7 @@ def MismatchedTags : DiagGroup<"mismatched-tags">; def MissingFieldInitializers : DiagGroup<"missing-field-initializers">; def ModuleBuild : DiagGroup<"module-build">; def ModuleConflict : DiagGroup<"module-conflict">; +def ModuleFileExtension : DiagGroup<"module-file-extension">; def NewlineEOF : DiagGroup<"newline-eof">; def Nullability : DiagGroup<"nullability">; def NullabilityDeclSpec : DiagGroup<"nullability-declspec">; diff --git a/clang/include/clang/Basic/DiagnosticSerializationKinds.td b/clang/include/clang/Basic/DiagnosticSerializationKinds.td index 272cc7bf6b01..16c77435558f 100644 --- a/clang/include/clang/Basic/DiagnosticSerializationKinds.td +++ b/clang/include/clang/Basic/DiagnosticSerializationKinds.td @@ -115,6 +115,10 @@ def warn_module_uses_date_time : Warning< "%select{precompiled header|module}0 uses __DATE__ or __TIME__">, InGroup>; +def warn_duplicate_module_file_extension : Warning< + "duplicate module file extension block name '%0'">, + InGroup; + } // let CategoryName } // let Component diff --git a/clang/include/clang/Driver/CC1Options.td b/clang/include/clang/Driver/CC1Options.td index 7a1a9edf2894..5bf175e3bd99 100644 --- a/clang/include/clang/Driver/CC1Options.td +++ b/clang/include/clang/Driver/CC1Options.td @@ -392,6 +392,10 @@ def fno_modules_hide_internal_linkage : Flag<["-"], "fno-modules-hide-internal-linkage">, HelpText<"Make all declarations visible to redeclaration lookup, " "even if they have internal linkage.">; +def ftest_module_file_extension_EQ : + Joined<["-"], "ftest-module-file-extension=">, + HelpText<"introduce a module file extension for testing purposes. " + "The argument is parsed as blockname:major:minor:hashed:user info">; def fconcepts_ts : Flag<["-"], "fconcepts-ts">, HelpText<"Enable C++ Extensions for Concepts.">; diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index 5c29f753db92..83eed2cdc592 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -655,6 +655,7 @@ public: StringRef Path, StringRef Sysroot, bool DisablePCHValidation, bool AllowPCHWithCompilerErrors, Preprocessor &PP, ASTContext &Context, const PCHContainerReader &PCHContainerRdr, + ArrayRef> Extensions, void *DeserializationListener, bool OwnDeserializationListener, bool Preamble, bool UseGlobalModuleIndex); diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h index 969f0c0a25e6..8c841e094245 100644 --- a/clang/include/clang/Frontend/FrontendOptions.h +++ b/clang/include/clang/Frontend/FrontendOptions.h @@ -11,6 +11,7 @@ #define LLVM_CLANG_FRONTEND_FRONTENDOPTIONS_H #include "clang/Frontend/CommandLineSourceLoc.h" +#include "clang/Serialization/ModuleFileExtension.h" #include "clang/Sema/CodeCompleteOptions.h" #include "llvm/ADT/StringRef.h" #include @@ -236,6 +237,9 @@ public: /// The list of plugins to load. std::vector Plugins; + /// The list of module file extensions. + std::vector> ModuleFileExtensions; + /// \brief The list of module map files to load before processing the input. std::vector ModuleMapFiles; diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h index 5d0669f4bfcf..7af3c6285561 100644 --- a/clang/include/clang/Serialization/ASTBitCodes.h +++ b/clang/include/clang/Serialization/ASTBitCodes.h @@ -241,6 +241,9 @@ namespace clang { /// /// This block is part of the control block. OPTIONS_BLOCK_ID, + + /// \brief A block containing a module file extension. + EXTENSION_BLOCK_ID, }; /// \brief Record types that occur within the control block. @@ -310,6 +313,15 @@ namespace clang { PREPROCESSOR_OPTIONS, }; + /// \brief Record code for extension blocks. + enum ExtensionBlockRecordTypes { + /// Metadata describing this particular extension. + EXTENSION_METADATA = 1, + + /// The first record ID allocated to the extensions themselves. + FIRST_EXTENSION_RECORD_ID = 4 + }; + /// \brief Record types that occur within the input-files block /// inside the control block. enum InputFileRecordTypes { diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index be37186b3eed..eb613802e3bf 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -30,6 +30,7 @@ #include "clang/Serialization/ASTBitCodes.h" #include "clang/Serialization/ContinuousRangeMap.h" #include "clang/Serialization/Module.h" +#include "clang/Serialization/ModuleFileExtension.h" #include "clang/Serialization/ModuleManager.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/APInt.h" @@ -38,6 +39,7 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/TinyPtrVector.h" #include "llvm/Bitcode/BitstreamReader.h" @@ -205,6 +207,10 @@ public: /// \brief If needsImportVisitation returns \c true, this is called for each /// AST file imported by this AST file. virtual void visitImport(StringRef Filename) {} + + /// Indicates that a particular module file extension has been read. + virtual void readModuleFileExtension( + const ModuleFileExtensionMetadata &Metadata) {} }; /// \brief Simple wrapper class for chaining listeners. @@ -247,6 +253,8 @@ public: serialization::ModuleKind Kind) override; bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden, bool isExplicitModule) override; + void readModuleFileExtension( + const ModuleFileExtensionMetadata &Metadata) override; }; /// \brief ASTReaderListener implementation to validate the information of @@ -382,6 +390,9 @@ private: /// \brief The module manager which manages modules and their dependencies ModuleManager ModuleMgr; + /// A mapping from extension block names to module file extensions. + llvm::StringMap> ModuleFileExtensions; + /// \brief A timer used to track the time spent deserializing. std::unique_ptr ReadTimer; @@ -1122,6 +1133,7 @@ private: bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener, std::string &SuggestedPredefines); ASTReadResult ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities); + ASTReadResult ReadExtensionBlock(ModuleFile &F); bool ParseLineTable(ModuleFile &F, const RecordData &Record); bool ReadSourceManagerBlock(ModuleFile &F); llvm::BitstreamCursor &SLocCursorForID(int ID); @@ -1273,6 +1285,9 @@ public: /// \param PCHContainerRdr the PCHContainerOperations to use for loading and /// creating modules. /// + /// \param Extensions the list of module file extensions that can be loaded + /// from the AST files. + /// /// \param isysroot If non-NULL, the system include path specified by the /// user. This is only used with relocatable PCH files. If non-NULL, /// a relocatable PCH file will use the default path "/". @@ -1299,6 +1314,7 @@ public: /// deserializing. ASTReader(Preprocessor &PP, ASTContext &Context, const PCHContainerReader &PCHContainerRdr, + ArrayRef> Extensions, StringRef isysroot = "", bool DisableValidation = false, bool AllowASTWithCompilerErrors = false, bool AllowConfigurationMismatch = false, @@ -1309,6 +1325,7 @@ public: SourceManager &getSourceManager() const { return SourceMgr; } FileManager &getFileManager() const { return FileMgr; } + DiagnosticsEngine &getDiags() const { return Diags; } /// \brief Flags that indicate what kind of AST loading failures the client /// of the AST reader can directly handle. @@ -1476,6 +1493,7 @@ public: static bool readASTFileControlBlock(StringRef Filename, FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, + bool FindModuleFileExtensions, ASTReaderListener &Listener); /// \brief Determine whether the given AST file is acceptable to load into a diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h index 45e29921a537..86dbceef4c47 100644 --- a/clang/include/clang/Serialization/ASTWriter.h +++ b/clang/include/clang/Serialization/ASTWriter.h @@ -58,6 +58,8 @@ class OpaqueValueExpr; class OpenCLOptions; class ASTReader; class Module; +class ModuleFileExtension; +class ModuleFileExtensionWriter; class PreprocessedEntity; class PreprocessingRecord; class Preprocessor; @@ -491,7 +493,11 @@ private: /// \brief A mapping from each known submodule to its ID number, which will /// be a positive integer. llvm::DenseMap SubmoduleIDs; - + + /// \brief A list of the module file extension writers. + std::vector> + ModuleFileExtensionWriters; + /// \brief Retrieve or create a submodule ID for this module. unsigned getSubmoduleID(Module *Mod); @@ -545,6 +551,7 @@ private: void WriteObjCCategories(); void WriteLateParsedTemplates(Sema &SemaRef); void WriteOptimizePragmaOptions(Sema &SemaRef); + void WriteModuleFileExtension(ModuleFileExtensionWriter &Writer); unsigned DeclParmVarAbbrev; unsigned DeclContextLexicalAbbrev; @@ -574,7 +581,9 @@ private: public: /// \brief Create a new precompiled header writer that outputs to /// the given bitstream. - ASTWriter(llvm::BitstreamWriter &Stream, bool IncludeTimestamps = true); + ASTWriter(llvm::BitstreamWriter &Stream, + ArrayRef> Extensions, + bool IncludeTimestamps = true); ~ASTWriter() override; const LangOptions &getLangOpts() const; @@ -890,11 +899,13 @@ protected: SmallVectorImpl &getPCH() const { return Buffer->Data; } public: - PCHGenerator(const Preprocessor &PP, StringRef OutputFile, - clang::Module *Module, StringRef isysroot, - std::shared_ptr Buffer, - bool AllowASTWithErrors = false, - bool IncludeTimestamps = true); + PCHGenerator( + const Preprocessor &PP, StringRef OutputFile, + clang::Module *Module, StringRef isysroot, + std::shared_ptr Buffer, + ArrayRef> Extensions, + bool AllowASTWithErrors = false, + bool IncludeTimestamps = true); ~PCHGenerator() override; void InitializeSema(Sema &S) override { SemaPtr = &S; } void HandleTranslationUnit(ASTContext &Ctx) override; diff --git a/clang/include/clang/Serialization/Module.h b/clang/include/clang/Serialization/Module.h index 1b9560188cb0..3fdb33f495d6 100644 --- a/clang/include/clang/Serialization/Module.h +++ b/clang/include/clang/Serialization/Module.h @@ -18,6 +18,7 @@ #include "clang/Basic/SourceLocation.h" #include "clang/Serialization/ASTBitCodes.h" #include "clang/Serialization/ContinuousRangeMap.h" +#include "clang/Serialization/ModuleFileExtension.h" #include "llvm/ADT/SetVector.h" #include "llvm/Bitcode/BitstreamReader.h" #include "llvm/Support/Endian.h" @@ -194,6 +195,10 @@ public: /// \brief The first source location in this module. SourceLocation FirstLoc; + /// The list of extension readers that are attached to this module + /// file. + std::vector> ExtensionReaders; + // === Input Files === /// \brief The cursor to the start of the input-files block. llvm::BitstreamCursor InputFilesCursor; diff --git a/clang/include/clang/Serialization/ModuleFileExtension.h b/clang/include/clang/Serialization/ModuleFileExtension.h new file mode 100644 index 000000000000..a8f8386e6492 --- /dev/null +++ b/clang/include/clang/Serialization/ModuleFileExtension.h @@ -0,0 +1,147 @@ +//===-- ModuleFileExtension.h - Module File Extensions ----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_SERIALIZATION_MODULEFILEEXTENSION_H +#define LLVM_CLANG_SERIALIZATION_MODULEFILEEXTENSION_H + +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include +#include + +namespace llvm { +class BitstreamCursor; +class BitstreamWriter; +class hash_code; +class raw_ostream; +} + +namespace clang { + +class ASTReader; +class ASTWriter; + +namespace serialization { + class ModuleFile; +} // end namespace serialization + +/// Metadata for a module file extension. +struct ModuleFileExtensionMetadata { + /// The name used to identify this particular extension block within + /// the resulting module file. It should be unique to the particular + /// extension, because this name will be used to match the name of + /// an extension block to the appropriate reader. + std::string BlockName; + + /// The major version of the extension data. + unsigned MajorVersion; + + /// The minor version of the extension data. + unsigned MinorVersion; + + /// A string containing additional user information that will be + /// stored with the metadata. + std::string UserInfo; +}; + +class ModuleFileExtensionReader; +class ModuleFileExtensionWriter; + +/// An abstract superclass that describes a custom extension to the +/// module/precompiled header file format. +/// +/// A module file extension can introduce additional information into +/// compiled module files (.pcm) and precompiled headers (.pch) via a +/// custom writer that can then be accessed via a custom reader when +/// the module file or precompiled header is loaded. +class ModuleFileExtension : public llvm::RefCountedBase { +public: + virtual ~ModuleFileExtension(); + + /// Retrieves the metadata for this module file extension. + virtual ModuleFileExtensionMetadata getExtensionMetadata() const = 0; + + /// Hash information about the presence of this extension into the + /// module hash code. + /// + /// The module hash code is used to distinguish different variants + /// of a module that are incompatible. If the presence, absence, or + /// version of the module file extension should force the creation + /// of a separate set of module files, override this method to + /// combine that distinguishing information into the module hash + /// code. + /// + /// The default implementation of this function simply returns the + /// hash code as given, so the presence/absence of this extension + /// does not distinguish module files. + virtual llvm::hash_code hashExtension(llvm::hash_code Code) const; + + /// Create a new module file extension writer, which will be + /// responsible for writing the extension contents into a particular + /// module file. + virtual std::unique_ptr + createExtensionWriter(ASTWriter &Writer) = 0; + + /// Create a new module file extension reader, given the + /// metadata read from the block and the cursor into the extension + /// block. + /// + /// May return null to indicate that an extension block with the + /// given metadata cannot be read. + virtual std::unique_ptr + createExtensionReader(const ModuleFileExtensionMetadata &Metadata, + ASTReader &Reader, serialization::ModuleFile &Mod, + const llvm::BitstreamCursor &Stream) = 0; +}; + +/// Abstract base class that writes a module file extension block into +/// a module file. +class ModuleFileExtensionWriter { + ModuleFileExtension *Extension; + +protected: + ModuleFileExtensionWriter(ModuleFileExtension *Extension) + : Extension(Extension) { } + +public: + virtual ~ModuleFileExtensionWriter(); + + /// Retrieve the module file extension with which this writer is + /// associated. + ModuleFileExtension *getExtension() const { return Extension; } + + /// Write the contents of the extension block into the given bitstream. + /// + /// Responsible for writing the contents of the extension into the + /// given stream. All of the contents should be written into custom + /// records with IDs >= FIRST_EXTENSION_RECORD_ID. + virtual void writeExtensionContents(llvm::BitstreamWriter &Stream) = 0; +}; + +/// Abstract base class that reads a module file extension block from +/// a module file. +/// +/// Subclasses +class ModuleFileExtensionReader { + ModuleFileExtension *Extension; + +protected: + ModuleFileExtensionReader(ModuleFileExtension *Extension) + : Extension(Extension) { } + +public: + /// Retrieve the module file extension with which this reader is + /// associated. + ModuleFileExtension *getExtension() const { return Extension; } + + virtual ~ModuleFileExtensionReader(); +}; + +} // end namespace clang + +#endif // LLVM_CLANG_FRONTEND_MODULEFILEEXTENSION_H diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp index d3d7185ea3ad..6b89447305fe 100644 --- a/clang/lib/Frontend/ASTUnit.cpp +++ b/clang/lib/Frontend/ASTUnit.cpp @@ -186,7 +186,7 @@ struct ASTUnit::ASTWriterData { llvm::BitstreamWriter Stream; ASTWriter Writer; - ASTWriterData() : Stream(Buffer), Writer(Stream) { } + ASTWriterData() : Stream(Buffer), Writer(Stream, { }) { } }; void ASTUnit::clearFileLevelDecls() { @@ -709,7 +709,7 @@ std::unique_ptr ASTUnit::LoadFromASTFile( bool disableValid = false; if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION")) disableValid = true; - AST->Reader = new ASTReader(PP, Context, PCHContainerRdr, + AST->Reader = new ASTReader(PP, Context, PCHContainerRdr, { }, /*isysroot=*/"", /*DisableValidation=*/disableValid, AllowPCHWithCompilerErrors); @@ -927,6 +927,7 @@ public: const Preprocessor &PP, StringRef isysroot, raw_ostream *Out) : PCHGenerator(PP, "", nullptr, isysroot, std::make_shared(), + ArrayRef>(), /*AllowASTWithErrors=*/true), Unit(Unit), Hash(Unit.getCurrentTopLevelHashValue()), Action(Action), Out(Out) { @@ -2500,7 +2501,7 @@ bool ASTUnit::serialize(raw_ostream &OS) { SmallString<128> Buffer; llvm::BitstreamWriter Stream(Buffer); - ASTWriter Writer(Stream); + ASTWriter Writer(Stream, { }); return serializeUnit(Writer, Buffer, getSema(), hasErrors, OS); } diff --git a/clang/lib/Frontend/CMakeLists.txt b/clang/lib/Frontend/CMakeLists.txt index 9a3e459640a5..af42a905b387 100644 --- a/clang/lib/Frontend/CMakeLists.txt +++ b/clang/lib/Frontend/CMakeLists.txt @@ -35,6 +35,7 @@ add_clang_library(clangFrontend PrintPreprocessedOutput.cpp SerializedDiagnosticPrinter.cpp SerializedDiagnosticReader.cpp + TestModuleFileExtension.cpp TextDiagnostic.cpp TextDiagnosticBuffer.cpp TextDiagnosticPrinter.cpp diff --git a/clang/lib/Frontend/ChainedIncludesSource.cpp b/clang/lib/Frontend/ChainedIncludesSource.cpp index d36162b8dab6..1c1081fbe08e 100644 --- a/clang/lib/Frontend/ChainedIncludesSource.cpp +++ b/clang/lib/Frontend/ChainedIncludesSource.cpp @@ -82,6 +82,7 @@ createASTReader(CompilerInstance &CI, StringRef pchFile, std::unique_ptr Reader; Reader.reset(new ASTReader(PP, CI.getASTContext(), CI.getPCHContainerReader(), + /*Extensions=*/{ }, /*isysroot=*/"", /*DisableValidation=*/true)); for (unsigned ti = 0; ti < bufNames.size(); ++ti) { StringRef sr(bufNames[ti]); @@ -160,8 +161,10 @@ IntrusiveRefCntPtr clang::createChainedIncludesSource( Clang->createASTContext(); auto Buffer = std::make_shared(); + ArrayRef> Extensions; auto consumer = llvm::make_unique( - Clang->getPreprocessor(), "-", nullptr, /*isysroot=*/"", Buffer); + Clang->getPreprocessor(), "-", nullptr, /*isysroot=*/"", Buffer, + Extensions); Clang->getASTContext().setASTMutationListener( consumer->GetASTMutationListener()); Clang->setASTConsumer(std::move(consumer)); diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 8220a6e49257..c3f19a3a00d3 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -408,7 +408,9 @@ void CompilerInstance::createPCHExternalASTSource( ModuleManager = createPCHExternalASTSource( Path, getHeaderSearchOpts().Sysroot, DisablePCHValidation, AllowPCHWithCompilerErrors, getPreprocessor(), getASTContext(), - getPCHContainerReader(), DeserializationListener, + getPCHContainerReader(), + getFrontendOpts().ModuleFileExtensions, + DeserializationListener, OwnDeserializationListener, Preamble, getFrontendOpts().UseGlobalModuleIndex); } @@ -417,15 +419,16 @@ IntrusiveRefCntPtr CompilerInstance::createPCHExternalASTSource( StringRef Path, StringRef Sysroot, bool DisablePCHValidation, bool AllowPCHWithCompilerErrors, Preprocessor &PP, ASTContext &Context, const PCHContainerReader &PCHContainerRdr, + ArrayRef> Extensions, void *DeserializationListener, bool OwnDeserializationListener, bool Preamble, bool UseGlobalModuleIndex) { HeaderSearchOptions &HSOpts = PP.getHeaderSearchInfo().getHeaderSearchOpts(); IntrusiveRefCntPtr Reader(new ASTReader( - PP, Context, PCHContainerRdr, Sysroot.empty() ? "" : Sysroot.data(), - DisablePCHValidation, AllowPCHWithCompilerErrors, - /*AllowConfigurationMismatch*/ false, HSOpts.ModulesValidateSystemHeaders, - UseGlobalModuleIndex)); + PP, Context, PCHContainerRdr, Extensions, + Sysroot.empty() ? "" : Sysroot.data(), DisablePCHValidation, + AllowPCHWithCompilerErrors, /*AllowConfigurationMismatch*/ false, + HSOpts.ModulesValidateSystemHeaders, UseGlobalModuleIndex)); // We need the external source to be set up before we read the AST, because // eagerly-deserialized declarations may use it. @@ -1267,6 +1270,7 @@ void CompilerInstance::createModuleManager() { *FrontendTimerGroup); ModuleManager = new ASTReader( getPreprocessor(), getASTContext(), getPCHContainerReader(), + getFrontendOpts().ModuleFileExtensions, Sysroot.empty() ? "" : Sysroot.c_str(), PPOpts.DisablePCHValidation, /*AllowASTWithCompilerErrors=*/false, /*AllowConfigurationMismatch=*/false, diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 0c1c4eaefa99..468cc5746cec 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -7,6 +7,7 @@ // //===----------------------------------------------------------------------===// +#include "TestModuleFileExtension.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/Version.h" @@ -19,6 +20,7 @@ #include "clang/Frontend/Utils.h" #include "clang/Lex/HeaderSearchOptions.h" #include "clang/Serialization/ASTReader.h" +#include "clang/Serialization/ModuleFileExtension.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" @@ -832,6 +834,30 @@ static void ParseFileSystemArgs(FileSystemOptions &Opts, ArgList &Args) { Opts.WorkingDir = Args.getLastArgValue(OPT_working_directory); } +/// Parse the argument to the -ftest-module-file-extension +/// command-line argument. +/// +/// \returns true on error, false on success. +static bool parseTestModuleFileExtensionArg(StringRef Arg, + std::string &BlockName, + unsigned &MajorVersion, + unsigned &MinorVersion, + bool &Hashed, + std::string &UserInfo) { + SmallVector Args; + Arg.split(Args, ':', 5); + if (Args.size() < 5) + return true; + + BlockName = Args[0]; + if (Args[1].getAsInteger(10, MajorVersion)) return true; + if (Args[2].getAsInteger(10, MinorVersion)) return true; + if (Args[3].getAsInteger(2, Hashed)) return true; + if (Args.size() > 4) + UserInfo = Args[4]; + return false; +} + static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, DiagnosticsEngine &Diags) { using namespace options; @@ -924,6 +950,26 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, if (A->getValue(0) == Opts.AddPluginActions[i]) Opts.AddPluginArgs[i].emplace_back(A->getValue(1)); + for (const std::string &Arg : + Args.getAllArgValues(OPT_ftest_module_file_extension_EQ)) { + std::string BlockName; + unsigned MajorVersion; + unsigned MinorVersion; + bool Hashed; + std::string UserInfo; + if (parseTestModuleFileExtensionArg(Arg, BlockName, MajorVersion, + MinorVersion, Hashed, UserInfo)) { + Diags.Report(diag::err_test_module_file_extension_format) << Arg; + + continue; + } + + // Add the testing module file extension. + Opts.ModuleFileExtensions.push_back( + new TestModuleFileExtension(BlockName, MajorVersion, MinorVersion, + Hashed, UserInfo)); + } + if (const Arg *A = Args.getLastArg(OPT_code_completion_at)) { Opts.CodeCompletionAt = ParsedSourceLocation::FromString(A->getValue()); @@ -2076,6 +2122,12 @@ std::string CompilerInvocation::getModuleHash() const { // Extend the signature with the user build path. code = hash_combine(code, hsOpts.ModuleUserBuildPath); + // Extend the signature with the module file extensions. + const FrontendOptions &frontendOpts = getFrontendOpts(); + for (auto ext : frontendOpts.ModuleFileExtensions) { + code = ext->hashExtension(code); + } + // Darwin-specific hack: if we have a sysroot, use the contents and // modification time of // $sysroot/System/Library/CoreServices/SystemVersion.plist diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index 6ba4b134c38e..865fb474020f 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -91,7 +91,8 @@ GeneratePCHAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { auto Buffer = std::make_shared(); std::vector> Consumers; Consumers.push_back(llvm::make_unique( - CI.getPreprocessor(), OutputFile, nullptr, Sysroot, Buffer)); + CI.getPreprocessor(), OutputFile, nullptr, Sysroot, + Buffer, CI.getFrontendOpts().ModuleFileExtensions)); Consumers.push_back(CI.getPCHContainerWriter().CreatePCHContainerGenerator( CI, InFile, OutputFile, OS, Buffer)); @@ -133,10 +134,13 @@ GenerateModuleAction::CreateASTConsumer(CompilerInstance &CI, auto Buffer = std::make_shared(); std::vector> Consumers; + Consumers.push_back(llvm::make_unique( - CI.getPreprocessor(), OutputFile, Module, Sysroot, Buffer, - /*AllowASTWithErrors*/false, - /*IncludeTimestamps*/+CI.getFrontendOpts().BuildingImplicitModule)); + CI.getPreprocessor(), OutputFile, Module, Sysroot, + Buffer, CI.getFrontendOpts().ModuleFileExtensions, + /*AllowASTWithErrors=*/false, + /*IncludeTimestamps=*/ + +CI.getFrontendOpts().BuildingImplicitModule)); Consumers.push_back(CI.getPCHContainerWriter().CreatePCHContainerGenerator( CI, InFile, OutputFile, OS, Buffer)); return llvm::make_unique(std::move(Consumers)); @@ -421,6 +425,7 @@ void VerifyPCHAction::ExecuteAction() { const std::string &Sysroot = CI.getHeaderSearchOpts().Sysroot; std::unique_ptr Reader(new ASTReader( CI.getPreprocessor(), CI.getASTContext(), CI.getPCHContainerReader(), + CI.getFrontendOpts().ModuleFileExtensions, Sysroot.empty() ? "" : Sysroot.c_str(), /*DisableValidation*/ false, /*AllowPCHWithCompilerErrors*/ false, @@ -564,6 +569,20 @@ namespace { } return false; } + + /// Indicates that a particular module file extension has been read. + void readModuleFileExtension( + const ModuleFileExtensionMetadata &Metadata) override { + Out.indent(2) << "Module file extension '" + << Metadata.BlockName << "' " << Metadata.MajorVersion + << "." << Metadata.MinorVersion; + if (!Metadata.UserInfo.empty()) { + Out << ": "; + Out.write_escaped(Metadata.UserInfo); + } + + Out << "\n"; + } #undef DUMP_BOOLEAN }; } @@ -583,7 +602,8 @@ void DumpModuleInfoAction::ExecuteAction() { DumpModuleInfoListener Listener(Out); ASTReader::readASTFileControlBlock( getCurrentFile(), getCompilerInstance().getFileManager(), - getCompilerInstance().getPCHContainerReader(), Listener); + getCompilerInstance().getPCHContainerReader(), + /*FindModuleFileExtensions=*/true, Listener); } //===----------------------------------------------------------------------===// diff --git a/clang/lib/Frontend/TestModuleFileExtension.cpp b/clang/lib/Frontend/TestModuleFileExtension.cpp new file mode 100644 index 000000000000..e17dc3c1ba6b --- /dev/null +++ b/clang/lib/Frontend/TestModuleFileExtension.cpp @@ -0,0 +1,121 @@ +//===-- TestModuleFileExtension.cpp - Module Extension Tester -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "TestModuleFileExtension.h" +#include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Serialization/ASTReader.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/Bitcode/BitstreamWriter.h" +#include "llvm/Support/raw_ostream.h" +using namespace clang; +using namespace clang::serialization; + +TestModuleFileExtension::Writer::~Writer() { } + +void TestModuleFileExtension::Writer::writeExtensionContents( + llvm::BitstreamWriter &Stream) { + using namespace llvm; + + // Write an abbreviation for this record. + BitCodeAbbrev *Abv = new llvm::BitCodeAbbrev(); + Abv->Add(BitCodeAbbrevOp(FIRST_EXTENSION_RECORD_ID)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of characters + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // message + auto Abbrev = Stream.EmitAbbrev(Abv); + + // Write a message into the extension block. + SmallString<64> Message; + { + auto Ext = static_cast(getExtension()); + raw_svector_ostream OS(Message); + OS << "Hello from " << Ext->BlockName << " v" << Ext->MajorVersion << "." + << Ext->MinorVersion; + } + SmallVector Record; + Record.push_back(FIRST_EXTENSION_RECORD_ID); + Record.push_back(Message.size()); + Stream.EmitRecordWithBlob(Abbrev, Record, Message); +} + +TestModuleFileExtension::Reader::Reader(ModuleFileExtension *Ext, + const llvm::BitstreamCursor &InStream) + : ModuleFileExtensionReader(Ext), Stream(InStream) +{ + // Read the extension block. + SmallVector Record; + while (true) { + llvm::BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + switch (Entry.Kind) { + case llvm::BitstreamEntry::SubBlock: + case llvm::BitstreamEntry::EndBlock: + case llvm::BitstreamEntry::Error: + return; + + case llvm::BitstreamEntry::Record: + break; + } + + Record.clear(); + StringRef Blob; + unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob); + switch (RecCode) { + case FIRST_EXTENSION_RECORD_ID: { + StringRef Message = Blob.substr(0, Record[0]); + fprintf(stderr, "Read extension block message: %s\n", + Message.str().c_str()); + break; + } + } + } +} + +TestModuleFileExtension::Reader::~Reader() { } + +TestModuleFileExtension::~TestModuleFileExtension() { } + +ModuleFileExtensionMetadata +TestModuleFileExtension::getExtensionMetadata() const { + return { BlockName, MajorVersion, MinorVersion, UserInfo }; +} + +llvm::hash_code TestModuleFileExtension::hashExtension( + llvm::hash_code Code) const { + if (Hashed) { + Code = llvm::hash_combine(Code, BlockName); + Code = llvm::hash_combine(Code, MajorVersion); + Code = llvm::hash_combine(Code, MinorVersion); + Code = llvm::hash_combine(Code, UserInfo); + } + + return Code; +} + +std::unique_ptr +TestModuleFileExtension::createExtensionWriter(ASTWriter &) { + return std::unique_ptr(new Writer(this)); +} + +std::unique_ptr +TestModuleFileExtension::createExtensionReader( + const ModuleFileExtensionMetadata &Metadata, + ASTReader &Reader, serialization::ModuleFile &Mod, + const llvm::BitstreamCursor &Stream) +{ + assert(Metadata.BlockName == BlockName && "Wrong block name"); + if (std::make_pair(Metadata.MajorVersion, Metadata.MinorVersion) != + std::make_pair(MajorVersion, MinorVersion)) { + Reader.getDiags().Report(Mod.ImportLoc, + diag::err_test_module_file_extension_version) + << BlockName << Metadata.MajorVersion << Metadata.MinorVersion + << MajorVersion << MinorVersion; + return nullptr; + } + + return std::unique_ptr( + new TestModuleFileExtension::Reader(this, Stream)); +} diff --git a/clang/lib/Frontend/TestModuleFileExtension.h b/clang/lib/Frontend/TestModuleFileExtension.h new file mode 100644 index 000000000000..2a7245463cf7 --- /dev/null +++ b/clang/lib/Frontend/TestModuleFileExtension.h @@ -0,0 +1,71 @@ +//===-- TestModuleFileExtension.h - Module Extension Tester -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_CLANG_FRONTEND_TESTMODULEFILEEXTENSION_H +#define LLVM_CLANG_FRONTEND_TESTMODULEFILEEXTENSION_H + +#include "clang/Serialization/ModuleFileExtension.h" +#include "clang/Basic/LLVM.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Bitcode/BitstreamReader.h" +#include + +namespace clang { + +/// A module file extension used for testing purposes. +class TestModuleFileExtension : public ModuleFileExtension { + std::string BlockName; + unsigned MajorVersion; + unsigned MinorVersion; + bool Hashed; + std::string UserInfo; + + class Writer : public ModuleFileExtensionWriter { + public: + Writer(ModuleFileExtension *Ext) : ModuleFileExtensionWriter(Ext) { } + ~Writer() override; + + void writeExtensionContents(llvm::BitstreamWriter &Stream) override; + }; + + class Reader : public ModuleFileExtensionReader { + llvm::BitstreamCursor Stream; + + public: + ~Reader() override; + + Reader(ModuleFileExtension *Ext, const llvm::BitstreamCursor &InStream); + }; + +public: + TestModuleFileExtension(StringRef BlockName, + unsigned MajorVersion, + unsigned MinorVersion, + bool Hashed, + StringRef UserInfo) + : BlockName(BlockName), + MajorVersion(MajorVersion), MinorVersion(MinorVersion), + Hashed(Hashed), UserInfo(UserInfo) { } + ~TestModuleFileExtension() override; + + ModuleFileExtensionMetadata getExtensionMetadata() const override; + + llvm::hash_code hashExtension(llvm::hash_code Code) const override; + + std::unique_ptr + createExtensionWriter(ASTWriter &Writer) override; + + std::unique_ptr + createExtensionReader(const ModuleFileExtensionMetadata &Metadata, + ASTReader &Reader, serialization::ModuleFile &Mod, + const llvm::BitstreamCursor &Stream) override; +}; + +} // end namespace clang + +#endif // LLVM_CLANG_FRONTEND_TESTMODULEFILEEXTENSION_H diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index c54f659f09f6..c09e97c1584b 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -160,6 +160,12 @@ bool ChainedASTReaderListener::visitInputFile(StringRef Filename, return Continue; } +void ChainedASTReaderListener::readModuleFileExtension( + const ModuleFileExtensionMetadata &Metadata) { + First->readModuleFileExtension(Metadata); + Second->readModuleFileExtension(Metadata); +} + //===----------------------------------------------------------------------===// // PCH validator implementation //===----------------------------------------------------------------------===// @@ -3423,6 +3429,36 @@ static void updateModuleTimestamp(ModuleFile &MF) { OS << "Timestamp file\n"; } +/// \brief Given a cursor at the start of an AST file, scan ahead and drop the +/// cursor into the start of the given block ID, returning false on success and +/// true on failure. +static bool SkipCursorToBlock(BitstreamCursor &Cursor, unsigned BlockID) { + while (1) { + llvm::BitstreamEntry Entry = Cursor.advance(); + switch (Entry.Kind) { + case llvm::BitstreamEntry::Error: + case llvm::BitstreamEntry::EndBlock: + return true; + + case llvm::BitstreamEntry::Record: + // Ignore top-level records. + Cursor.skipRecord(Entry.ID); + break; + + case llvm::BitstreamEntry::SubBlock: + if (Entry.ID == BlockID) { + if (Cursor.EnterSubBlock(BlockID)) + return true; + // Found it! + return false; + } + + if (Cursor.SkipBlock()) + return true; + } + } +} + ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName, ModuleKind Type, SourceLocation ImportLoc, @@ -3480,6 +3516,12 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName, if (ASTReadResult Result = ReadASTBlock(F, ClientLoadCapabilities)) return Result; + // Read the extension blocks. + while (!SkipCursorToBlock(F.Stream, EXTENSION_BLOCK_ID)) { + if (ASTReadResult Result = ReadExtensionBlock(F)) + return Result; + } + // Once read, set the ModuleFile bit base offset and update the size in // bits of all files we've seen. F.GlobalBitOffset = TotalModulesSizeInBits; @@ -3738,14 +3780,13 @@ ASTReader::ReadASTCore(StringRef FileName, // This is used for compatibility with older PCH formats. bool HaveReadControlBlock = false; - while (1) { llvm::BitstreamEntry Entry = Stream.advance(); switch (Entry.Kind) { case llvm::BitstreamEntry::Error: - case llvm::BitstreamEntry::EndBlock: case llvm::BitstreamEntry::Record: + case llvm::BitstreamEntry::EndBlock: Error("invalid record at top-level of AST file"); return Failure; @@ -3800,6 +3841,79 @@ ASTReader::ReadASTCore(StringRef FileName, break; } } + + return Success; +} + +/// Parse a record and blob containing module file extension metadata. +static bool parseModuleFileExtensionMetadata( + const SmallVectorImpl &Record, + StringRef Blob, + ModuleFileExtensionMetadata &Metadata) { + if (Record.size() < 4) return true; + + Metadata.MajorVersion = Record[0]; + Metadata.MinorVersion = Record[1]; + + unsigned BlockNameLen = Record[2]; + unsigned UserInfoLen = Record[3]; + + if (BlockNameLen + UserInfoLen > Blob.size()) return true; + + Metadata.BlockName = std::string(Blob.data(), Blob.data() + BlockNameLen); + Metadata.UserInfo = std::string(Blob.data() + BlockNameLen, + Blob.data() + BlockNameLen + UserInfoLen); + return false; +} + +ASTReader::ASTReadResult ASTReader::ReadExtensionBlock(ModuleFile &F) { + BitstreamCursor &Stream = F.Stream; + + RecordData Record; + while (true) { + llvm::BitstreamEntry Entry = Stream.advance(); + switch (Entry.Kind) { + case llvm::BitstreamEntry::SubBlock: + if (Stream.SkipBlock()) + return Failure; + + continue; + + case llvm::BitstreamEntry::EndBlock: + return Success; + + case llvm::BitstreamEntry::Error: + return HadErrors; + + case llvm::BitstreamEntry::Record: + break; + } + + Record.clear(); + StringRef Blob; + unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob); + switch (RecCode) { + case EXTENSION_METADATA: { + ModuleFileExtensionMetadata Metadata; + if (parseModuleFileExtensionMetadata(Record, Blob, Metadata)) + return Failure; + + // Find a module file extension with this block name. + auto Known = ModuleFileExtensions.find(Metadata.BlockName); + if (Known == ModuleFileExtensions.end()) break; + + // Form a reader. + if (auto Reader = Known->second->createExtensionReader(Metadata, *this, + F, Stream)) { + F.ExtensionReaders.push_back(std::move(Reader)); + } + + break; + } + } + } + + return Success; } void ASTReader::InitializeContext() { @@ -3940,36 +4054,6 @@ void ASTReader::finalizeForWriting() { // Nothing to do for now. } -/// \brief Given a cursor at the start of an AST file, scan ahead and drop the -/// cursor into the start of the given block ID, returning false on success and -/// true on failure. -static bool SkipCursorToBlock(BitstreamCursor &Cursor, unsigned BlockID) { - while (1) { - llvm::BitstreamEntry Entry = Cursor.advance(); - switch (Entry.Kind) { - case llvm::BitstreamEntry::Error: - case llvm::BitstreamEntry::EndBlock: - return true; - - case llvm::BitstreamEntry::Record: - // Ignore top-level records. - Cursor.skipRecord(Entry.ID); - break; - - case llvm::BitstreamEntry::SubBlock: - if (Entry.ID == BlockID) { - if (Cursor.EnterSubBlock(BlockID)) - return true; - // Found it! - return false; - } - - if (Cursor.SkipBlock()) - return true; - } - } -} - /// \brief Reads and return the signature record from \p StreamFile's control /// block, or else returns 0. static ASTFileSignature readASTFileSignature(llvm::BitstreamReader &StreamFile){ @@ -4097,6 +4181,7 @@ namespace { bool ASTReader::readASTFileControlBlock( StringRef Filename, FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, + bool FindModuleFileExtensions, ASTReaderListener &Listener) { // Open the AST file. // FIXME: This allows use of the VFS; we do not allow use of the @@ -4126,7 +4211,8 @@ bool ASTReader::readASTFileControlBlock( RecordData Record; std::string ModuleDir; - while (1) { + bool DoneWithControlBlock = false; + while (!DoneWithControlBlock) { llvm::BitstreamEntry Entry = Stream.advance(); switch (Entry.Kind) { @@ -4159,7 +4245,8 @@ bool ASTReader::readASTFileControlBlock( } case llvm::BitstreamEntry::EndBlock: - return false; + DoneWithControlBlock = true; + break; case llvm::BitstreamEntry::Error: return true; @@ -4168,6 +4255,8 @@ bool ASTReader::readASTFileControlBlock( break; } + if (DoneWithControlBlock) break; + Record.clear(); StringRef Blob; unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob); @@ -4251,6 +4340,50 @@ bool ASTReader::readASTFileControlBlock( break; } } + + // Look for module file extension blocks, if requested. + if (FindModuleFileExtensions) { + while (!SkipCursorToBlock(Stream, EXTENSION_BLOCK_ID)) { + bool DoneWithExtensionBlock = false; + while (!DoneWithExtensionBlock) { + llvm::BitstreamEntry Entry = Stream.advance(); + + switch (Entry.Kind) { + case llvm::BitstreamEntry::SubBlock: + if (Stream.SkipBlock()) + return true; + + continue; + + case llvm::BitstreamEntry::EndBlock: + DoneWithExtensionBlock = true; + continue; + + case llvm::BitstreamEntry::Error: + return true; + + case llvm::BitstreamEntry::Record: + break; + } + + Record.clear(); + StringRef Blob; + unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob); + switch (RecCode) { + case EXTENSION_METADATA: { + ModuleFileExtensionMetadata Metadata; + if (parseModuleFileExtensionMetadata(Record, Blob, Metadata)) + return true; + + Listener.readModuleFileExtension(Metadata); + break; + } + } + } + } + } + + return false; } bool ASTReader::isAcceptableASTFile( @@ -4261,6 +4394,7 @@ bool ASTReader::isAcceptableASTFile( SimplePCHValidator validator(LangOpts, TargetOpts, PPOpts, ExistingModuleCachePath, FileMgr); return !readASTFileControlBlock(Filename, FileMgr, PCHContainerRdr, + /*FindModuleFileExtensions=*/false, validator); } @@ -8483,13 +8617,15 @@ void ASTReader::pushExternalDeclIntoScope(NamedDecl *D, DeclarationName Name) { } } -ASTReader::ASTReader(Preprocessor &PP, ASTContext &Context, - const PCHContainerReader &PCHContainerRdr, - StringRef isysroot, bool DisableValidation, - bool AllowASTWithCompilerErrors, - bool AllowConfigurationMismatch, bool ValidateSystemInputs, - bool UseGlobalIndex, - std::unique_ptr ReadTimer) +ASTReader::ASTReader( + Preprocessor &PP, ASTContext &Context, + const PCHContainerReader &PCHContainerRdr, + ArrayRef> Extensions, + StringRef isysroot, bool DisableValidation, + bool AllowASTWithCompilerErrors, + bool AllowConfigurationMismatch, bool ValidateSystemInputs, + bool UseGlobalIndex, + std::unique_ptr ReadTimer) : Listener(new PCHValidator(PP, *this)), DeserializationListener(nullptr), OwnsDeserializationListener(false), SourceMgr(PP.getSourceManager()), FileMgr(PP.getFileManager()), PCHContainerRdr(PCHContainerRdr), @@ -8513,6 +8649,18 @@ ASTReader::ASTReader(Preprocessor &PP, ASTContext &Context, TotalModulesSizeInBits(0), NumCurrentElementsDeserializing(0), PassingDeclsToConsumer(false), ReadingKind(Read_None) { SourceMgr.setExternalSLocEntrySource(this); + + for (const auto &Ext : Extensions) { + auto BlockName = Ext->getExtensionMetadata().BlockName; + auto Known = ModuleFileExtensions.find(BlockName); + if (Known != ModuleFileExtensions.end()) { + Diags.Report(diag::warn_duplicate_module_file_extension) + << BlockName; + continue; + } + + ModuleFileExtensions.insert({BlockName, Ext}); + } } ASTReader::~ASTReader() { diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 01e9f220f975..c5a9755fed32 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "clang/Serialization/ASTWriter.h" +#include "clang/Serialization/ModuleFileExtension.h" #include "ASTCommon.h" #include "ASTReaderInternals.h" #include "MultiOnDiskHashTable.h" @@ -1094,7 +1095,11 @@ void ASTWriter::WriteBlockInfoBlock() { RECORD(PPD_MACRO_EXPANSION); RECORD(PPD_MACRO_DEFINITION); RECORD(PPD_INCLUSION_DIRECTIVE); - + + // Decls and Types block. + BLOCK(EXTENSION_BLOCK); + RECORD(EXTENSION_METADATA); + #undef RECORD #undef BLOCK Stream.ExitBlock(); @@ -3876,6 +3881,40 @@ void ASTWriter::WriteOptimizePragmaOptions(Sema &SemaRef) { Stream.EmitRecord(OPTIMIZE_PRAGMA_OPTIONS, Record); } +void ASTWriter::WriteModuleFileExtension(ModuleFileExtensionWriter &Writer) { + // Enter the extension block. + Stream.EnterSubblock(EXTENSION_BLOCK_ID, 4); + + // Emit the metadata record abbreviation. + llvm::BitCodeAbbrev *Abv = new llvm::BitCodeAbbrev(); + Abv->Add(llvm::BitCodeAbbrevOp(EXTENSION_METADATA)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob)); + unsigned Abbrev = Stream.EmitAbbrev(Abv); + + // Emit the metadata record. + RecordData Record; + auto Metadata = Writer.getExtension()->getExtensionMetadata(); + Record.push_back(EXTENSION_METADATA); + Record.push_back(Metadata.MajorVersion); + Record.push_back(Metadata.MinorVersion); + Record.push_back(Metadata.BlockName.size()); + Record.push_back(Metadata.UserInfo.size()); + SmallString<64> Buffer; + Buffer += Metadata.BlockName; + Buffer += Metadata.UserInfo; + Stream.EmitRecordWithBlob(Abbrev, Record, Buffer); + + // Emit the contents of the extension block. + Writer.writeExtensionContents(Stream); + + // Exit the extension block. + Stream.ExitBlock(); +} + //===----------------------------------------------------------------------===// // General Serialization Routines //===----------------------------------------------------------------------===// @@ -3979,7 +4018,10 @@ void ASTWriter::SetSelectorOffset(Selector Sel, uint32_t Offset) { SelectorOffsets[ID - FirstSelectorID] = Offset; } -ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream, bool IncludeTimestamps) +ASTWriter::ASTWriter( + llvm::BitstreamWriter &Stream, + ArrayRef> Extensions, + bool IncludeTimestamps) : Stream(Stream), Context(nullptr), PP(nullptr), Chain(nullptr), WritingModule(nullptr), IncludeTimestamps(IncludeTimestamps), WritingAST(false), DoneWritingDeclsAndTypes(false), @@ -3999,7 +4041,12 @@ ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream, bool IncludeTimestamps) DeclVarAbbrev(0), DeclFieldAbbrev(0), DeclEnumAbbrev(0), DeclObjCIvarAbbrev(0), DeclCXXMethodAbbrev(0), DeclRefExprAbbrev(0), CharacterLiteralAbbrev(0), IntegerLiteralAbbrev(0), - ExprImplicitCastAbbrev(0) {} + ExprImplicitCastAbbrev(0) { + for (const auto &Ext : Extensions) { + if (auto Writer = Ext->createExtensionWriter(*this)) + ModuleFileExtensionWriters.push_back(std::move(Writer)); + } +} ASTWriter::~ASTWriter() { llvm::DeleteContainerSeconds(FileDeclIDs); @@ -4396,7 +4443,6 @@ uint64_t ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, WriteCXXCtorInitializersOffsets(); WriteFileDeclIDsMap(); WriteSourceManagerBlock(Context.getSourceManager(), PP); - WriteComments(); WritePreprocessor(PP, isModule); WriteHeaderSearch(PP.getHeaderSearchInfo()); @@ -4526,6 +4572,10 @@ uint64_t ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, Stream.EmitRecord(STATISTICS, Record); Stream.ExitBlock(); + // Write the module file extension blocks. + for (const auto &ExtWriter : ModuleFileExtensionWriters) + WriteModuleFileExtension(*ExtWriter); + return Signature; } diff --git a/clang/lib/Serialization/CMakeLists.txt b/clang/lib/Serialization/CMakeLists.txt index d885db22975e..95b33c388c56 100644 --- a/clang/lib/Serialization/CMakeLists.txt +++ b/clang/lib/Serialization/CMakeLists.txt @@ -15,6 +15,7 @@ add_clang_library(clangSerialization GeneratePCH.cpp GlobalModuleIndex.cpp Module.cpp + ModuleFileExtension.cpp ModuleManager.cpp ADDITIONAL_HEADERS diff --git a/clang/lib/Serialization/GeneratePCH.cpp b/clang/lib/Serialization/GeneratePCH.cpp index 9de2fdb75a07..4a2255ab6d39 100644 --- a/clang/lib/Serialization/GeneratePCH.cpp +++ b/clang/lib/Serialization/GeneratePCH.cpp @@ -23,13 +23,15 @@ using namespace clang; -PCHGenerator::PCHGenerator(const Preprocessor &PP, StringRef OutputFile, - clang::Module *Module, StringRef isysroot, - std::shared_ptr Buffer, - bool AllowASTWithErrors, bool IncludeTimestamps) +PCHGenerator::PCHGenerator( + const Preprocessor &PP, StringRef OutputFile, + clang::Module *Module, StringRef isysroot, + std::shared_ptr Buffer, + ArrayRef> Extensions, + bool AllowASTWithErrors, bool IncludeTimestamps) : PP(PP), OutputFile(OutputFile), Module(Module), isysroot(isysroot.str()), SemaPtr(nullptr), Buffer(Buffer), Stream(Buffer->Data), - Writer(Stream, IncludeTimestamps), + Writer(Stream, Extensions, IncludeTimestamps), AllowASTWithErrors(AllowASTWithErrors) { Buffer->IsComplete = false; } diff --git a/clang/lib/Serialization/ModuleFileExtension.cpp b/clang/lib/Serialization/ModuleFileExtension.cpp new file mode 100644 index 000000000000..81dcfd60ce8e --- /dev/null +++ b/clang/lib/Serialization/ModuleFileExtension.cpp @@ -0,0 +1,22 @@ +//===-- ModuleFileExtension.cpp - Module File Extensions ------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "clang/Serialization/ModuleFileExtension.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/Support/raw_ostream.h" +using namespace clang; + +ModuleFileExtension::~ModuleFileExtension() { } + +llvm::hash_code ModuleFileExtension::hashExtension(llvm::hash_code Code) const { + return Code; +} + +ModuleFileExtensionWriter::~ModuleFileExtensionWriter() { } + +ModuleFileExtensionReader::~ModuleFileExtensionReader() { } diff --git a/clang/test/Modules/Inputs/ExtensionTestA.h b/clang/test/Modules/Inputs/ExtensionTestA.h new file mode 100644 index 000000000000..fee0bb953e7b --- /dev/null +++ b/clang/test/Modules/Inputs/ExtensionTestA.h @@ -0,0 +1 @@ +extern int ExtensionA; diff --git a/clang/test/Modules/Inputs/module.map b/clang/test/Modules/Inputs/module.map index 27751d8fd51f..d195e2f0980f 100644 --- a/clang/test/Modules/Inputs/module.map +++ b/clang/test/Modules/Inputs/module.map @@ -375,3 +375,7 @@ module DebugSubmodules { export * } } + +module ExtensionTestA { + header "ExtensionTestA.h" +} diff --git a/clang/test/Modules/extensions.c b/clang/test/Modules/extensions.c new file mode 100644 index 000000000000..1858f02e681a --- /dev/null +++ b/clang/test/Modules/extensions.c @@ -0,0 +1,44 @@ +// Test creation of modules that include extension blocks. +// RUN: rm -rf %t +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -ftest-module-file-extension=clang.testB:2:3:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s + +// Make sure the extension blocks are actually there. +// RUN: llvm-bcanalyzer %t/ExtensionTestA.pcm | FileCheck -check-prefix=CHECK-BCANALYZER %s +// RUN: %clang_cc1 -module-file-info %t/ExtensionTestA.pcm | FileCheck -check-prefix=CHECK-INFO %s + +// Make sure that the readers are able to check the metadata. +// RUN: rm -rf %t +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -ftest-module-file-extension=clang.testB:2:3:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:1:3:0:user_info_for_A -ftest-module-file-extension=clang.testB:3:2:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s -verify + +// Make sure that extension blocks can be part of the module hash. +// We test this in an obscure way, by making sure we don't get conflicts when +// using different "versions" of the extensions. Above, the "-verify" test +// checks that such conflicts produce errors. +// RUN: rm -rf %t +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:1:5:1:user_info_for_A -ftest-module-file-extension=clang.testB:2:3:1:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:1:3:1:user_info_for_A -ftest-module-file-extension=clang.testB:3:2:1:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:2:5:0:user_info_for_A -ftest-module-file-extension=clang.testB:7:3:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s + +// Make sure we can read the message back. +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -ftest-module-file-extension=clang.testB:2:3:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s > %t.log 2>&1 +// RUN: FileCheck -check-prefix=CHECK-MESSAGE %s < %t.log + +// Make sure we diagnose duplicate module file extensions. +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -fmodules-cache-path=%t -I %S/Inputs %s > %t.log 2>&1 +// RUN: FileCheck -check-prefix=CHECK-DUPLICATE %s < %t.log + +#include "ExtensionTestA.h" +// expected-error@-1{{test module file extension 'clang.testA' has different version (1.5) than expected (1.3)}} +// expected-error@-2{{test module file extension 'clang.testB' has different version (2.3) than expected (3.2)}} + +// CHECK-BCANALYZER: {{Block ID.*EXTENSION_BLOCK}} +// CHECK-BCANALYZER: {{100.00.*EXTENSION_METADATA}} + +// CHECK-INFO: Module file extension 'clang.testA' 1.5: user_info_for_A +// CHECK-INFO: Module file extension 'clang.testB' 2.3: user_info_for_B + +// CHECK-MESSAGE: Read extension block message: Hello from clang.testA v1.5 +// CHECK-MESSAGE: Read extension block message: Hello from clang.testB v2.3 + +// CHECK-DUPLICATE: warning: duplicate module file extension block name 'clang.testA'