From 6623e1f10f9546cfe68268ca6724a6fc187b3c1d Mon Sep 17 00:00:00 2001 From: Douglas Gregor Date: Tue, 3 Nov 2015 18:33:07 +0000 Subject: [PATCH] Introduce module file extensions to piggy-back data onto module files. Introduce the notion of a module file extension, which introduces additional information into a module file at the time it is built that can then be queried when the module file is read. Module file extensions are identified by a block name (which must be unique to the extension) and can write any bitstream records into their own extension block within the module file. When a module file is loaded, any extension blocks are matched up with module file extension readers, that are per-module-file and are given access to the input bitstream. Note that module file extensions can only be introduced by programmatic clients that have access to the CompilerInvocation. There is only one such extension at the moment, which is used for testing the module file extension harness. As a future direction, one could imagine allowing the plugin mechanism to introduce new module file extensions. llvm-svn: 251955 --- .../clang/Basic/DiagnosticDriverKinds.td | 4 + .../clang/Basic/DiagnosticFrontendKinds.td | 4 + clang/include/clang/Basic/DiagnosticGroups.td | 1 + .../Basic/DiagnosticSerializationKinds.td | 4 + clang/include/clang/Driver/CC1Options.td | 4 + .../include/clang/Frontend/CompilerInstance.h | 1 + .../include/clang/Frontend/FrontendOptions.h | 4 + .../include/clang/Serialization/ASTBitCodes.h | 12 + clang/include/clang/Serialization/ASTReader.h | 18 ++ clang/include/clang/Serialization/ASTWriter.h | 25 +- clang/include/clang/Serialization/Module.h | 5 + .../clang/Serialization/ModuleFileExtension.h | 147 +++++++++++ clang/lib/Frontend/ASTUnit.cpp | 7 +- clang/lib/Frontend/CMakeLists.txt | 1 + clang/lib/Frontend/ChainedIncludesSource.cpp | 5 +- clang/lib/Frontend/CompilerInstance.cpp | 14 +- clang/lib/Frontend/CompilerInvocation.cpp | 52 ++++ clang/lib/Frontend/FrontendActions.cpp | 30 ++- .../lib/Frontend/TestModuleFileExtension.cpp | 121 +++++++++ clang/lib/Frontend/TestModuleFileExtension.h | 71 ++++++ clang/lib/Serialization/ASTReader.cpp | 230 ++++++++++++++---- clang/lib/Serialization/ASTWriter.cpp | 58 ++++- clang/lib/Serialization/CMakeLists.txt | 1 + clang/lib/Serialization/GeneratePCH.cpp | 12 +- .../lib/Serialization/ModuleFileExtension.cpp | 22 ++ clang/test/Modules/Inputs/ExtensionTestA.h | 1 + clang/test/Modules/Inputs/module.map | 4 + clang/test/Modules/extensions.c | 44 ++++ 28 files changed, 831 insertions(+), 71 deletions(-) create mode 100644 clang/include/clang/Serialization/ModuleFileExtension.h create mode 100644 clang/lib/Frontend/TestModuleFileExtension.cpp create mode 100644 clang/lib/Frontend/TestModuleFileExtension.h create mode 100644 clang/lib/Serialization/ModuleFileExtension.cpp create mode 100644 clang/test/Modules/Inputs/ExtensionTestA.h create mode 100644 clang/test/Modules/extensions.c diff --git a/clang/include/clang/Basic/DiagnosticDriverKinds.td b/clang/include/clang/Basic/DiagnosticDriverKinds.td index 282dd711719b..7a71285482f6 100644 --- a/clang/include/clang/Basic/DiagnosticDriverKinds.td +++ b/clang/include/clang/Basic/DiagnosticDriverKinds.td @@ -193,6 +193,10 @@ def err_drv_modules_validate_once_requires_timestamp : Error< "option '-fmodules-validate-once-per-build-session' requires " "'-fbuild-session-timestamp=' or '-fbuild-session-file='">; +def err_test_module_file_extension_format : Error< + "-ftest-module-file-extension argument '%0' is not of the required form " + "'blockname:major:minor:hashed:user info'">; + def warn_drv_invoking_fallback : Warning<"falling back to %0">, InGroup; diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index 28fc3059b258..033834bc505d 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -204,6 +204,10 @@ def err_modules_embed_file_not_found : Error<"file '%0' specified by '-fmodules-embed-file=' not found">, DefaultFatal; +def err_test_module_file_extension_version : Error< + "test module file extension '%0' has different version (%1.%2) than expected " + "(%3.%4)">; + def err_conflicting_module_names : Error< "conflicting module names specified: '-fmodule-name=%0' and " "'-fmodule-implementation-of %1'">; diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index 042244125b8d..4b94b186cd6a 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -253,6 +253,7 @@ def MismatchedTags : DiagGroup<"mismatched-tags">; def MissingFieldInitializers : DiagGroup<"missing-field-initializers">; def ModuleBuild : DiagGroup<"module-build">; def ModuleConflict : DiagGroup<"module-conflict">; +def ModuleFileExtension : DiagGroup<"module-file-extension">; def NewlineEOF : DiagGroup<"newline-eof">; def Nullability : DiagGroup<"nullability">; def NullabilityDeclSpec : DiagGroup<"nullability-declspec">; diff --git a/clang/include/clang/Basic/DiagnosticSerializationKinds.td b/clang/include/clang/Basic/DiagnosticSerializationKinds.td index 272cc7bf6b01..16c77435558f 100644 --- a/clang/include/clang/Basic/DiagnosticSerializationKinds.td +++ b/clang/include/clang/Basic/DiagnosticSerializationKinds.td @@ -115,6 +115,10 @@ def warn_module_uses_date_time : Warning< "%select{precompiled header|module}0 uses __DATE__ or __TIME__">, InGroup>; +def warn_duplicate_module_file_extension : Warning< + "duplicate module file extension block name '%0'">, + InGroup; + } // let CategoryName } // let Component diff --git a/clang/include/clang/Driver/CC1Options.td b/clang/include/clang/Driver/CC1Options.td index 7a1a9edf2894..5bf175e3bd99 100644 --- a/clang/include/clang/Driver/CC1Options.td +++ b/clang/include/clang/Driver/CC1Options.td @@ -392,6 +392,10 @@ def fno_modules_hide_internal_linkage : Flag<["-"], "fno-modules-hide-internal-linkage">, HelpText<"Make all declarations visible to redeclaration lookup, " "even if they have internal linkage.">; +def ftest_module_file_extension_EQ : + Joined<["-"], "ftest-module-file-extension=">, + HelpText<"introduce a module file extension for testing purposes. " + "The argument is parsed as blockname:major:minor:hashed:user info">; def fconcepts_ts : Flag<["-"], "fconcepts-ts">, HelpText<"Enable C++ Extensions for Concepts.">; diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h index 5c29f753db92..83eed2cdc592 100644 --- a/clang/include/clang/Frontend/CompilerInstance.h +++ b/clang/include/clang/Frontend/CompilerInstance.h @@ -655,6 +655,7 @@ public: StringRef Path, StringRef Sysroot, bool DisablePCHValidation, bool AllowPCHWithCompilerErrors, Preprocessor &PP, ASTContext &Context, const PCHContainerReader &PCHContainerRdr, + ArrayRef> Extensions, void *DeserializationListener, bool OwnDeserializationListener, bool Preamble, bool UseGlobalModuleIndex); diff --git a/clang/include/clang/Frontend/FrontendOptions.h b/clang/include/clang/Frontend/FrontendOptions.h index 969f0c0a25e6..8c841e094245 100644 --- a/clang/include/clang/Frontend/FrontendOptions.h +++ b/clang/include/clang/Frontend/FrontendOptions.h @@ -11,6 +11,7 @@ #define LLVM_CLANG_FRONTEND_FRONTENDOPTIONS_H #include "clang/Frontend/CommandLineSourceLoc.h" +#include "clang/Serialization/ModuleFileExtension.h" #include "clang/Sema/CodeCompleteOptions.h" #include "llvm/ADT/StringRef.h" #include @@ -236,6 +237,9 @@ public: /// The list of plugins to load. std::vector Plugins; + /// The list of module file extensions. + std::vector> ModuleFileExtensions; + /// \brief The list of module map files to load before processing the input. std::vector ModuleMapFiles; diff --git a/clang/include/clang/Serialization/ASTBitCodes.h b/clang/include/clang/Serialization/ASTBitCodes.h index 5d0669f4bfcf..7af3c6285561 100644 --- a/clang/include/clang/Serialization/ASTBitCodes.h +++ b/clang/include/clang/Serialization/ASTBitCodes.h @@ -241,6 +241,9 @@ namespace clang { /// /// This block is part of the control block. OPTIONS_BLOCK_ID, + + /// \brief A block containing a module file extension. + EXTENSION_BLOCK_ID, }; /// \brief Record types that occur within the control block. @@ -310,6 +313,15 @@ namespace clang { PREPROCESSOR_OPTIONS, }; + /// \brief Record code for extension blocks. + enum ExtensionBlockRecordTypes { + /// Metadata describing this particular extension. + EXTENSION_METADATA = 1, + + /// The first record ID allocated to the extensions themselves. + FIRST_EXTENSION_RECORD_ID = 4 + }; + /// \brief Record types that occur within the input-files block /// inside the control block. enum InputFileRecordTypes { diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index be37186b3eed..eb613802e3bf 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -30,6 +30,7 @@ #include "clang/Serialization/ASTBitCodes.h" #include "clang/Serialization/ContinuousRangeMap.h" #include "clang/Serialization/Module.h" +#include "clang/Serialization/ModuleFileExtension.h" #include "clang/Serialization/ModuleManager.h" #include "llvm/ADT/APFloat.h" #include "llvm/ADT/APInt.h" @@ -38,6 +39,7 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/TinyPtrVector.h" #include "llvm/Bitcode/BitstreamReader.h" @@ -205,6 +207,10 @@ public: /// \brief If needsImportVisitation returns \c true, this is called for each /// AST file imported by this AST file. virtual void visitImport(StringRef Filename) {} + + /// Indicates that a particular module file extension has been read. + virtual void readModuleFileExtension( + const ModuleFileExtensionMetadata &Metadata) {} }; /// \brief Simple wrapper class for chaining listeners. @@ -247,6 +253,8 @@ public: serialization::ModuleKind Kind) override; bool visitInputFile(StringRef Filename, bool isSystem, bool isOverridden, bool isExplicitModule) override; + void readModuleFileExtension( + const ModuleFileExtensionMetadata &Metadata) override; }; /// \brief ASTReaderListener implementation to validate the information of @@ -382,6 +390,9 @@ private: /// \brief The module manager which manages modules and their dependencies ModuleManager ModuleMgr; + /// A mapping from extension block names to module file extensions. + llvm::StringMap> ModuleFileExtensions; + /// \brief A timer used to track the time spent deserializing. std::unique_ptr ReadTimer; @@ -1122,6 +1133,7 @@ private: bool AllowCompatibleConfigurationMismatch, ASTReaderListener &Listener, std::string &SuggestedPredefines); ASTReadResult ReadASTBlock(ModuleFile &F, unsigned ClientLoadCapabilities); + ASTReadResult ReadExtensionBlock(ModuleFile &F); bool ParseLineTable(ModuleFile &F, const RecordData &Record); bool ReadSourceManagerBlock(ModuleFile &F); llvm::BitstreamCursor &SLocCursorForID(int ID); @@ -1273,6 +1285,9 @@ public: /// \param PCHContainerRdr the PCHContainerOperations to use for loading and /// creating modules. /// + /// \param Extensions the list of module file extensions that can be loaded + /// from the AST files. + /// /// \param isysroot If non-NULL, the system include path specified by the /// user. This is only used with relocatable PCH files. If non-NULL, /// a relocatable PCH file will use the default path "/". @@ -1299,6 +1314,7 @@ public: /// deserializing. ASTReader(Preprocessor &PP, ASTContext &Context, const PCHContainerReader &PCHContainerRdr, + ArrayRef> Extensions, StringRef isysroot = "", bool DisableValidation = false, bool AllowASTWithCompilerErrors = false, bool AllowConfigurationMismatch = false, @@ -1309,6 +1325,7 @@ public: SourceManager &getSourceManager() const { return SourceMgr; } FileManager &getFileManager() const { return FileMgr; } + DiagnosticsEngine &getDiags() const { return Diags; } /// \brief Flags that indicate what kind of AST loading failures the client /// of the AST reader can directly handle. @@ -1476,6 +1493,7 @@ public: static bool readASTFileControlBlock(StringRef Filename, FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, + bool FindModuleFileExtensions, ASTReaderListener &Listener); /// \brief Determine whether the given AST file is acceptable to load into a diff --git a/clang/include/clang/Serialization/ASTWriter.h b/clang/include/clang/Serialization/ASTWriter.h index 45e29921a537..86dbceef4c47 100644 --- a/clang/include/clang/Serialization/ASTWriter.h +++ b/clang/include/clang/Serialization/ASTWriter.h @@ -58,6 +58,8 @@ class OpaqueValueExpr; class OpenCLOptions; class ASTReader; class Module; +class ModuleFileExtension; +class ModuleFileExtensionWriter; class PreprocessedEntity; class PreprocessingRecord; class Preprocessor; @@ -491,7 +493,11 @@ private: /// \brief A mapping from each known submodule to its ID number, which will /// be a positive integer. llvm::DenseMap SubmoduleIDs; - + + /// \brief A list of the module file extension writers. + std::vector> + ModuleFileExtensionWriters; + /// \brief Retrieve or create a submodule ID for this module. unsigned getSubmoduleID(Module *Mod); @@ -545,6 +551,7 @@ private: void WriteObjCCategories(); void WriteLateParsedTemplates(Sema &SemaRef); void WriteOptimizePragmaOptions(Sema &SemaRef); + void WriteModuleFileExtension(ModuleFileExtensionWriter &Writer); unsigned DeclParmVarAbbrev; unsigned DeclContextLexicalAbbrev; @@ -574,7 +581,9 @@ private: public: /// \brief Create a new precompiled header writer that outputs to /// the given bitstream. - ASTWriter(llvm::BitstreamWriter &Stream, bool IncludeTimestamps = true); + ASTWriter(llvm::BitstreamWriter &Stream, + ArrayRef> Extensions, + bool IncludeTimestamps = true); ~ASTWriter() override; const LangOptions &getLangOpts() const; @@ -890,11 +899,13 @@ protected: SmallVectorImpl &getPCH() const { return Buffer->Data; } public: - PCHGenerator(const Preprocessor &PP, StringRef OutputFile, - clang::Module *Module, StringRef isysroot, - std::shared_ptr Buffer, - bool AllowASTWithErrors = false, - bool IncludeTimestamps = true); + PCHGenerator( + const Preprocessor &PP, StringRef OutputFile, + clang::Module *Module, StringRef isysroot, + std::shared_ptr Buffer, + ArrayRef> Extensions, + bool AllowASTWithErrors = false, + bool IncludeTimestamps = true); ~PCHGenerator() override; void InitializeSema(Sema &S) override { SemaPtr = &S; } void HandleTranslationUnit(ASTContext &Ctx) override; diff --git a/clang/include/clang/Serialization/Module.h b/clang/include/clang/Serialization/Module.h index 1b9560188cb0..3fdb33f495d6 100644 --- a/clang/include/clang/Serialization/Module.h +++ b/clang/include/clang/Serialization/Module.h @@ -18,6 +18,7 @@ #include "clang/Basic/SourceLocation.h" #include "clang/Serialization/ASTBitCodes.h" #include "clang/Serialization/ContinuousRangeMap.h" +#include "clang/Serialization/ModuleFileExtension.h" #include "llvm/ADT/SetVector.h" #include "llvm/Bitcode/BitstreamReader.h" #include "llvm/Support/Endian.h" @@ -194,6 +195,10 @@ public: /// \brief The first source location in this module. SourceLocation FirstLoc; + /// The list of extension readers that are attached to this module + /// file. + std::vector> ExtensionReaders; + // === Input Files === /// \brief The cursor to the start of the input-files block. llvm::BitstreamCursor InputFilesCursor; diff --git a/clang/include/clang/Serialization/ModuleFileExtension.h b/clang/include/clang/Serialization/ModuleFileExtension.h new file mode 100644 index 000000000000..a8f8386e6492 --- /dev/null +++ b/clang/include/clang/Serialization/ModuleFileExtension.h @@ -0,0 +1,147 @@ +//===-- ModuleFileExtension.h - Module File Extensions ----------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_SERIALIZATION_MODULEFILEEXTENSION_H +#define LLVM_CLANG_SERIALIZATION_MODULEFILEEXTENSION_H + +#include "llvm/ADT/IntrusiveRefCntPtr.h" +#include +#include + +namespace llvm { +class BitstreamCursor; +class BitstreamWriter; +class hash_code; +class raw_ostream; +} + +namespace clang { + +class ASTReader; +class ASTWriter; + +namespace serialization { + class ModuleFile; +} // end namespace serialization + +/// Metadata for a module file extension. +struct ModuleFileExtensionMetadata { + /// The name used to identify this particular extension block within + /// the resulting module file. It should be unique to the particular + /// extension, because this name will be used to match the name of + /// an extension block to the appropriate reader. + std::string BlockName; + + /// The major version of the extension data. + unsigned MajorVersion; + + /// The minor version of the extension data. + unsigned MinorVersion; + + /// A string containing additional user information that will be + /// stored with the metadata. + std::string UserInfo; +}; + +class ModuleFileExtensionReader; +class ModuleFileExtensionWriter; + +/// An abstract superclass that describes a custom extension to the +/// module/precompiled header file format. +/// +/// A module file extension can introduce additional information into +/// compiled module files (.pcm) and precompiled headers (.pch) via a +/// custom writer that can then be accessed via a custom reader when +/// the module file or precompiled header is loaded. +class ModuleFileExtension : public llvm::RefCountedBase { +public: + virtual ~ModuleFileExtension(); + + /// Retrieves the metadata for this module file extension. + virtual ModuleFileExtensionMetadata getExtensionMetadata() const = 0; + + /// Hash information about the presence of this extension into the + /// module hash code. + /// + /// The module hash code is used to distinguish different variants + /// of a module that are incompatible. If the presence, absence, or + /// version of the module file extension should force the creation + /// of a separate set of module files, override this method to + /// combine that distinguishing information into the module hash + /// code. + /// + /// The default implementation of this function simply returns the + /// hash code as given, so the presence/absence of this extension + /// does not distinguish module files. + virtual llvm::hash_code hashExtension(llvm::hash_code Code) const; + + /// Create a new module file extension writer, which will be + /// responsible for writing the extension contents into a particular + /// module file. + virtual std::unique_ptr + createExtensionWriter(ASTWriter &Writer) = 0; + + /// Create a new module file extension reader, given the + /// metadata read from the block and the cursor into the extension + /// block. + /// + /// May return null to indicate that an extension block with the + /// given metadata cannot be read. + virtual std::unique_ptr + createExtensionReader(const ModuleFileExtensionMetadata &Metadata, + ASTReader &Reader, serialization::ModuleFile &Mod, + const llvm::BitstreamCursor &Stream) = 0; +}; + +/// Abstract base class that writes a module file extension block into +/// a module file. +class ModuleFileExtensionWriter { + ModuleFileExtension *Extension; + +protected: + ModuleFileExtensionWriter(ModuleFileExtension *Extension) + : Extension(Extension) { } + +public: + virtual ~ModuleFileExtensionWriter(); + + /// Retrieve the module file extension with which this writer is + /// associated. + ModuleFileExtension *getExtension() const { return Extension; } + + /// Write the contents of the extension block into the given bitstream. + /// + /// Responsible for writing the contents of the extension into the + /// given stream. All of the contents should be written into custom + /// records with IDs >= FIRST_EXTENSION_RECORD_ID. + virtual void writeExtensionContents(llvm::BitstreamWriter &Stream) = 0; +}; + +/// Abstract base class that reads a module file extension block from +/// a module file. +/// +/// Subclasses +class ModuleFileExtensionReader { + ModuleFileExtension *Extension; + +protected: + ModuleFileExtensionReader(ModuleFileExtension *Extension) + : Extension(Extension) { } + +public: + /// Retrieve the module file extension with which this reader is + /// associated. + ModuleFileExtension *getExtension() const { return Extension; } + + virtual ~ModuleFileExtensionReader(); +}; + +} // end namespace clang + +#endif // LLVM_CLANG_FRONTEND_MODULEFILEEXTENSION_H diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp index d3d7185ea3ad..6b89447305fe 100644 --- a/clang/lib/Frontend/ASTUnit.cpp +++ b/clang/lib/Frontend/ASTUnit.cpp @@ -186,7 +186,7 @@ struct ASTUnit::ASTWriterData { llvm::BitstreamWriter Stream; ASTWriter Writer; - ASTWriterData() : Stream(Buffer), Writer(Stream) { } + ASTWriterData() : Stream(Buffer), Writer(Stream, { }) { } }; void ASTUnit::clearFileLevelDecls() { @@ -709,7 +709,7 @@ std::unique_ptr ASTUnit::LoadFromASTFile( bool disableValid = false; if (::getenv("LIBCLANG_DISABLE_PCH_VALIDATION")) disableValid = true; - AST->Reader = new ASTReader(PP, Context, PCHContainerRdr, + AST->Reader = new ASTReader(PP, Context, PCHContainerRdr, { }, /*isysroot=*/"", /*DisableValidation=*/disableValid, AllowPCHWithCompilerErrors); @@ -927,6 +927,7 @@ public: const Preprocessor &PP, StringRef isysroot, raw_ostream *Out) : PCHGenerator(PP, "", nullptr, isysroot, std::make_shared(), + ArrayRef>(), /*AllowASTWithErrors=*/true), Unit(Unit), Hash(Unit.getCurrentTopLevelHashValue()), Action(Action), Out(Out) { @@ -2500,7 +2501,7 @@ bool ASTUnit::serialize(raw_ostream &OS) { SmallString<128> Buffer; llvm::BitstreamWriter Stream(Buffer); - ASTWriter Writer(Stream); + ASTWriter Writer(Stream, { }); return serializeUnit(Writer, Buffer, getSema(), hasErrors, OS); } diff --git a/clang/lib/Frontend/CMakeLists.txt b/clang/lib/Frontend/CMakeLists.txt index 9a3e459640a5..af42a905b387 100644 --- a/clang/lib/Frontend/CMakeLists.txt +++ b/clang/lib/Frontend/CMakeLists.txt @@ -35,6 +35,7 @@ add_clang_library(clangFrontend PrintPreprocessedOutput.cpp SerializedDiagnosticPrinter.cpp SerializedDiagnosticReader.cpp + TestModuleFileExtension.cpp TextDiagnostic.cpp TextDiagnosticBuffer.cpp TextDiagnosticPrinter.cpp diff --git a/clang/lib/Frontend/ChainedIncludesSource.cpp b/clang/lib/Frontend/ChainedIncludesSource.cpp index d36162b8dab6..1c1081fbe08e 100644 --- a/clang/lib/Frontend/ChainedIncludesSource.cpp +++ b/clang/lib/Frontend/ChainedIncludesSource.cpp @@ -82,6 +82,7 @@ createASTReader(CompilerInstance &CI, StringRef pchFile, std::unique_ptr Reader; Reader.reset(new ASTReader(PP, CI.getASTContext(), CI.getPCHContainerReader(), + /*Extensions=*/{ }, /*isysroot=*/"", /*DisableValidation=*/true)); for (unsigned ti = 0; ti < bufNames.size(); ++ti) { StringRef sr(bufNames[ti]); @@ -160,8 +161,10 @@ IntrusiveRefCntPtr clang::createChainedIncludesSource( Clang->createASTContext(); auto Buffer = std::make_shared(); + ArrayRef> Extensions; auto consumer = llvm::make_unique( - Clang->getPreprocessor(), "-", nullptr, /*isysroot=*/"", Buffer); + Clang->getPreprocessor(), "-", nullptr, /*isysroot=*/"", Buffer, + Extensions); Clang->getASTContext().setASTMutationListener( consumer->GetASTMutationListener()); Clang->setASTConsumer(std::move(consumer)); diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index 8220a6e49257..c3f19a3a00d3 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -408,7 +408,9 @@ void CompilerInstance::createPCHExternalASTSource( ModuleManager = createPCHExternalASTSource( Path, getHeaderSearchOpts().Sysroot, DisablePCHValidation, AllowPCHWithCompilerErrors, getPreprocessor(), getASTContext(), - getPCHContainerReader(), DeserializationListener, + getPCHContainerReader(), + getFrontendOpts().ModuleFileExtensions, + DeserializationListener, OwnDeserializationListener, Preamble, getFrontendOpts().UseGlobalModuleIndex); } @@ -417,15 +419,16 @@ IntrusiveRefCntPtr CompilerInstance::createPCHExternalASTSource( StringRef Path, StringRef Sysroot, bool DisablePCHValidation, bool AllowPCHWithCompilerErrors, Preprocessor &PP, ASTContext &Context, const PCHContainerReader &PCHContainerRdr, + ArrayRef> Extensions, void *DeserializationListener, bool OwnDeserializationListener, bool Preamble, bool UseGlobalModuleIndex) { HeaderSearchOptions &HSOpts = PP.getHeaderSearchInfo().getHeaderSearchOpts(); IntrusiveRefCntPtr Reader(new ASTReader( - PP, Context, PCHContainerRdr, Sysroot.empty() ? "" : Sysroot.data(), - DisablePCHValidation, AllowPCHWithCompilerErrors, - /*AllowConfigurationMismatch*/ false, HSOpts.ModulesValidateSystemHeaders, - UseGlobalModuleIndex)); + PP, Context, PCHContainerRdr, Extensions, + Sysroot.empty() ? "" : Sysroot.data(), DisablePCHValidation, + AllowPCHWithCompilerErrors, /*AllowConfigurationMismatch*/ false, + HSOpts.ModulesValidateSystemHeaders, UseGlobalModuleIndex)); // We need the external source to be set up before we read the AST, because // eagerly-deserialized declarations may use it. @@ -1267,6 +1270,7 @@ void CompilerInstance::createModuleManager() { *FrontendTimerGroup); ModuleManager = new ASTReader( getPreprocessor(), getASTContext(), getPCHContainerReader(), + getFrontendOpts().ModuleFileExtensions, Sysroot.empty() ? "" : Sysroot.c_str(), PPOpts.DisablePCHValidation, /*AllowASTWithCompilerErrors=*/false, /*AllowConfigurationMismatch=*/false, diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 0c1c4eaefa99..468cc5746cec 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -7,6 +7,7 @@ // //===----------------------------------------------------------------------===// +#include "TestModuleFileExtension.h" #include "clang/Frontend/CompilerInvocation.h" #include "clang/Basic/FileManager.h" #include "clang/Basic/Version.h" @@ -19,6 +20,7 @@ #include "clang/Frontend/Utils.h" #include "clang/Lex/HeaderSearchOptions.h" #include "clang/Serialization/ASTReader.h" +#include "clang/Serialization/ModuleFileExtension.h" #include "llvm/ADT/Hashing.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" @@ -832,6 +834,30 @@ static void ParseFileSystemArgs(FileSystemOptions &Opts, ArgList &Args) { Opts.WorkingDir = Args.getLastArgValue(OPT_working_directory); } +/// Parse the argument to the -ftest-module-file-extension +/// command-line argument. +/// +/// \returns true on error, false on success. +static bool parseTestModuleFileExtensionArg(StringRef Arg, + std::string &BlockName, + unsigned &MajorVersion, + unsigned &MinorVersion, + bool &Hashed, + std::string &UserInfo) { + SmallVector Args; + Arg.split(Args, ':', 5); + if (Args.size() < 5) + return true; + + BlockName = Args[0]; + if (Args[1].getAsInteger(10, MajorVersion)) return true; + if (Args[2].getAsInteger(10, MinorVersion)) return true; + if (Args[3].getAsInteger(2, Hashed)) return true; + if (Args.size() > 4) + UserInfo = Args[4]; + return false; +} + static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, DiagnosticsEngine &Diags) { using namespace options; @@ -924,6 +950,26 @@ static InputKind ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args, if (A->getValue(0) == Opts.AddPluginActions[i]) Opts.AddPluginArgs[i].emplace_back(A->getValue(1)); + for (const std::string &Arg : + Args.getAllArgValues(OPT_ftest_module_file_extension_EQ)) { + std::string BlockName; + unsigned MajorVersion; + unsigned MinorVersion; + bool Hashed; + std::string UserInfo; + if (parseTestModuleFileExtensionArg(Arg, BlockName, MajorVersion, + MinorVersion, Hashed, UserInfo)) { + Diags.Report(diag::err_test_module_file_extension_format) << Arg; + + continue; + } + + // Add the testing module file extension. + Opts.ModuleFileExtensions.push_back( + new TestModuleFileExtension(BlockName, MajorVersion, MinorVersion, + Hashed, UserInfo)); + } + if (const Arg *A = Args.getLastArg(OPT_code_completion_at)) { Opts.CodeCompletionAt = ParsedSourceLocation::FromString(A->getValue()); @@ -2076,6 +2122,12 @@ std::string CompilerInvocation::getModuleHash() const { // Extend the signature with the user build path. code = hash_combine(code, hsOpts.ModuleUserBuildPath); + // Extend the signature with the module file extensions. + const FrontendOptions &frontendOpts = getFrontendOpts(); + for (auto ext : frontendOpts.ModuleFileExtensions) { + code = ext->hashExtension(code); + } + // Darwin-specific hack: if we have a sysroot, use the contents and // modification time of // $sysroot/System/Library/CoreServices/SystemVersion.plist diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index 6ba4b134c38e..865fb474020f 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -91,7 +91,8 @@ GeneratePCHAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) { auto Buffer = std::make_shared(); std::vector> Consumers; Consumers.push_back(llvm::make_unique( - CI.getPreprocessor(), OutputFile, nullptr, Sysroot, Buffer)); + CI.getPreprocessor(), OutputFile, nullptr, Sysroot, + Buffer, CI.getFrontendOpts().ModuleFileExtensions)); Consumers.push_back(CI.getPCHContainerWriter().CreatePCHContainerGenerator( CI, InFile, OutputFile, OS, Buffer)); @@ -133,10 +134,13 @@ GenerateModuleAction::CreateASTConsumer(CompilerInstance &CI, auto Buffer = std::make_shared(); std::vector> Consumers; + Consumers.push_back(llvm::make_unique( - CI.getPreprocessor(), OutputFile, Module, Sysroot, Buffer, - /*AllowASTWithErrors*/false, - /*IncludeTimestamps*/+CI.getFrontendOpts().BuildingImplicitModule)); + CI.getPreprocessor(), OutputFile, Module, Sysroot, + Buffer, CI.getFrontendOpts().ModuleFileExtensions, + /*AllowASTWithErrors=*/false, + /*IncludeTimestamps=*/ + +CI.getFrontendOpts().BuildingImplicitModule)); Consumers.push_back(CI.getPCHContainerWriter().CreatePCHContainerGenerator( CI, InFile, OutputFile, OS, Buffer)); return llvm::make_unique(std::move(Consumers)); @@ -421,6 +425,7 @@ void VerifyPCHAction::ExecuteAction() { const std::string &Sysroot = CI.getHeaderSearchOpts().Sysroot; std::unique_ptr Reader(new ASTReader( CI.getPreprocessor(), CI.getASTContext(), CI.getPCHContainerReader(), + CI.getFrontendOpts().ModuleFileExtensions, Sysroot.empty() ? "" : Sysroot.c_str(), /*DisableValidation*/ false, /*AllowPCHWithCompilerErrors*/ false, @@ -564,6 +569,20 @@ namespace { } return false; } + + /// Indicates that a particular module file extension has been read. + void readModuleFileExtension( + const ModuleFileExtensionMetadata &Metadata) override { + Out.indent(2) << "Module file extension '" + << Metadata.BlockName << "' " << Metadata.MajorVersion + << "." << Metadata.MinorVersion; + if (!Metadata.UserInfo.empty()) { + Out << ": "; + Out.write_escaped(Metadata.UserInfo); + } + + Out << "\n"; + } #undef DUMP_BOOLEAN }; } @@ -583,7 +602,8 @@ void DumpModuleInfoAction::ExecuteAction() { DumpModuleInfoListener Listener(Out); ASTReader::readASTFileControlBlock( getCurrentFile(), getCompilerInstance().getFileManager(), - getCompilerInstance().getPCHContainerReader(), Listener); + getCompilerInstance().getPCHContainerReader(), + /*FindModuleFileExtensions=*/true, Listener); } //===----------------------------------------------------------------------===// diff --git a/clang/lib/Frontend/TestModuleFileExtension.cpp b/clang/lib/Frontend/TestModuleFileExtension.cpp new file mode 100644 index 000000000000..e17dc3c1ba6b --- /dev/null +++ b/clang/lib/Frontend/TestModuleFileExtension.cpp @@ -0,0 +1,121 @@ +//===-- TestModuleFileExtension.cpp - Module Extension Tester -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "TestModuleFileExtension.h" +#include "clang/Frontend/FrontendDiagnostic.h" +#include "clang/Serialization/ASTReader.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/Bitcode/BitstreamWriter.h" +#include "llvm/Support/raw_ostream.h" +using namespace clang; +using namespace clang::serialization; + +TestModuleFileExtension::Writer::~Writer() { } + +void TestModuleFileExtension::Writer::writeExtensionContents( + llvm::BitstreamWriter &Stream) { + using namespace llvm; + + // Write an abbreviation for this record. + BitCodeAbbrev *Abv = new llvm::BitCodeAbbrev(); + Abv->Add(BitCodeAbbrevOp(FIRST_EXTENSION_RECORD_ID)); + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // # of characters + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Blob)); // message + auto Abbrev = Stream.EmitAbbrev(Abv); + + // Write a message into the extension block. + SmallString<64> Message; + { + auto Ext = static_cast(getExtension()); + raw_svector_ostream OS(Message); + OS << "Hello from " << Ext->BlockName << " v" << Ext->MajorVersion << "." + << Ext->MinorVersion; + } + SmallVector Record; + Record.push_back(FIRST_EXTENSION_RECORD_ID); + Record.push_back(Message.size()); + Stream.EmitRecordWithBlob(Abbrev, Record, Message); +} + +TestModuleFileExtension::Reader::Reader(ModuleFileExtension *Ext, + const llvm::BitstreamCursor &InStream) + : ModuleFileExtensionReader(Ext), Stream(InStream) +{ + // Read the extension block. + SmallVector Record; + while (true) { + llvm::BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + switch (Entry.Kind) { + case llvm::BitstreamEntry::SubBlock: + case llvm::BitstreamEntry::EndBlock: + case llvm::BitstreamEntry::Error: + return; + + case llvm::BitstreamEntry::Record: + break; + } + + Record.clear(); + StringRef Blob; + unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob); + switch (RecCode) { + case FIRST_EXTENSION_RECORD_ID: { + StringRef Message = Blob.substr(0, Record[0]); + fprintf(stderr, "Read extension block message: %s\n", + Message.str().c_str()); + break; + } + } + } +} + +TestModuleFileExtension::Reader::~Reader() { } + +TestModuleFileExtension::~TestModuleFileExtension() { } + +ModuleFileExtensionMetadata +TestModuleFileExtension::getExtensionMetadata() const { + return { BlockName, MajorVersion, MinorVersion, UserInfo }; +} + +llvm::hash_code TestModuleFileExtension::hashExtension( + llvm::hash_code Code) const { + if (Hashed) { + Code = llvm::hash_combine(Code, BlockName); + Code = llvm::hash_combine(Code, MajorVersion); + Code = llvm::hash_combine(Code, MinorVersion); + Code = llvm::hash_combine(Code, UserInfo); + } + + return Code; +} + +std::unique_ptr +TestModuleFileExtension::createExtensionWriter(ASTWriter &) { + return std::unique_ptr(new Writer(this)); +} + +std::unique_ptr +TestModuleFileExtension::createExtensionReader( + const ModuleFileExtensionMetadata &Metadata, + ASTReader &Reader, serialization::ModuleFile &Mod, + const llvm::BitstreamCursor &Stream) +{ + assert(Metadata.BlockName == BlockName && "Wrong block name"); + if (std::make_pair(Metadata.MajorVersion, Metadata.MinorVersion) != + std::make_pair(MajorVersion, MinorVersion)) { + Reader.getDiags().Report(Mod.ImportLoc, + diag::err_test_module_file_extension_version) + << BlockName << Metadata.MajorVersion << Metadata.MinorVersion + << MajorVersion << MinorVersion; + return nullptr; + } + + return std::unique_ptr( + new TestModuleFileExtension::Reader(this, Stream)); +} diff --git a/clang/lib/Frontend/TestModuleFileExtension.h b/clang/lib/Frontend/TestModuleFileExtension.h new file mode 100644 index 000000000000..2a7245463cf7 --- /dev/null +++ b/clang/lib/Frontend/TestModuleFileExtension.h @@ -0,0 +1,71 @@ +//===-- TestModuleFileExtension.h - Module Extension Tester -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_CLANG_FRONTEND_TESTMODULEFILEEXTENSION_H +#define LLVM_CLANG_FRONTEND_TESTMODULEFILEEXTENSION_H + +#include "clang/Serialization/ModuleFileExtension.h" +#include "clang/Basic/LLVM.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Bitcode/BitstreamReader.h" +#include + +namespace clang { + +/// A module file extension used for testing purposes. +class TestModuleFileExtension : public ModuleFileExtension { + std::string BlockName; + unsigned MajorVersion; + unsigned MinorVersion; + bool Hashed; + std::string UserInfo; + + class Writer : public ModuleFileExtensionWriter { + public: + Writer(ModuleFileExtension *Ext) : ModuleFileExtensionWriter(Ext) { } + ~Writer() override; + + void writeExtensionContents(llvm::BitstreamWriter &Stream) override; + }; + + class Reader : public ModuleFileExtensionReader { + llvm::BitstreamCursor Stream; + + public: + ~Reader() override; + + Reader(ModuleFileExtension *Ext, const llvm::BitstreamCursor &InStream); + }; + +public: + TestModuleFileExtension(StringRef BlockName, + unsigned MajorVersion, + unsigned MinorVersion, + bool Hashed, + StringRef UserInfo) + : BlockName(BlockName), + MajorVersion(MajorVersion), MinorVersion(MinorVersion), + Hashed(Hashed), UserInfo(UserInfo) { } + ~TestModuleFileExtension() override; + + ModuleFileExtensionMetadata getExtensionMetadata() const override; + + llvm::hash_code hashExtension(llvm::hash_code Code) const override; + + std::unique_ptr + createExtensionWriter(ASTWriter &Writer) override; + + std::unique_ptr + createExtensionReader(const ModuleFileExtensionMetadata &Metadata, + ASTReader &Reader, serialization::ModuleFile &Mod, + const llvm::BitstreamCursor &Stream) override; +}; + +} // end namespace clang + +#endif // LLVM_CLANG_FRONTEND_TESTMODULEFILEEXTENSION_H diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index c54f659f09f6..c09e97c1584b 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -160,6 +160,12 @@ bool ChainedASTReaderListener::visitInputFile(StringRef Filename, return Continue; } +void ChainedASTReaderListener::readModuleFileExtension( + const ModuleFileExtensionMetadata &Metadata) { + First->readModuleFileExtension(Metadata); + Second->readModuleFileExtension(Metadata); +} + //===----------------------------------------------------------------------===// // PCH validator implementation //===----------------------------------------------------------------------===// @@ -3423,6 +3429,36 @@ static void updateModuleTimestamp(ModuleFile &MF) { OS << "Timestamp file\n"; } +/// \brief Given a cursor at the start of an AST file, scan ahead and drop the +/// cursor into the start of the given block ID, returning false on success and +/// true on failure. +static bool SkipCursorToBlock(BitstreamCursor &Cursor, unsigned BlockID) { + while (1) { + llvm::BitstreamEntry Entry = Cursor.advance(); + switch (Entry.Kind) { + case llvm::BitstreamEntry::Error: + case llvm::BitstreamEntry::EndBlock: + return true; + + case llvm::BitstreamEntry::Record: + // Ignore top-level records. + Cursor.skipRecord(Entry.ID); + break; + + case llvm::BitstreamEntry::SubBlock: + if (Entry.ID == BlockID) { + if (Cursor.EnterSubBlock(BlockID)) + return true; + // Found it! + return false; + } + + if (Cursor.SkipBlock()) + return true; + } + } +} + ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName, ModuleKind Type, SourceLocation ImportLoc, @@ -3480,6 +3516,12 @@ ASTReader::ASTReadResult ASTReader::ReadAST(const std::string &FileName, if (ASTReadResult Result = ReadASTBlock(F, ClientLoadCapabilities)) return Result; + // Read the extension blocks. + while (!SkipCursorToBlock(F.Stream, EXTENSION_BLOCK_ID)) { + if (ASTReadResult Result = ReadExtensionBlock(F)) + return Result; + } + // Once read, set the ModuleFile bit base offset and update the size in // bits of all files we've seen. F.GlobalBitOffset = TotalModulesSizeInBits; @@ -3738,14 +3780,13 @@ ASTReader::ReadASTCore(StringRef FileName, // This is used for compatibility with older PCH formats. bool HaveReadControlBlock = false; - while (1) { llvm::BitstreamEntry Entry = Stream.advance(); switch (Entry.Kind) { case llvm::BitstreamEntry::Error: - case llvm::BitstreamEntry::EndBlock: case llvm::BitstreamEntry::Record: + case llvm::BitstreamEntry::EndBlock: Error("invalid record at top-level of AST file"); return Failure; @@ -3800,6 +3841,79 @@ ASTReader::ReadASTCore(StringRef FileName, break; } } + + return Success; +} + +/// Parse a record and blob containing module file extension metadata. +static bool parseModuleFileExtensionMetadata( + const SmallVectorImpl &Record, + StringRef Blob, + ModuleFileExtensionMetadata &Metadata) { + if (Record.size() < 4) return true; + + Metadata.MajorVersion = Record[0]; + Metadata.MinorVersion = Record[1]; + + unsigned BlockNameLen = Record[2]; + unsigned UserInfoLen = Record[3]; + + if (BlockNameLen + UserInfoLen > Blob.size()) return true; + + Metadata.BlockName = std::string(Blob.data(), Blob.data() + BlockNameLen); + Metadata.UserInfo = std::string(Blob.data() + BlockNameLen, + Blob.data() + BlockNameLen + UserInfoLen); + return false; +} + +ASTReader::ASTReadResult ASTReader::ReadExtensionBlock(ModuleFile &F) { + BitstreamCursor &Stream = F.Stream; + + RecordData Record; + while (true) { + llvm::BitstreamEntry Entry = Stream.advance(); + switch (Entry.Kind) { + case llvm::BitstreamEntry::SubBlock: + if (Stream.SkipBlock()) + return Failure; + + continue; + + case llvm::BitstreamEntry::EndBlock: + return Success; + + case llvm::BitstreamEntry::Error: + return HadErrors; + + case llvm::BitstreamEntry::Record: + break; + } + + Record.clear(); + StringRef Blob; + unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob); + switch (RecCode) { + case EXTENSION_METADATA: { + ModuleFileExtensionMetadata Metadata; + if (parseModuleFileExtensionMetadata(Record, Blob, Metadata)) + return Failure; + + // Find a module file extension with this block name. + auto Known = ModuleFileExtensions.find(Metadata.BlockName); + if (Known == ModuleFileExtensions.end()) break; + + // Form a reader. + if (auto Reader = Known->second->createExtensionReader(Metadata, *this, + F, Stream)) { + F.ExtensionReaders.push_back(std::move(Reader)); + } + + break; + } + } + } + + return Success; } void ASTReader::InitializeContext() { @@ -3940,36 +4054,6 @@ void ASTReader::finalizeForWriting() { // Nothing to do for now. } -/// \brief Given a cursor at the start of an AST file, scan ahead and drop the -/// cursor into the start of the given block ID, returning false on success and -/// true on failure. -static bool SkipCursorToBlock(BitstreamCursor &Cursor, unsigned BlockID) { - while (1) { - llvm::BitstreamEntry Entry = Cursor.advance(); - switch (Entry.Kind) { - case llvm::BitstreamEntry::Error: - case llvm::BitstreamEntry::EndBlock: - return true; - - case llvm::BitstreamEntry::Record: - // Ignore top-level records. - Cursor.skipRecord(Entry.ID); - break; - - case llvm::BitstreamEntry::SubBlock: - if (Entry.ID == BlockID) { - if (Cursor.EnterSubBlock(BlockID)) - return true; - // Found it! - return false; - } - - if (Cursor.SkipBlock()) - return true; - } - } -} - /// \brief Reads and return the signature record from \p StreamFile's control /// block, or else returns 0. static ASTFileSignature readASTFileSignature(llvm::BitstreamReader &StreamFile){ @@ -4097,6 +4181,7 @@ namespace { bool ASTReader::readASTFileControlBlock( StringRef Filename, FileManager &FileMgr, const PCHContainerReader &PCHContainerRdr, + bool FindModuleFileExtensions, ASTReaderListener &Listener) { // Open the AST file. // FIXME: This allows use of the VFS; we do not allow use of the @@ -4126,7 +4211,8 @@ bool ASTReader::readASTFileControlBlock( RecordData Record; std::string ModuleDir; - while (1) { + bool DoneWithControlBlock = false; + while (!DoneWithControlBlock) { llvm::BitstreamEntry Entry = Stream.advance(); switch (Entry.Kind) { @@ -4159,7 +4245,8 @@ bool ASTReader::readASTFileControlBlock( } case llvm::BitstreamEntry::EndBlock: - return false; + DoneWithControlBlock = true; + break; case llvm::BitstreamEntry::Error: return true; @@ -4168,6 +4255,8 @@ bool ASTReader::readASTFileControlBlock( break; } + if (DoneWithControlBlock) break; + Record.clear(); StringRef Blob; unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob); @@ -4251,6 +4340,50 @@ bool ASTReader::readASTFileControlBlock( break; } } + + // Look for module file extension blocks, if requested. + if (FindModuleFileExtensions) { + while (!SkipCursorToBlock(Stream, EXTENSION_BLOCK_ID)) { + bool DoneWithExtensionBlock = false; + while (!DoneWithExtensionBlock) { + llvm::BitstreamEntry Entry = Stream.advance(); + + switch (Entry.Kind) { + case llvm::BitstreamEntry::SubBlock: + if (Stream.SkipBlock()) + return true; + + continue; + + case llvm::BitstreamEntry::EndBlock: + DoneWithExtensionBlock = true; + continue; + + case llvm::BitstreamEntry::Error: + return true; + + case llvm::BitstreamEntry::Record: + break; + } + + Record.clear(); + StringRef Blob; + unsigned RecCode = Stream.readRecord(Entry.ID, Record, &Blob); + switch (RecCode) { + case EXTENSION_METADATA: { + ModuleFileExtensionMetadata Metadata; + if (parseModuleFileExtensionMetadata(Record, Blob, Metadata)) + return true; + + Listener.readModuleFileExtension(Metadata); + break; + } + } + } + } + } + + return false; } bool ASTReader::isAcceptableASTFile( @@ -4261,6 +4394,7 @@ bool ASTReader::isAcceptableASTFile( SimplePCHValidator validator(LangOpts, TargetOpts, PPOpts, ExistingModuleCachePath, FileMgr); return !readASTFileControlBlock(Filename, FileMgr, PCHContainerRdr, + /*FindModuleFileExtensions=*/false, validator); } @@ -8483,13 +8617,15 @@ void ASTReader::pushExternalDeclIntoScope(NamedDecl *D, DeclarationName Name) { } } -ASTReader::ASTReader(Preprocessor &PP, ASTContext &Context, - const PCHContainerReader &PCHContainerRdr, - StringRef isysroot, bool DisableValidation, - bool AllowASTWithCompilerErrors, - bool AllowConfigurationMismatch, bool ValidateSystemInputs, - bool UseGlobalIndex, - std::unique_ptr ReadTimer) +ASTReader::ASTReader( + Preprocessor &PP, ASTContext &Context, + const PCHContainerReader &PCHContainerRdr, + ArrayRef> Extensions, + StringRef isysroot, bool DisableValidation, + bool AllowASTWithCompilerErrors, + bool AllowConfigurationMismatch, bool ValidateSystemInputs, + bool UseGlobalIndex, + std::unique_ptr ReadTimer) : Listener(new PCHValidator(PP, *this)), DeserializationListener(nullptr), OwnsDeserializationListener(false), SourceMgr(PP.getSourceManager()), FileMgr(PP.getFileManager()), PCHContainerRdr(PCHContainerRdr), @@ -8513,6 +8649,18 @@ ASTReader::ASTReader(Preprocessor &PP, ASTContext &Context, TotalModulesSizeInBits(0), NumCurrentElementsDeserializing(0), PassingDeclsToConsumer(false), ReadingKind(Read_None) { SourceMgr.setExternalSLocEntrySource(this); + + for (const auto &Ext : Extensions) { + auto BlockName = Ext->getExtensionMetadata().BlockName; + auto Known = ModuleFileExtensions.find(BlockName); + if (Known != ModuleFileExtensions.end()) { + Diags.Report(diag::warn_duplicate_module_file_extension) + << BlockName; + continue; + } + + ModuleFileExtensions.insert({BlockName, Ext}); + } } ASTReader::~ASTReader() { diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 01e9f220f975..c5a9755fed32 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -12,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "clang/Serialization/ASTWriter.h" +#include "clang/Serialization/ModuleFileExtension.h" #include "ASTCommon.h" #include "ASTReaderInternals.h" #include "MultiOnDiskHashTable.h" @@ -1094,7 +1095,11 @@ void ASTWriter::WriteBlockInfoBlock() { RECORD(PPD_MACRO_EXPANSION); RECORD(PPD_MACRO_DEFINITION); RECORD(PPD_INCLUSION_DIRECTIVE); - + + // Decls and Types block. + BLOCK(EXTENSION_BLOCK); + RECORD(EXTENSION_METADATA); + #undef RECORD #undef BLOCK Stream.ExitBlock(); @@ -3876,6 +3881,40 @@ void ASTWriter::WriteOptimizePragmaOptions(Sema &SemaRef) { Stream.EmitRecord(OPTIMIZE_PRAGMA_OPTIONS, Record); } +void ASTWriter::WriteModuleFileExtension(ModuleFileExtensionWriter &Writer) { + // Enter the extension block. + Stream.EnterSubblock(EXTENSION_BLOCK_ID, 4); + + // Emit the metadata record abbreviation. + llvm::BitCodeAbbrev *Abv = new llvm::BitCodeAbbrev(); + Abv->Add(llvm::BitCodeAbbrevOp(EXTENSION_METADATA)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::VBR, 6)); + Abv->Add(llvm::BitCodeAbbrevOp(llvm::BitCodeAbbrevOp::Blob)); + unsigned Abbrev = Stream.EmitAbbrev(Abv); + + // Emit the metadata record. + RecordData Record; + auto Metadata = Writer.getExtension()->getExtensionMetadata(); + Record.push_back(EXTENSION_METADATA); + Record.push_back(Metadata.MajorVersion); + Record.push_back(Metadata.MinorVersion); + Record.push_back(Metadata.BlockName.size()); + Record.push_back(Metadata.UserInfo.size()); + SmallString<64> Buffer; + Buffer += Metadata.BlockName; + Buffer += Metadata.UserInfo; + Stream.EmitRecordWithBlob(Abbrev, Record, Buffer); + + // Emit the contents of the extension block. + Writer.writeExtensionContents(Stream); + + // Exit the extension block. + Stream.ExitBlock(); +} + //===----------------------------------------------------------------------===// // General Serialization Routines //===----------------------------------------------------------------------===// @@ -3979,7 +4018,10 @@ void ASTWriter::SetSelectorOffset(Selector Sel, uint32_t Offset) { SelectorOffsets[ID - FirstSelectorID] = Offset; } -ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream, bool IncludeTimestamps) +ASTWriter::ASTWriter( + llvm::BitstreamWriter &Stream, + ArrayRef> Extensions, + bool IncludeTimestamps) : Stream(Stream), Context(nullptr), PP(nullptr), Chain(nullptr), WritingModule(nullptr), IncludeTimestamps(IncludeTimestamps), WritingAST(false), DoneWritingDeclsAndTypes(false), @@ -3999,7 +4041,12 @@ ASTWriter::ASTWriter(llvm::BitstreamWriter &Stream, bool IncludeTimestamps) DeclVarAbbrev(0), DeclFieldAbbrev(0), DeclEnumAbbrev(0), DeclObjCIvarAbbrev(0), DeclCXXMethodAbbrev(0), DeclRefExprAbbrev(0), CharacterLiteralAbbrev(0), IntegerLiteralAbbrev(0), - ExprImplicitCastAbbrev(0) {} + ExprImplicitCastAbbrev(0) { + for (const auto &Ext : Extensions) { + if (auto Writer = Ext->createExtensionWriter(*this)) + ModuleFileExtensionWriters.push_back(std::move(Writer)); + } +} ASTWriter::~ASTWriter() { llvm::DeleteContainerSeconds(FileDeclIDs); @@ -4396,7 +4443,6 @@ uint64_t ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, WriteCXXCtorInitializersOffsets(); WriteFileDeclIDsMap(); WriteSourceManagerBlock(Context.getSourceManager(), PP); - WriteComments(); WritePreprocessor(PP, isModule); WriteHeaderSearch(PP.getHeaderSearchInfo()); @@ -4526,6 +4572,10 @@ uint64_t ASTWriter::WriteASTCore(Sema &SemaRef, StringRef isysroot, Stream.EmitRecord(STATISTICS, Record); Stream.ExitBlock(); + // Write the module file extension blocks. + for (const auto &ExtWriter : ModuleFileExtensionWriters) + WriteModuleFileExtension(*ExtWriter); + return Signature; } diff --git a/clang/lib/Serialization/CMakeLists.txt b/clang/lib/Serialization/CMakeLists.txt index d885db22975e..95b33c388c56 100644 --- a/clang/lib/Serialization/CMakeLists.txt +++ b/clang/lib/Serialization/CMakeLists.txt @@ -15,6 +15,7 @@ add_clang_library(clangSerialization GeneratePCH.cpp GlobalModuleIndex.cpp Module.cpp + ModuleFileExtension.cpp ModuleManager.cpp ADDITIONAL_HEADERS diff --git a/clang/lib/Serialization/GeneratePCH.cpp b/clang/lib/Serialization/GeneratePCH.cpp index 9de2fdb75a07..4a2255ab6d39 100644 --- a/clang/lib/Serialization/GeneratePCH.cpp +++ b/clang/lib/Serialization/GeneratePCH.cpp @@ -23,13 +23,15 @@ using namespace clang; -PCHGenerator::PCHGenerator(const Preprocessor &PP, StringRef OutputFile, - clang::Module *Module, StringRef isysroot, - std::shared_ptr Buffer, - bool AllowASTWithErrors, bool IncludeTimestamps) +PCHGenerator::PCHGenerator( + const Preprocessor &PP, StringRef OutputFile, + clang::Module *Module, StringRef isysroot, + std::shared_ptr Buffer, + ArrayRef> Extensions, + bool AllowASTWithErrors, bool IncludeTimestamps) : PP(PP), OutputFile(OutputFile), Module(Module), isysroot(isysroot.str()), SemaPtr(nullptr), Buffer(Buffer), Stream(Buffer->Data), - Writer(Stream, IncludeTimestamps), + Writer(Stream, Extensions, IncludeTimestamps), AllowASTWithErrors(AllowASTWithErrors) { Buffer->IsComplete = false; } diff --git a/clang/lib/Serialization/ModuleFileExtension.cpp b/clang/lib/Serialization/ModuleFileExtension.cpp new file mode 100644 index 000000000000..81dcfd60ce8e --- /dev/null +++ b/clang/lib/Serialization/ModuleFileExtension.cpp @@ -0,0 +1,22 @@ +//===-- ModuleFileExtension.cpp - Module File Extensions ------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +#include "clang/Serialization/ModuleFileExtension.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/Support/raw_ostream.h" +using namespace clang; + +ModuleFileExtension::~ModuleFileExtension() { } + +llvm::hash_code ModuleFileExtension::hashExtension(llvm::hash_code Code) const { + return Code; +} + +ModuleFileExtensionWriter::~ModuleFileExtensionWriter() { } + +ModuleFileExtensionReader::~ModuleFileExtensionReader() { } diff --git a/clang/test/Modules/Inputs/ExtensionTestA.h b/clang/test/Modules/Inputs/ExtensionTestA.h new file mode 100644 index 000000000000..fee0bb953e7b --- /dev/null +++ b/clang/test/Modules/Inputs/ExtensionTestA.h @@ -0,0 +1 @@ +extern int ExtensionA; diff --git a/clang/test/Modules/Inputs/module.map b/clang/test/Modules/Inputs/module.map index 27751d8fd51f..d195e2f0980f 100644 --- a/clang/test/Modules/Inputs/module.map +++ b/clang/test/Modules/Inputs/module.map @@ -375,3 +375,7 @@ module DebugSubmodules { export * } } + +module ExtensionTestA { + header "ExtensionTestA.h" +} diff --git a/clang/test/Modules/extensions.c b/clang/test/Modules/extensions.c new file mode 100644 index 000000000000..1858f02e681a --- /dev/null +++ b/clang/test/Modules/extensions.c @@ -0,0 +1,44 @@ +// Test creation of modules that include extension blocks. +// RUN: rm -rf %t +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -ftest-module-file-extension=clang.testB:2:3:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s + +// Make sure the extension blocks are actually there. +// RUN: llvm-bcanalyzer %t/ExtensionTestA.pcm | FileCheck -check-prefix=CHECK-BCANALYZER %s +// RUN: %clang_cc1 -module-file-info %t/ExtensionTestA.pcm | FileCheck -check-prefix=CHECK-INFO %s + +// Make sure that the readers are able to check the metadata. +// RUN: rm -rf %t +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -ftest-module-file-extension=clang.testB:2:3:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:1:3:0:user_info_for_A -ftest-module-file-extension=clang.testB:3:2:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s -verify + +// Make sure that extension blocks can be part of the module hash. +// We test this in an obscure way, by making sure we don't get conflicts when +// using different "versions" of the extensions. Above, the "-verify" test +// checks that such conflicts produce errors. +// RUN: rm -rf %t +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:1:5:1:user_info_for_A -ftest-module-file-extension=clang.testB:2:3:1:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:1:3:1:user_info_for_A -ftest-module-file-extension=clang.testB:3:2:1:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -ftest-module-file-extension=clang.testA:2:5:0:user_info_for_A -ftest-module-file-extension=clang.testB:7:3:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s + +// Make sure we can read the message back. +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -ftest-module-file-extension=clang.testB:2:3:0:user_info_for_B -fmodules-cache-path=%t -I %S/Inputs %s > %t.log 2>&1 +// RUN: FileCheck -check-prefix=CHECK-MESSAGE %s < %t.log + +// Make sure we diagnose duplicate module file extensions. +// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fdisable-module-hash -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -ftest-module-file-extension=clang.testA:1:5:0:user_info_for_A -fmodules-cache-path=%t -I %S/Inputs %s > %t.log 2>&1 +// RUN: FileCheck -check-prefix=CHECK-DUPLICATE %s < %t.log + +#include "ExtensionTestA.h" +// expected-error@-1{{test module file extension 'clang.testA' has different version (1.5) than expected (1.3)}} +// expected-error@-2{{test module file extension 'clang.testB' has different version (2.3) than expected (3.2)}} + +// CHECK-BCANALYZER: {{Block ID.*EXTENSION_BLOCK}} +// CHECK-BCANALYZER: {{100.00.*EXTENSION_METADATA}} + +// CHECK-INFO: Module file extension 'clang.testA' 1.5: user_info_for_A +// CHECK-INFO: Module file extension 'clang.testB' 2.3: user_info_for_B + +// CHECK-MESSAGE: Read extension block message: Hello from clang.testA v1.5 +// CHECK-MESSAGE: Read extension block message: Hello from clang.testB v2.3 + +// CHECK-DUPLICATE: warning: duplicate module file extension block name 'clang.testA'