replica
/
hanchenye-llvm-project
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Macro history (de-)serialization. Deserialization currently reads only the latest macro definition. Needs more work. 

Summary: Passes all tests (+ the new one with code completion), but needs a thorough review in part related to modules.

Reviewers: doug.gregor

Reviewed By: alexfh

CC: cfe-commits, rsmith

Differential Revision: http://llvm-reviews.chandlerc.com/D41

llvm-svn: 164610
This commit is contained in:
Alexander Kornienko 2012-09-25 17:18:14 +00:00
parent 24719e3236
commit 1d26c02722
8 changed files with 166 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
clang/include/clang/Basic/IdentifierTable.h
View File

@ -54,6 +54,7 @@ class IdentifierInfo {
// are for builtins.
unsigned ObjCOrBuiltinID :11;
bool HasMacro : 1; // True if there is a #define for this.
bool HadMacro : 1; // True if there was a #define for this.
bool IsExtension : 1; // True if identifier is a lang extension.
bool IsCXX11CompatKeyword : 1; // True if identifier is a keyword in C++11.
bool IsPoisoned : 1; // True if identifier is poisoned.
@ -70,8 +71,8 @@ class IdentifierInfo {
// stored externally.
bool IsModulesImport : 1; // True if this is the 'import' contextual
// keyword.
// 1 bit left in 32-bit word.
// 32-bit word is filled.
void *FETokenInfo; // Managed by the language front-end.
llvm::StringMapEntry<IdentifierInfo*> *Entry;
@ -133,10 +134,18 @@ public:
if (HasMacro == Val) return;
HasMacro = Val;
if (Val)
if (Val) {
NeedsHandleIdentifier = 1;
else
HadMacro = true;
} else {
RecomputeNeedsHandleIdentifier();
}
}
/// \brief Returns true if this identifier was \#defined to some value at any
/// moment. In this case there should be an entry for the identifier in the
/// macro history table in Preprocessor.
bool hadMacroDefinition() const {
return HadMacro;
}
/// getTokenID - If this is a source-language token (e.g. 'for'), this API

12
clang/include/clang/Lex/Preprocessor.h
View File

@ -366,8 +366,6 @@ private: // Cached tokens state.
/// allocation.
MacroInfoChain *MICache;
MacroInfo *getInfoForMacro(IdentifierInfo *II) const;
public:
Preprocessor(DiagnosticsEngine &diags, LangOptions &opts,
const TargetInfo *target,
@ -467,9 +465,17 @@ public:
if (!II->hasMacroDefinition())
return 0;
return getInfoForMacro(II);
MacroInfo *MI = getMacroInfoHistory(II);
assert(MI->getUndefLoc().isInvalid() && "Macro is undefined!");
return MI;
}
/// \brief Given an identifier, return the (probably #undef'd) MacroInfo
/// representing the most recent macro definition. One can iterate over all
/// previous macro definitions from it. This method should only be called for
/// identifiers that hadMacroDefinition().
MacroInfo *getMacroInfoHistory(IdentifierInfo *II) const;
/// \brief Specify a macro for this identifier.
void setMacroInfo(IdentifierInfo *II, MacroInfo *MI,
bool LoadedFromAST = false);

1
clang/lib/Basic/IdentifierTable.cpp
View File

@ -33,6 +33,7 @@ IdentifierInfo::IdentifierInfo() {
TokenID = tok::identifier;
ObjCOrBuiltinID = 0;
HasMacro = false;
HadMacro = false;
IsExtension = false;
IsCXX11CompatKeyword = false;
IsPoisoned = false;

9
clang/lib/Lex/PPMacroExpansion.cpp
View File

@ -32,8 +32,8 @@
#include <ctime>
using namespace clang;
MacroInfo *Preprocessor::getInfoForMacro(IdentifierInfo *II) const {
assert(II->hasMacroDefinition() && "Identifier is not a macro!");
MacroInfo *Preprocessor::getMacroInfoHistory(IdentifierInfo *II) const {
assert(II->hadMacroDefinition() && "Identifier has not been not a macro!");
macro_iterator Pos = Macros.find(II);
if (Pos == Macros.end()) {
@ -42,7 +42,6 @@ MacroInfo *Preprocessor::getInfoForMacro(IdentifierInfo *II) const {
Pos = Macros.find(II);
}
assert(Pos != Macros.end() && "Identifier macro info is missing!");
assert(Pos->second->getUndefLoc().isInvalid() && "Macro is undefined!");
return Pos->second;
}
@ -51,9 +50,11 @@ MacroInfo *Preprocessor::getInfoForMacro(IdentifierInfo *II) const {
void Preprocessor::setMacroInfo(IdentifierInfo *II, MacroInfo *MI,
bool LoadedFromAST) {
assert(MI && "MacroInfo should be non-zero!");
assert((LoadedFromAST || MI->getUndefLoc().isInvalid()) &&
"Undefined macros can only be registered when just LoadedFromAST");
MI->setPreviousDefinition(Macros[II]);
Macros[II] = MI;
II->setHasMacroDefinition(true);
II->setHasMacroDefinition(MI->getUndefLoc().isInvalid());
if (II->isFromAST() && !LoadedFromAST)
II->setChangedSinceDeserialization();
}

33
clang/lib/Serialization/ASTReader.cpp
View File

@ -527,6 +527,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
return II;
}
unsigned ObjCOrBuiltinID = ReadUnalignedLE16(d);
unsigned Bits = ReadUnalignedLE16(d);
bool CPlusPlusOperatorKeyword = Bits & 0x01;
Bits >>= 1;
@ -536,13 +537,13 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
Bits >>= 1;
bool ExtensionToken = Bits & 0x01;
Bits >>= 1;
bool hadMacroDefinition = Bits & 0x01;
Bits >>= 1;
bool hasMacroDefinition = Bits & 0x01;
Bits >>= 1;
unsigned ObjCOrBuiltinID = Bits & 0x7FF;
Bits >>= 11;
assert(Bits == 0 && "Extra bits in the identifier?");
DataLen -= 6;
DataLen -= 8;
// Build the IdentifierInfo itself and link the identifier ID with
// the new IdentifierInfo.
@ -570,7 +571,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
// If this identifier is a macro, deserialize the macro
// definition.
if (hasMacroDefinition) {
if (hadMacroDefinition) {
// FIXME: Check for conflicts?
uint32_t Offset = ReadUnalignedLE32(d);
unsigned LocalSubmoduleID = ReadUnalignedLE32(d);
@ -590,10 +591,10 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
// module is not yet visible.
Reader.HiddenNamesMap[Owner].push_back(II);
}
}
}
}
Reader.setIdentifierIsMacro(II, F, Offset, Visible);
Reader.setIdentifierIsMacro(II, F, Offset, Visible && hasMacroDefinition);
DataLen -= 8;
}
@ -1312,18 +1313,21 @@ void ASTReader::ReadMacroRecord(ModuleFile &F, uint64_t Offset) {
Error("macro must have a name in AST file");
return;
}
SourceLocation Loc = ReadSourceLocation(F, Record[1]);
bool isUsed = Record[2];
unsigned NextIndex = 1;
SourceLocation Loc = ReadSourceLocation(F, Record, NextIndex);
MacroInfo *MI = PP.AllocateMacroInfo(Loc);
MI->setIsUsed(isUsed);
SourceLocation UndefLoc = ReadSourceLocation(F, Record, NextIndex);
if (UndefLoc.isValid())
MI->setUndefLoc(UndefLoc);
MI->setIsUsed(Record[NextIndex++]);
MI->setIsFromAST();
bool IsPublic = Record[3];
unsigned NextIndex = 4;
bool IsPublic = Record[NextIndex++];
MI->setVisibility(IsPublic, ReadSourceLocation(F, Record, NextIndex));
if (RecType == PP_MACRO_FUNCTION_LIKE) {
// Decode function-like macro info.
bool isC99VarArgs = Record[NextIndex++];
@ -2551,6 +2555,7 @@ void ASTReader::makeNamesVisible(const HiddenNames &Names) {
D->Hidden = false;
else {
IdentifierInfo *II = Names[I].get<IdentifierInfo *>();
// FIXME: Check if this works correctly with macro history.
if (!II->hasMacroDefinition()) {
// Make sure that this macro hasn't been #undef'd in the mean-time.
llvm::DenseMap<IdentifierInfo*, MacroInfo*>::iterator Known

186
clang/lib/Serialization/ASTWriter.cpp
View File

@ -1674,102 +1674,112 @@ void ASTWriter::WritePreprocessor(const Preprocessor &PP, bool IsModule) {
SmallVector<std::pair<const IdentifierInfo *, MacroInfo *>, 2>
MacrosToEmit;
llvm::SmallPtrSet<const IdentifierInfo*, 4> MacroDefinitionsSeen;
for (Preprocessor::macro_iterator I = PP.macro_begin(Chain == 0),
for (Preprocessor::macro_iterator I = PP.macro_begin(Chain == 0),
E = PP.macro_end(Chain == 0);
I != E; ++I) {
// FIXME: We'll need to store macro history in PCH.
if (I->first->hasMacroDefinition()) {
if (!IsModule || I->second->isPublic()) {
MacroDefinitionsSeen.insert(I->first);
MacrosToEmit.push_back(std::make_pair(I->first, I->second));
}
if (!IsModule || I->second->isPublic()) {
MacroDefinitionsSeen.insert(I->first);
MacrosToEmit.push_back(std::make_pair(I->first, I->second));
}
}
// Sort the set of macro definitions that need to be serialized by the
// name of the macro, to provide a stable ordering.
llvm::array_pod_sort(MacrosToEmit.begin(), MacrosToEmit.end(),
llvm::array_pod_sort(MacrosToEmit.begin(), MacrosToEmit.end(),
&compareMacroDefinitions);
// Resolve any identifiers that defined macros at the time they were
// deserialized, adding them to the list of macros to emit (if appropriate).
for (unsigned I = 0, N = DeserializedMacroNames.size(); I != N; ++I) {
IdentifierInfo *Name
= const_cast<IdentifierInfo *>(DeserializedMacroNames[I]);
if (Name->hasMacroDefinition() && MacroDefinitionsSeen.insert(Name))
if (Name->hadMacroDefinition() && MacroDefinitionsSeen.insert(Name))
MacrosToEmit.push_back(std::make_pair(Name, PP.getMacroInfo(Name)));
}
for (unsigned I = 0, N = MacrosToEmit.size(); I != N; ++I) {
const IdentifierInfo *Name = MacrosToEmit[I].first;
MacroInfo *MI = MacrosToEmit[I].second;
if (!MI)
continue;
// Don't emit builtin macros like __LINE__ to the AST file unless they have
// been redefined by the header (in which case they are not isBuiltinMacro).
// Also skip macros from a AST file if we're chaining.
// FIXME: There is a (probably minor) optimization we could do here, if
// the macro comes from the original PCH but the identifier comes from a
// chained PCH, by storing the offset into the original PCH rather than
// writing the macro definition a second time.
if (MI->isBuiltinMacro() ||
(Chain &&
Name->isFromAST() && !Name->hasChangedSinceDeserialization() &&
MI->isFromAST() && !MI->hasChangedAfterLoad()))
continue;
AddIdentifierRef(Name, Record);
MacroOffsets[Name] = Stream.GetCurrentBitNo();
Record.push_back(MI->getDefinitionLoc().getRawEncoding());
Record.push_back(MI->isUsed());
Record.push_back(MI->isPublic());
AddSourceLocation(MI->getVisibilityLocation(), Record);
unsigned Code;
if (MI->isObjectLike()) {
Code = PP_MACRO_OBJECT_LIKE;
} else {
Code = PP_MACRO_FUNCTION_LIKE;
Record.push_back(MI->isC99Varargs());
Record.push_back(MI->isGNUVarargs());
Record.push_back(MI->getNumArgs());
for (MacroInfo::arg_iterator I = MI->arg_begin(), E = MI->arg_end();
I != E; ++I)
AddIdentifierRef(*I, Record);
// History of macro definitions for this identifier in chronological order.
SmallVector<MacroInfo*, 8> MacroHistory;
while (MI) {
MacroHistory.push_back(MI);
MI = MI->getPreviousDefinition();
}
// If we have a detailed preprocessing record, record the macro definition
// ID that corresponds to this macro.
if (PPRec)
Record.push_back(MacroDefinitions[PPRec->findMacroDefinition(MI)]);
while (!MacroHistory.empty()) {
MI = MacroHistory.pop_back_val();
Stream.EmitRecord(Code, Record);
Record.clear();
// Don't emit builtin macros like __LINE__ to the AST file unless they
// have been redefined by the header (in which case they are not
// isBuiltinMacro).
// Also skip macros from a AST file if we're chaining.
// Emit the tokens array.
for (unsigned TokNo = 0, e = MI->getNumTokens(); TokNo != e; ++TokNo) {
// Note that we know that the preprocessor does not have any annotation
// tokens in it because they are created by the parser, and thus can't be
// in a macro definition.
const Token &Tok = MI->getReplacementToken(TokNo);
// FIXME: There is a (probably minor) optimization we could do here, if
// the macro comes from the original PCH but the identifier comes from a
// chained PCH, by storing the offset into the original PCH rather than
// writing the macro definition a second time.
if (MI->isBuiltinMacro() ||
(Chain &&
Name->isFromAST() && !Name->hasChangedSinceDeserialization() &&
MI->isFromAST() && !MI->hasChangedAfterLoad()))
continue;
Record.push_back(Tok.getLocation().getRawEncoding());
Record.push_back(Tok.getLength());
AddIdentifierRef(Name, Record);
MacroOffsets[Name] = Stream.GetCurrentBitNo();
AddSourceLocation(MI->getDefinitionLoc(), Record);
AddSourceLocation(MI->getUndefLoc(), Record);
Record.push_back(MI->isUsed());
Record.push_back(MI->isPublic());
AddSourceLocation(MI->getVisibilityLocation(), Record);
unsigned Code;
if (MI->isObjectLike()) {
Code = PP_MACRO_OBJECT_LIKE;
} else {
Code = PP_MACRO_FUNCTION_LIKE;
// FIXME: When reading literal tokens, reconstruct the literal pointer if
// it is needed.
AddIdentifierRef(Tok.getIdentifierInfo(), Record);
// FIXME: Should translate token kind to a stable encoding.
Record.push_back(Tok.getKind());
// FIXME: Should translate token flags to a stable encoding.
Record.push_back(Tok.getFlags());
Record.push_back(MI->isC99Varargs());
Record.push_back(MI->isGNUVarargs());
Record.push_back(MI->getNumArgs());
for (MacroInfo::arg_iterator I = MI->arg_begin(), E = MI->arg_end();
I != E; ++I)
AddIdentifierRef(*I, Record);
}
Stream.EmitRecord(PP_TOKEN, Record);
// If we have a detailed preprocessing record, record the macro definition
// ID that corresponds to this macro.
if (PPRec)
Record.push_back(MacroDefinitions[PPRec->findMacroDefinition(MI)]);
Stream.EmitRecord(Code, Record);
Record.clear();
// Emit the tokens array.
for (unsigned TokNo = 0, e = MI->getNumTokens(); TokNo != e; ++TokNo) {
// Note that we know that the preprocessor does not have any annotation
// tokens in it because they are created by the parser, and thus can't
// be in a macro definition.
const Token &Tok = MI->getReplacementToken(TokNo);
Record.push_back(Tok.getLocation().getRawEncoding());
Record.push_back(Tok.getLength());
// FIXME: When reading literal tokens, reconstruct the literal pointer
// if it is needed.
AddIdentifierRef(Tok.getIdentifierInfo(), Record);
// FIXME: Should translate token kind to a stable encoding.
Record.push_back(Tok.getKind());
// FIXME: Should translate token flags to a stable encoding.
Record.push_back(Tok.getFlags());
Stream.EmitRecord(PP_TOKEN, Record);
Record.clear();
}
++NumMacros;
}
++NumMacros;
}
Stream.ExitBlock();
}
@ -2496,17 +2506,17 @@ class ASTIdentifierTableTrait {
II->getFETokenInfo<void>())
return true;
return hasMacroDefinition(II, Macro);
return hadMacroDefinition(II, Macro);
}
bool hasMacroDefinition(IdentifierInfo *II, MacroInfo *&Macro) {
if (!II->hasMacroDefinition())
bool hadMacroDefinition(IdentifierInfo *II, MacroInfo *&Macro) {
if (!II->hadMacroDefinition())
return false;
if (Macro || (Macro = PP.getMacroInfo(II)))
if (Macro || (Macro = PP.getMacroInfoHistory(II)))
return !Macro->isBuiltinMacro() && (!IsModule || Macro->isPublic());
return false;
return false;
}
public:
@ -2530,10 +2540,11 @@ public:
unsigned DataLen = 4; // 4 bytes for the persistent ID << 1
MacroInfo *Macro = 0;
if (isInterestingIdentifier(II, Macro)) {
DataLen += 2; // 2 bytes for builtin ID, flags
if (hasMacroDefinition(II, Macro))
DataLen += 2; // 2 bytes for builtin ID
DataLen += 2; // 2 bytes for flags
if (hadMacroDefinition(II, Macro))
DataLen += 8;
for (IdentifierResolver::iterator D = IdResolver.begin(II),
DEnd = IdResolver.end();
D != DEnd; ++D)
@ -2564,23 +2575,26 @@ public:
}
clang::io::Emit32(Out, (ID << 1) | 0x01);
uint32_t Bits = 0;
bool HasMacroDefinition = hasMacroDefinition(II, Macro);
Bits = (uint32_t)II->getObjCOrBuiltinID();
assert((Bits & 0x7ff) == Bits && "ObjCOrBuiltinID too big for ASTReader.");
uint32_t Bits = (uint32_t)II->getObjCOrBuiltinID();
assert((Bits & 0xffff) == Bits && "ObjCOrBuiltinID too big for ASTReader.");
clang::io::Emit16(Out, Bits);
Bits = 0;
bool HadMacroDefinition = hadMacroDefinition(II, Macro);
bool HasMacroDefinition = HadMacroDefinition && II->hasMacroDefinition();
Bits = (Bits << 1) | unsigned(HasMacroDefinition);
Bits = (Bits << 1) | unsigned(HadMacroDefinition);
Bits = (Bits << 1) | unsigned(II->isExtensionToken());
Bits = (Bits << 1) | unsigned(II->isPoisoned());
Bits = (Bits << 1) | unsigned(II->hasRevertedTokenIDToIdentifier());
Bits = (Bits << 1) | unsigned(II->isCPlusPlusOperatorKeyword());
clang::io::Emit16(Out, Bits);
if (HasMacroDefinition) {
if (HadMacroDefinition) {
clang::io::Emit32(Out, Writer.getMacroOffset(II));
clang::io::Emit32(Out,
clang::io::Emit32(Out,
Writer.inferSubmoduleIDFromLocation(Macro->getDefinitionLoc()));
}
// Emit the declaration IDs in reverse order, because the
// IdentifierResolver provides the declarations as they would be
// visible (e.g., the function "stat" would come before the struct
@ -4447,7 +4461,7 @@ void ASTWriter::ReaderInitialized(ASTReader *Reader) {
void ASTWriter::IdentifierRead(IdentID ID, IdentifierInfo *II) {
IdentifierIDs[II] = ID;
if (II->hasMacroDefinition())
if (II->hadMacroDefinition())
DeserializedMacroNames.push_back(II);
}

7
clang/test/CodeCompletion/Inputs/macros.h
View File

@ -2,3 +2,10 @@
#define BAR(X, Y) X, Y
#define IDENTITY(X) X
#define WIBBLE(...)
#define DEAD_MACRO
#undef DEAD_MACRO
#define MACRO_WITH_HISTORY a
#undef MACRO_WITH_HISTORY
#define MACRO_WITH_HISTORY b, c
#undef MACRO_WITH_HISTORY
#define MACRO_WITH_HISTORY(X, Y) X->Y

12
clang/test/CodeCompletion/macros.c
View File

@ -13,11 +13,15 @@ void test(struct Point *p) {
// RUN: %clang_cc1 -include %S/Inputs/macros.h -fsyntax-only -code-completion-macros -code-completion-at=%s:14:9 %s -o - | FileCheck -check-prefix=CC2 %s
case
}
// RUN: %clang_cc1 -include %S/Inputs/macros.h -fsyntax-only -code-completion-macros -code-completion-at=%s:17:7 %s -o - | FileCheck -check-prefix=CC3 %s
#ifdef Q
#endif
// Run the same tests, this time with macros loaded from the PCH file.
// RUN: %clang_cc1 -emit-pch -o %t %S/Inputs/macros.h
// RUN: %clang_cc1 -include-pch %t -fsyntax-only -code-completion-macros -code-completion-at=%s:12:14 %s -o - | FileCheck -check-prefix=CC1 %s
// RUN: %clang_cc1 -include-pch %t -fsyntax-only -code-completion-macros -code-completion-at=%s:14:9 %s -o - | FileCheck -check-prefix=CC2 %s
// RUN: %clang_cc1 -include-pch %t -fsyntax-only -code-completion-macros -code-completion-at=%s:17:7 %s -o - | FileCheck -check-prefix=CC3 %s
// CC1: color
// CC1: x
@ -29,6 +33,14 @@ void test(struct Point *p) {
// CC2: FOO
// CC2: Green
// CC2: IDENTITY(<#X#>)
// CC2: MACRO_WITH_HISTORY(<#X#>, <#Y#>)
// CC2: Red
// CC2: WIBBLE
// CC3: BAR
// CC3: DEAD_MACRO
// CC3: FOO
// CC3: IDENTITY
// CC3: MACRO_WITH_HISTORY
// CC3: WIBBLE
}