replica
/
hanchenye-llvm-project
2
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement support for lexing from a pre-constructed token stream. 

Use this support to implement function-like macro argument preexpansion.

This implements test/Preprocessor/macro_fn_preexpand.c

llvm-svn: 38724
This commit is contained in:
Chris Lattner 2006-07-16 18:16:58 +00:00
parent 1e17abb2fd
commit 7667d0d942
5 changed files with 166 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
clang/Lex/MacroExpander.cpp
View File

@ -70,6 +70,40 @@ bool MacroArgs::ArgNeedsPreexpansion(unsigned ArgNo) const {
return false;
}
/// getPreExpArgument - Return the pre-expanded form of the specified
/// argument.
const std::vector<LexerToken> &
MacroArgs::getPreExpArgument(unsigned Arg, Preprocessor &PP) {
assert(Arg < UnexpArgTokens.size() && "Invalid argument number!");
// If we have already computed this, return it.
if (PreExpArgTokens.empty())
PreExpArgTokens.resize(UnexpArgTokens.size());
std::vector<LexerToken> &Result = PreExpArgTokens[Arg];
if (!Result.empty()) return Result;
// Otherwise, we have to pre-expand this argument, populating Result. To do
// this, we set up a fake MacroExpander to lex from the unexpanded argument
// list. With this installed, we lex expanded tokens until we hit the EOF
// token at the end of the unexp list.
PP.EnterTokenStream(UnexpArgTokens[Arg]);
// Lex all of the macro-expanded tokens into Result.
do {
Result.push_back(LexerToken());
PP.Lex(Result.back());
} while (Result.back().getKind() != tok::eof);
// Pop the token stream off the top of the stack. We know that the internal
// pointer inside of it is to the "end" of the token stream, but the stack
// will not otherwise be popped until the next token is lexed. The problem is
// that the token may be lexed sometime after the vector of tokens itself is
// destroyed, which would be badness.
PP.RemoveTopOfLexerStack();
return Result;
}
/// StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of
/// tokens into the literal string token that should be produced by the C #
@ -83,9 +117,9 @@ static LexerToken StringifyArgument(const std::vector<LexerToken> &Toks,
// Stringify all the tokens.
std::string Result = "\"";
// FIXME: Optimize this loop to not use std::strings.
for (unsigned i = 0, e = Toks.size()-1 /*no eof*/; i != e; ++i) {
const LexerToken &Tok = Toks[i];
// FIXME: Optimize this.
if (i != 0 && Tok.hasLeadingSpace())
Result += ' ';
@ -163,25 +197,49 @@ const LexerToken &MacroArgs::getStringifiedArgument(unsigned ArgNo,
// MacroExpander Implementation
//===----------------------------------------------------------------------===//
/// Create a macro expander for the specified macro with the specified actual
/// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
MacroExpander::MacroExpander(LexerToken &Tok, MacroArgs *Actuals,
Preprocessor &pp)
: Macro(*Tok.getIdentifierInfo()->getMacroInfo()),
: Macro(Tok.getIdentifierInfo()->getMacroInfo()),
ActualArgs(Actuals), PP(pp), CurToken(0),
InstantiateLoc(Tok.getLocation()),
AtStartOfLine(Tok.isAtStartOfLine()),
HasLeadingSpace(Tok.hasLeadingSpace()) {
MacroTokens = &Macro.getReplacementTokens();
MacroTokens = &Macro->getReplacementTokens();
// If this is a function-like macro, expand the arguments and change
// MacroTokens to point to the expanded tokens.
if (Macro.isFunctionLike() && Macro.getNumArgs())
if (Macro->isFunctionLike() && Macro->getNumArgs())
ExpandFunctionArguments();
// Mark the macro as currently disabled, so that it is not recursively
// expanded. The macro must be disabled only after argument pre-expansion of
// function-like macro arguments occurs.
Macro->DisableMacro();
}
/// Create a macro expander for the specified token stream. This does not
/// take ownership of the specified token vector.
MacroExpander::MacroExpander(const std::vector<LexerToken> &TokStream,
Preprocessor &pp)
: Macro(0), ActualArgs(0), PP(pp), MacroTokens(&TokStream), CurToken(0),
InstantiateLoc(SourceLocation()), AtStartOfLine(false),
HasLeadingSpace(false) {
// Set HasLeadingSpace/AtStartOfLine so that the first token will be
// returned unmodified.
if (!TokStream.empty()) {
AtStartOfLine = TokStream[0].isAtStartOfLine();
HasLeadingSpace = TokStream[0].hasLeadingSpace();
}
}
MacroExpander::~MacroExpander() {
// If this was a function-like macro that actually uses its arguments, delete
// the expanded tokens.
if (MacroTokens != &Macro.getReplacementTokens())
if (Macro && MacroTokens != &Macro->getReplacementTokens())
delete MacroTokens;
// MacroExpander owns its formal arguments.
@ -205,7 +263,7 @@ void MacroExpander::ExpandFunctionArguments() {
// when the #define was parsed.
const LexerToken &CurTok = (*MacroTokens)[i];
if (CurTok.getKind() == tok::hash || CurTok.getKind() == tok::hashat) {
int ArgNo = Macro.getArgumentNum((*MacroTokens)[i+1].getIdentifierInfo());
int ArgNo =Macro->getArgumentNum((*MacroTokens)[i+1].getIdentifierInfo());
assert(ArgNo != -1 && "Token following # is not an argument?");
if (CurTok.getKind() == tok::hash) // Stringify
@ -227,7 +285,7 @@ void MacroExpander::ExpandFunctionArguments() {
// Otherwise, if this is not an argument token, just add the token to the
// output buffer.
IdentifierInfo *II = CurTok.getIdentifierInfo();
int ArgNo = II ? Macro.getArgumentNum(II) : -1;
int ArgNo = II ? Macro->getArgumentNum(II) : -1;
if (ArgNo == -1) {
ResultToks.push_back(CurTok);
continue;
@ -251,14 +309,10 @@ void MacroExpander::ExpandFunctionArguments() {
const std::vector<LexerToken> *ArgToks;
// Only preexpand the argument if it could possibly need it. This
// avoids some work in common cases.
if (ActualArgs->ArgNeedsPreexpansion(ArgNo)) {
// FIXME: WRONG
if (ActualArgs->ArgNeedsPreexpansion(ArgNo))
ArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP);
else
ArgToks = &ActualArgs->getUnexpArgument(ArgNo);
} else {
// If we don't need to pre-expand the argument, just substitute in the
// unexpanded tokens.
ArgToks = &ActualArgs->getUnexpArgument(ArgNo);
}
unsigned FirstTok = ResultToks.size();
ResultToks.insert(ResultToks.end(), ArgToks->begin(), ArgToks->end()-1);
@ -272,7 +326,12 @@ void MacroExpander::ExpandFunctionArguments() {
continue;
}
// FIXME: handle pasted args.
// Okay, we have a token that is either the LHS or RHS of a paste (##)
// argument.
// FIXME: Handle comma swallowing GNU extension.
// FIXME: handle pasted args. Handle 'placemarker' stuff.
ResultToks.push_back(CurTok);
}
}
@ -290,8 +349,15 @@ void MacroExpander::ExpandFunctionArguments() {
///
void MacroExpander::Lex(LexerToken &Tok) {
// Lexing off the end of the macro, pop this macro off the expansion stack.
if (isAtEnd())
if (isAtEnd()) {
// If this is a macro (not a token stream), mark the macro enabled now
// that it is no longer being expanded.
if (Macro) Macro->EnableMacro();
// Pop this context off the preprocessors lexer stack and get the next
// token.
return PP.HandleEndOfMacro(Tok);
}
// Get the next token to return.
Tok = (*MacroTokens)[CurToken++];
@ -301,9 +367,15 @@ void MacroExpander::Lex(LexerToken &Tok) {
// diagnostics for the expanded token should appear as if they came from
// InstantiationLoc. Pull this information together into a new SourceLocation
// that captures all of this.
Tok.SetLocation(PP.getSourceManager().getInstantiationLoc(Tok.getLocation(),
InstantiateLoc));
if (InstantiateLoc.isValid()) { // Don't do this for token streams.
SourceManager &SrcMgr = PP.getSourceManager();
// The token could have come from a prior macro expansion. In that case,
// ignore the macro expand part to get to the physloc. This happens for
// stuff like: #define A(X) X A(A(X)) A(1)
SourceLocation PhysLoc = SrcMgr.getPhysicalLoc(Tok.getLocation());
Tok.SetLocation(SrcMgr.getInstantiationLoc(PhysLoc, InstantiateLoc));
}
// If this is the first token, set the lexical properties of the token to
// match the lexical properties of the macro identifier.
if (CurToken == 1) {

42
clang/Lex/Preprocessor.cpp
View File

@ -450,12 +450,38 @@ void Preprocessor::EnterMacro(LexerToken &Tok, MacroArgs *Args) {
CurLexer = 0;
CurDirLookup = 0;
// Mark the macro as currently disabled, so that it is not recursively
// expanded.
MI.DisableMacro();
CurMacroExpander = new MacroExpander(Tok, Args, *this);
}
/// EnterTokenStream - Add a "macro" context to the top of the include stack,
/// which will cause the lexer to start returning the specified tokens. Note
/// that these tokens will be re-macro-expanded when/if expansion is enabled.
/// This method assumes that the specified stream of tokens has a permanent
/// owner somewhere, so they do not need to be copied.
void Preprocessor::EnterTokenStream(const std::vector<LexerToken> &Stream) {
// Save our current state.
IncludeMacroStack.push_back(IncludeStackInfo(CurLexer, CurDirLookup,
CurMacroExpander));
CurLexer = 0;
CurDirLookup = 0;
// Create a macro expander to expand from the specified token stream.
CurMacroExpander = new MacroExpander(Stream, *this);
}
/// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
/// lexer stack. This should only be used in situations where the current
/// state of the top-of-stack lexer is known.
void Preprocessor::RemoveTopOfLexerStack() {
assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load");
delete CurLexer;
delete CurMacroExpander;
CurLexer = IncludeMacroStack.back().TheLexer;
CurDirLookup = IncludeMacroStack.back().TheDirLookup;
CurMacroExpander = IncludeMacroStack.back().TheMacroExpander;
IncludeMacroStack.pop_back();
}
//===----------------------------------------------------------------------===//
// Macro Expansion Handling.
//===----------------------------------------------------------------------===//
@ -1011,11 +1037,7 @@ void Preprocessor::HandleEndOfFile(LexerToken &Result, bool isEndOfMacro) {
// lexing the #includer file.
if (!IncludeMacroStack.empty()) {
// We're done with the #included file.
delete CurLexer;
CurLexer = IncludeMacroStack.back().TheLexer;
CurDirLookup = IncludeMacroStack.back().TheDirLookup;
CurMacroExpander = IncludeMacroStack.back().TheMacroExpander;
IncludeMacroStack.pop_back();
RemoveTopOfLexerStack();
// Notify the client, if desired, that we are in a new source file.
if (FileChangeHandler && !isEndOfMacro && CurLexer) {
@ -1050,13 +1072,11 @@ void Preprocessor::HandleEndOfFile(LexerToken &Result, bool isEndOfMacro) {
}
/// HandleEndOfMacro - This callback is invoked when the lexer hits the end of
/// the current macro expansion.
/// the current macro expansion or token stream expansion.
void Preprocessor::HandleEndOfMacro(LexerToken &Result) {
assert(CurMacroExpander && !CurLexer &&
"Ending a macro when currently in a #include file!");
// Mark macro not ignored now that it is no longer being expanded.
CurMacroExpander->getMacro().EnableMacro();
delete CurMacroExpander;
// Handle this like a #include file being popped off the stack.

13
clang/include/clang/Basic/SourceManager.h
View File

@ -262,6 +262,19 @@ public:
return Loc;
}
/// getPhysicalLoc - Given a SourceLocation object, return the physical
/// location referenced by the ID.
SourceLocation getPhysicalLoc(SourceLocation Loc) const {
if (Loc.getFileID() == 0) return Loc;
// For Macros, the physical loc is specified by the MacroTokenFileID.
const SrcMgr::FileIDInfo *FIDInfo = getFIDInfo(Loc.getFileID());
if (FIDInfo->IDType == SrcMgr::FileIDInfo::MacroExpansion)
return SourceLocation(FIDInfo->u.MacroTokenFileID,
Loc.getRawFilePos());
return Loc;
}
/// PrintStats - Print statistics to stderr.
///
void PrintStats() const;

30
clang/include/clang/Lex/MacroExpander.h
View File

@ -30,9 +30,10 @@ class MacroArgs {
/// an 'EOF' marker at the end of each argument.
std::vector<std::vector<LexerToken> > UnexpArgTokens;
/// ExpArgTokens - Pre-expanded tokens for arguments that need them. Empty if
/// not yet computed. This includes the EOF marker at the end of the stream.
std::vector<std::vector<LexerToken> > ExpArgTokens;
/// PreExpArgTokens - Pre-expanded tokens for arguments that need them. Empty
/// if not yet computed. This includes the EOF marker at the end of the
/// stream.
std::vector<std::vector<LexerToken> > PreExpArgTokens;
/// StringifiedArgs - This contains arguments in 'stringified' form. If the
/// stringified form of an argument has not yet been computed, this is empty.
@ -57,6 +58,11 @@ public:
return UnexpArgTokens[Arg];
}
/// getPreExpArgument - Return the pre-expanded form of the specified
/// argument.
const std::vector<LexerToken> &
getPreExpArgument(unsigned Arg, Preprocessor &PP);
/// getStringifiedArgument - Compute, cache, and return the specified argument
/// that has been 'stringified' as required by the # operator.
const LexerToken &getStringifiedArgument(unsigned ArgNo, Preprocessor &PP);
@ -68,12 +74,13 @@ public:
/// MacroExpander - This implements a lexer that returns token from a macro body
/// instead of lexing from a character buffer.
/// or token stream instead of lexing from a character buffer.
///
class MacroExpander {
/// Macro - The macro we are expanding from.
/// Macro - The macro we are expanding from. This is null if expanding a
/// token stream.
///
MacroInfo &Macro;
MacroInfo *Macro;
/// ActualArgs - The actual arguments specified for a function-like macro, or
/// null. The MacroExpander owns the pointed-to object.
@ -84,7 +91,8 @@ class MacroExpander {
Preprocessor &PP;
/// MacroTokens - This is the pointer to the list of tokens that the macro is
/// defined to, with arguments expanded for function-like macros.
/// defined to, with arguments expanded for function-like macros. If this is
/// a token stream, this are the tokens we are returning.
const std::vector<LexerToken> *MacroTokens;
/// CurToken - This is the next token that Lex will return.
@ -102,9 +110,13 @@ class MacroExpander {
MacroExpander(const MacroExpander&); // DO NOT IMPLEMENT
void operator=(const MacroExpander&); // DO NOT IMPLEMENT
public:
/// Create a macro expander of the specified macro with the specified actual
/// Create a macro expander for the specified macro with the specified actual
/// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
MacroExpander(LexerToken &Tok, MacroArgs *ActualArgs, Preprocessor &PP);
/// Create a macro expander for the specified token stream. This does not
/// take ownership of the specified token vector.
MacroExpander(const std::vector<LexerToken> &TokStream, Preprocessor &PP);
~MacroExpander();
/// isNextTokenLParen - If the next token lexed will pop this macro off the
@ -112,8 +124,6 @@ public:
/// 1, otherwise return 0.
unsigned isNextTokenLParen() const;
MacroInfo &getMacro() const { return Macro; }
/// Lex - Lex and return a token from this macro stream.
void Lex(LexerToken &Tok);

11
clang/include/clang/Lex/Preprocessor.h
View File

@ -325,6 +325,17 @@ public:
/// tokens input to a function-like macro.
void EnterMacro(LexerToken &Identifier, MacroArgs *Args);
/// EnterTokenStream - Add a "macro" context to the top of the include stack,
/// which will cause the lexer to start returning the specified tokens. Note
/// that these tokens will be re-macro-expanded when/if expansion is enabled.
/// This method assumes that the specified stream of tokens has a permanent
/// owner somewhere, so they do not need to be copied.
void EnterTokenStream(const std::vector<LexerToken> &Stream);
/// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
/// lexer stack. This should only be used in situations where the current
/// state of the top-of-stack lexer is known.
void RemoveTopOfLexerStack();
/// Lex - To lex a token from the preprocessor, just pull a token from the
/// current lexer or macro object.