Remove `NtIdent` and `NtLifetime`.

The extra span is now recorded in the new `TokenKind::NtIdent` and
`TokenKind::NtLifetime`. These both consist of a single token, and so
there's no operator precedence problems with inserting them directly
into the token stream.

The other way to do this would be to wrap the ident/lifetime in invisible
delimiters, but there's a lot of code that assumes an interpolated
ident/lifetime fits in a single token, and changing all that code to work with
invisible delimiters would have been a pain. (Maybe it could be done in a
follow-up.)

This change might not seem like much of a win, but it's a first step toward the
much bigger and long-desired removal of `Nonterminal` and
`TokenKind::Interpolated`. That change is big and complex enough that it's
worth doing this piece separately. (Indeed, this commit is based on part of a
late commit in #114647, a prior attempt at that big and complex change.)
This commit is contained in:
Nicholas Nethercote 2024-04-22 19:46:51 +10:00
parent 9a63a42cb7
commit 95e519ecbf
11 changed files with 131 additions and 104 deletions

View File

@ -240,7 +240,6 @@ impl HasTokens for Nonterminal {
Nonterminal::NtPath(path) => path.tokens(), Nonterminal::NtPath(path) => path.tokens(),
Nonterminal::NtVis(vis) => vis.tokens(), Nonterminal::NtVis(vis) => vis.tokens(),
Nonterminal::NtBlock(block) => block.tokens(), Nonterminal::NtBlock(block) => block.tokens(),
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
} }
} }
fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> { fn tokens_mut(&mut self) -> Option<&mut Option<LazyAttrTokenStream>> {
@ -254,7 +253,6 @@ impl HasTokens for Nonterminal {
Nonterminal::NtPath(path) => path.tokens_mut(), Nonterminal::NtPath(path) => path.tokens_mut(),
Nonterminal::NtVis(vis) => vis.tokens_mut(), Nonterminal::NtVis(vis) => vis.tokens_mut(),
Nonterminal::NtBlock(block) => block.tokens_mut(), Nonterminal::NtBlock(block) => block.tokens_mut(),
Nonterminal::NtIdent(..) | Nonterminal::NtLifetime(..) => None,
} }
} }
} }

View File

@ -781,6 +781,12 @@ pub fn visit_token<T: MutVisitor>(t: &mut Token, vis: &mut T) {
*span = ident.span; *span = ident.span;
return; // Avoid visiting the span for the second time. return; // Avoid visiting the span for the second time.
} }
token::NtIdent(ident, _is_raw) => {
vis.visit_ident(ident);
}
token::NtLifetime(ident) => {
vis.visit_ident(ident);
}
token::Interpolated(nt) => { token::Interpolated(nt) => {
let nt = Lrc::make_mut(nt); let nt = Lrc::make_mut(nt);
visit_nonterminal(nt, vis); visit_nonterminal(nt, vis);
@ -832,8 +838,6 @@ fn visit_nonterminal<T: MutVisitor>(nt: &mut token::Nonterminal, vis: &mut T) {
token::NtPat(pat) => vis.visit_pat(pat), token::NtPat(pat) => vis.visit_pat(pat),
token::NtExpr(expr) => vis.visit_expr(expr), token::NtExpr(expr) => vis.visit_expr(expr),
token::NtTy(ty) => vis.visit_ty(ty), token::NtTy(ty) => vis.visit_ty(ty),
token::NtIdent(ident, _is_raw) => vis.visit_ident(ident),
token::NtLifetime(ident) => vis.visit_ident(ident),
token::NtLiteral(expr) => vis.visit_expr(expr), token::NtLiteral(expr) => vis.visit_expr(expr),
token::NtMeta(item) => { token::NtMeta(item) => {
let AttrItem { path, args, tokens } = item.deref_mut(); let AttrItem { path, args, tokens } = item.deref_mut();

View File

@ -318,11 +318,20 @@ pub enum TokenKind {
/// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to /// It's recommended to use `Token::(ident,uninterpolate,uninterpolated_span)` to
/// treat regular and interpolated identifiers in the same way. /// treat regular and interpolated identifiers in the same way.
Ident(Symbol, IdentIsRaw), Ident(Symbol, IdentIsRaw),
/// This identifier (and its span) is the identifier passed to the
/// declarative macro. The span in the surrounding `Token` is the span of
/// the `ident` metavariable in the macro's RHS.
NtIdent(Ident, IdentIsRaw),
/// Lifetime identifier token. /// Lifetime identifier token.
/// Do not forget about `NtLifetime` when you want to match on lifetime identifiers. /// Do not forget about `NtLifetime` when you want to match on lifetime identifiers.
/// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to /// It's recommended to use `Token::(lifetime,uninterpolate,uninterpolated_span)` to
/// treat regular and interpolated lifetime identifiers in the same way. /// treat regular and interpolated lifetime identifiers in the same way.
Lifetime(Symbol), Lifetime(Symbol),
/// This identifier (and its span) is the lifetime passed to the
/// declarative macro. The span in the surrounding `Token` is the span of
/// the `lifetime` metavariable in the macro's RHS.
NtLifetime(Ident),
/// An embedded AST node, as produced by a macro. This only exists for /// An embedded AST node, as produced by a macro. This only exists for
/// historical reasons. We'd like to get rid of it, for multiple reasons. /// historical reasons. We'd like to get rid of it, for multiple reasons.
@ -444,8 +453,9 @@ impl Token {
/// Note that keywords are also identifiers, so they should use this /// Note that keywords are also identifiers, so they should use this
/// if they keep spans or perform edition checks. /// if they keep spans or perform edition checks.
pub fn uninterpolated_span(&self) -> Span { pub fn uninterpolated_span(&self) -> Span {
match &self.kind { match self.kind {
Interpolated(nt) => nt.use_span(), NtIdent(ident, _) | NtLifetime(ident) => ident.span,
Interpolated(ref nt) => nt.use_span(),
_ => self.span, _ => self.span,
} }
} }
@ -463,7 +473,7 @@ impl Token {
} }
OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..) OpenDelim(..) | CloseDelim(..) | Literal(..) | DocComment(..) | Ident(..)
| Lifetime(..) | Interpolated(..) | Eof => false, | NtIdent(..) | Lifetime(..) | NtLifetime(..) | Interpolated(..) | Eof => false,
} }
} }
@ -613,14 +623,9 @@ impl Token {
/// into the regular identifier or lifetime token it refers to, /// into the regular identifier or lifetime token it refers to,
/// otherwise returns the original token. /// otherwise returns the original token.
pub fn uninterpolate(&self) -> Cow<'_, Token> { pub fn uninterpolate(&self) -> Cow<'_, Token> {
match &self.kind { match self.kind {
Interpolated(nt) => match &**nt { NtIdent(ident, is_raw) => Cow::Owned(Token::new(Ident(ident.name, is_raw), ident.span)),
NtIdent(ident, is_raw) => { NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
Cow::Owned(Token::new(Ident(ident.name, *is_raw), ident.span))
}
NtLifetime(ident) => Cow::Owned(Token::new(Lifetime(ident.name), ident.span)),
_ => Cow::Borrowed(self),
},
_ => Cow::Borrowed(self), _ => Cow::Borrowed(self),
} }
} }
@ -629,12 +634,9 @@ impl Token {
#[inline] #[inline]
pub fn ident(&self) -> Option<(Ident, IdentIsRaw)> { pub fn ident(&self) -> Option<(Ident, IdentIsRaw)> {
// We avoid using `Token::uninterpolate` here because it's slow. // We avoid using `Token::uninterpolate` here because it's slow.
match &self.kind { match self.kind {
&Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)), Ident(name, is_raw) => Some((Ident::new(name, self.span), is_raw)),
Interpolated(nt) => match &**nt { NtIdent(ident, is_raw) => Some((ident, is_raw)),
NtIdent(ident, is_raw) => Some((*ident, *is_raw)),
_ => None,
},
_ => None, _ => None,
} }
} }
@ -643,12 +645,9 @@ impl Token {
#[inline] #[inline]
pub fn lifetime(&self) -> Option<Ident> { pub fn lifetime(&self) -> Option<Ident> {
// We avoid using `Token::uninterpolate` here because it's slow. // We avoid using `Token::uninterpolate` here because it's slow.
match &self.kind { match self.kind {
&Lifetime(name) => Some(Ident::new(name, self.span)), Lifetime(name) => Some(Ident::new(name, self.span)),
Interpolated(nt) => match &**nt { NtLifetime(ident) => Some(ident),
NtLifetime(ident) => Some(*ident),
_ => None,
},
_ => None, _ => None,
} }
} }
@ -837,8 +836,10 @@ impl Token {
Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot Le | EqEq | Ne | Ge | AndAnd | OrOr | Tilde | BinOpEq(..) | At | DotDotDot
| DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar | DotDotEq | Comma | Semi | PathSep | RArrow | LArrow | FatArrow | Pound | Dollar
| Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | Question | OpenDelim(..) | CloseDelim(..) | Literal(..) | Ident(..) | NtIdent(..)
| Lifetime(..) | Interpolated(..) | DocComment(..) | Eof => return None, | Lifetime(..) | NtLifetime(..) | Interpolated(..) | DocComment(..) | Eof => {
return None;
}
}; };
Some(Token::new(kind, self.span.to(joint.span))) Some(Token::new(kind, self.span.to(joint.span)))
@ -861,9 +862,6 @@ pub enum Nonterminal {
NtPat(P<ast::Pat>), NtPat(P<ast::Pat>),
NtExpr(P<ast::Expr>), NtExpr(P<ast::Expr>),
NtTy(P<ast::Ty>), NtTy(P<ast::Ty>),
/// The span is for the identifier argument passed to the macro.
NtIdent(Ident, IdentIsRaw),
NtLifetime(Ident),
NtLiteral(P<ast::Expr>), NtLiteral(P<ast::Expr>),
/// Stuff inside brackets for attributes /// Stuff inside brackets for attributes
NtMeta(P<ast::AttrItem>), NtMeta(P<ast::AttrItem>),
@ -958,7 +956,6 @@ impl Nonterminal {
NtPat(pat) => pat.span, NtPat(pat) => pat.span,
NtExpr(expr) | NtLiteral(expr) => expr.span, NtExpr(expr) | NtLiteral(expr) => expr.span,
NtTy(ty) => ty.span, NtTy(ty) => ty.span,
NtIdent(ident, _) | NtLifetime(ident) => ident.span,
NtMeta(attr_item) => attr_item.span(), NtMeta(attr_item) => attr_item.span(),
NtPath(path) => path.span, NtPath(path) => path.span,
NtVis(vis) => vis.span, NtVis(vis) => vis.span,
@ -974,8 +971,6 @@ impl Nonterminal {
NtExpr(..) => "expression", NtExpr(..) => "expression",
NtLiteral(..) => "literal", NtLiteral(..) => "literal",
NtTy(..) => "type", NtTy(..) => "type",
NtIdent(..) => "identifier",
NtLifetime(..) => "lifetime",
NtMeta(..) => "attribute", NtMeta(..) => "attribute",
NtPath(..) => "path", NtPath(..) => "path",
NtVis(..) => "visibility", NtVis(..) => "visibility",
@ -984,18 +979,12 @@ impl Nonterminal {
} }
impl PartialEq for Nonterminal { impl PartialEq for Nonterminal {
fn eq(&self, rhs: &Self) -> bool { fn eq(&self, _rhs: &Self) -> bool {
match (self, rhs) { // FIXME: Assume that all nonterminals are not equal, we can't compare them
(NtIdent(ident_lhs, is_raw_lhs), NtIdent(ident_rhs, is_raw_rhs)) => { // correctly based on data from AST. This will prevent them from matching each other
ident_lhs == ident_rhs && is_raw_lhs == is_raw_rhs // in macros. The comparison will become possible only when each nonterminal has an
} // attached token stream from which it was parsed.
(NtLifetime(ident_lhs), NtLifetime(ident_rhs)) => ident_lhs == ident_rhs, false
// FIXME: Assume that all "complex" nonterminal are not equal, we can't compare them
// correctly based on data from AST. This will prevent them from matching each other
// in macros. The comparison will become possible only when each nonterminal has an
// attached token stream from which it was parsed.
_ => false,
}
} }
} }
@ -1008,12 +997,10 @@ impl fmt::Debug for Nonterminal {
NtPat(..) => f.pad("NtPat(..)"), NtPat(..) => f.pad("NtPat(..)"),
NtExpr(..) => f.pad("NtExpr(..)"), NtExpr(..) => f.pad("NtExpr(..)"),
NtTy(..) => f.pad("NtTy(..)"), NtTy(..) => f.pad("NtTy(..)"),
NtIdent(..) => f.pad("NtIdent(..)"),
NtLiteral(..) => f.pad("NtLiteral(..)"), NtLiteral(..) => f.pad("NtLiteral(..)"),
NtMeta(..) => f.pad("NtMeta(..)"), NtMeta(..) => f.pad("NtMeta(..)"),
NtPath(..) => f.pad("NtPath(..)"), NtPath(..) => f.pad("NtPath(..)"),
NtVis(..) => f.pad("NtVis(..)"), NtVis(..) => f.pad("NtVis(..)"),
NtLifetime(..) => f.pad("NtLifetime(..)"),
} }
} }
} }

View File

@ -466,12 +466,6 @@ impl TokenStream {
pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream { pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
match nt { match nt {
Nonterminal::NtIdent(ident, is_raw) => {
TokenStream::token_alone(token::Ident(ident.name, *is_raw), ident.span)
}
Nonterminal::NtLifetime(ident) => {
TokenStream::token_alone(token::Lifetime(ident.name), ident.span)
}
Nonterminal::NtItem(item) => TokenStream::from_ast(item), Nonterminal::NtItem(item) => TokenStream::from_ast(item),
Nonterminal::NtBlock(block) => TokenStream::from_ast(block), Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => { Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
@ -489,11 +483,17 @@ impl TokenStream {
} }
fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree { fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
match &token.kind { match token.kind {
token::Interpolated(nt) if let token::NtIdent(ident, is_raw) = &**nt => { token::NtIdent(ident, is_raw) => {
TokenTree::Token(Token::new(token::Ident(ident.name, *is_raw), ident.span), spacing) TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
} }
token::Interpolated(nt) => TokenTree::Delimited( token::NtLifetime(ident) => TokenTree::Delimited(
DelimSpan::from_single(token.span),
DelimSpacing::new(Spacing::JointHidden, spacing),
Delimiter::Invisible,
TokenStream::token_alone(token::Lifetime(ident.name), ident.span),
),
token::Interpolated(ref nt) => TokenTree::Delimited(
DelimSpan::from_single(token.span), DelimSpan::from_single(token.span),
DelimSpacing::new(Spacing::JointHidden, spacing), DelimSpacing::new(Spacing::JointHidden, spacing),
Delimiter::Invisible, Delimiter::Invisible,
@ -516,7 +516,10 @@ impl TokenStream {
pub fn flattened(&self) -> TokenStream { pub fn flattened(&self) -> TokenStream {
fn can_skip(stream: &TokenStream) -> bool { fn can_skip(stream: &TokenStream) -> bool {
stream.trees().all(|tree| match tree { stream.trees().all(|tree| match tree {
TokenTree::Token(token, _) => !matches!(token.kind, token::Interpolated(_)), TokenTree::Token(token, _) => !matches!(
token.kind,
token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
),
TokenTree::Delimited(.., inner) => can_skip(inner), TokenTree::Delimited(.., inner) => can_skip(inner),
}) })
} }

View File

@ -852,8 +852,6 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
token::NtBlock(e) => self.block_to_string(e), token::NtBlock(e) => self.block_to_string(e),
token::NtStmt(e) => self.stmt_to_string(e), token::NtStmt(e) => self.stmt_to_string(e),
token::NtPat(e) => self.pat_to_string(e), token::NtPat(e) => self.pat_to_string(e),
&token::NtIdent(e, is_raw) => IdentPrinter::for_ast_ident(e, is_raw.into()).to_string(),
token::NtLifetime(e) => e.to_string(),
token::NtLiteral(e) => self.expr_to_string(e), token::NtLiteral(e) => self.expr_to_string(e),
token::NtVis(e) => self.vis_to_string(e), token::NtVis(e) => self.vis_to_string(e),
} }
@ -915,10 +913,14 @@ pub trait PrintState<'a>: std::ops::Deref<Target = pp::Printer> + std::ops::Dere
token::Literal(lit) => literal_to_string(lit).into(), token::Literal(lit) => literal_to_string(lit).into(),
/* Name components */ /* Name components */
token::Ident(s, is_raw) => { token::Ident(name, is_raw) => {
IdentPrinter::new(s, is_raw.into(), convert_dollar_crate).to_string().into() IdentPrinter::new(name, is_raw.into(), convert_dollar_crate).to_string().into()
} }
token::Lifetime(s) => s.to_string().into(), token::NtIdent(ident, is_raw) => {
IdentPrinter::for_ast_ident(ident, is_raw.into()).to_string().into()
}
token::Lifetime(name) => name.to_string().into(),
token::NtLifetime(ident) => ident.name.to_string().into(),
/* Other */ /* Other */
token::DocComment(comment_kind, attr_style, data) => { token::DocComment(comment_kind, attr_style, data) => {

View File

@ -201,10 +201,17 @@ impl<'a> StripUnconfigured<'a> {
inner = self.configure_tokens(&inner); inner = self.configure_tokens(&inner);
Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)).into_iter() Some(AttrTokenTree::Delimited(sp, spacing, delim, inner)).into_iter()
} }
AttrTokenTree::Token(ref token, _) AttrTokenTree::Token(
if let TokenKind::Interpolated(nt) = &token.kind => Token {
{ kind:
panic!("Nonterminal should have been flattened at {:?}: {:?}", token.span, nt); TokenKind::NtIdent(..)
| TokenKind::NtLifetime(..)
| TokenKind::Interpolated(..),
..
},
_,
) => {
panic!("Nonterminal should have been flattened: {:?}", tree);
} }
AttrTokenTree::Token(token, spacing) => { AttrTokenTree::Token(token, spacing) => {
Some(AttrTokenTree::Token(token, spacing)).into_iter() Some(AttrTokenTree::Token(token, spacing)).into_iter()

View File

@ -261,6 +261,16 @@ pub(super) fn transcribe<'a>(
// without wrapping them into groups. // without wrapping them into groups.
maybe_use_metavar_location(cx, &stack, sp, tt, &mut marker) maybe_use_metavar_location(cx, &stack, sp, tt, &mut marker)
} }
MatchedSingle(ParseNtResult::Ident(ident, is_raw)) => {
marker.visit_span(&mut sp);
let kind = token::NtIdent(*ident, *is_raw);
TokenTree::token_alone(kind, sp)
}
MatchedSingle(ParseNtResult::Lifetime(ident)) => {
marker.visit_span(&mut sp);
let kind = token::NtLifetime(*ident);
TokenTree::token_alone(kind, sp)
}
MatchedSingle(ParseNtResult::Nt(nt)) => { MatchedSingle(ParseNtResult::Nt(nt)) => {
// Other variables are emitted into the output stream as groups with // Other variables are emitted into the output stream as groups with
// `Delimiter::Invisible` to maintain parsing priorities. // `Delimiter::Invisible` to maintain parsing priorities.

View File

@ -220,6 +220,12 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
Ident(sym, is_raw) => { Ident(sym, is_raw) => {
trees.push(TokenTree::Ident(Ident { sym, is_raw: is_raw.into(), span })) trees.push(TokenTree::Ident(Ident { sym, is_raw: is_raw.into(), span }))
} }
NtIdent(ident, is_raw) => trees.push(TokenTree::Ident(Ident {
sym: ident.name,
is_raw: is_raw.into(),
span: ident.span,
})),
Lifetime(name) => { Lifetime(name) => {
let ident = symbol::Ident::new(name, span).without_first_quote(); let ident = symbol::Ident::new(name, span).without_first_quote();
trees.extend([ trees.extend([
@ -227,6 +233,15 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
TokenTree::Ident(Ident { sym: ident.name, is_raw: false, span }), TokenTree::Ident(Ident { sym: ident.name, is_raw: false, span }),
]); ]);
} }
NtLifetime(ident) => {
let stream = TokenStream::token_alone(token::Lifetime(ident.name), ident.span);
trees.push(TokenTree::Group(Group {
delimiter: pm::Delimiter::None,
stream: Some(stream),
span: DelimSpan::from_single(span),
}))
}
Literal(token::Lit { kind, symbol, suffix }) => { Literal(token::Lit { kind, symbol, suffix }) => {
trees.push(TokenTree::Literal(self::Literal { trees.push(TokenTree::Literal(self::Literal {
kind: FromInternal::from_internal(kind), kind: FromInternal::from_internal(kind),
@ -259,14 +274,6 @@ impl FromInternal<(TokenStream, &mut Rustc<'_, '_>)> for Vec<TokenTree<TokenStre
})); }));
} }
Interpolated(ref nt) if let NtIdent(ident, is_raw) = &**nt => {
trees.push(TokenTree::Ident(Ident {
sym: ident.name,
is_raw: matches!(is_raw, IdentIsRaw::Yes),
span: ident.span,
}))
}
Interpolated(nt) => { Interpolated(nt) => {
let stream = TokenStream::from_nonterminal_ast(&nt); let stream = TokenStream::from_nonterminal_ast(&nt);
// A hack used to pass AST fragments to attribute and derive // A hack used to pass AST fragments to attribute and derive

View File

@ -724,7 +724,9 @@ impl<'a> Parser<'a> {
/// Returns the span of expr if it was not interpolated, or the span of the interpolated token. /// Returns the span of expr if it was not interpolated, or the span of the interpolated token.
fn interpolated_or_expr_span(&self, expr: &Expr) -> Span { fn interpolated_or_expr_span(&self, expr: &Expr) -> Span {
match self.prev_token.kind { match self.prev_token.kind {
TokenKind::Interpolated(..) => self.prev_token.span, TokenKind::NtIdent(..) | TokenKind::NtLifetime(..) | TokenKind::Interpolated(..) => {
self.prev_token.span
}
_ => expr.span, _ => expr.span,
} }
} }

View File

@ -407,6 +407,8 @@ pub(super) fn token_descr(token: &Token) -> String {
(Some(TokenDescription::Keyword), _) => Some("keyword"), (Some(TokenDescription::Keyword), _) => Some("keyword"),
(Some(TokenDescription::ReservedKeyword), _) => Some("reserved keyword"), (Some(TokenDescription::ReservedKeyword), _) => Some("reserved keyword"),
(Some(TokenDescription::DocComment), _) => Some("doc comment"), (Some(TokenDescription::DocComment), _) => Some("doc comment"),
(None, TokenKind::NtIdent(..)) => Some("identifier"),
(None, TokenKind::NtLifetime(..)) => Some("lifetime"),
(None, TokenKind::Interpolated(node)) => Some(node.descr()), (None, TokenKind::Interpolated(node)) => Some(node.descr()),
(None, _) => None, (None, _) => None,
}; };
@ -1633,5 +1635,9 @@ pub enum FlatToken {
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub enum ParseNtResult { pub enum ParseNtResult {
Tt(TokenTree), Tt(TokenTree),
Ident(Ident, IdentIsRaw),
Lifetime(Ident),
/// This case will eventually be removed, along with `Token::Interpolate`.
Nt(Lrc<Nonterminal>), Nt(Lrc<Nonterminal>),
} }

View File

@ -25,15 +25,13 @@ impl<'a> Parser<'a> {
| NtPat(_) | NtPat(_)
| NtExpr(_) | NtExpr(_)
| NtTy(_) | NtTy(_)
| NtIdent(..)
| NtLiteral(_) // `true`, `false` | NtLiteral(_) // `true`, `false`
| NtMeta(_) | NtMeta(_)
| NtPath(_) => true, | NtPath(_) => true,
NtItem(_) NtItem(_)
| NtBlock(_) | NtBlock(_)
| NtVis(_) | NtVis(_) => false,
| NtLifetime(_) => false,
} }
} }
@ -50,25 +48,30 @@ impl<'a> Parser<'a> {
NonterminalKind::Literal => token.can_begin_literal_maybe_minus(), NonterminalKind::Literal => token.can_begin_literal_maybe_minus(),
NonterminalKind::Vis => match token.kind { NonterminalKind::Vis => match token.kind {
// The follow-set of :vis + "priv" keyword + interpolated // The follow-set of :vis + "priv" keyword + interpolated
token::Comma | token::Ident(..) | token::Interpolated(_) => true, token::Comma
| token::Ident(..)
| token::NtIdent(..)
| token::NtLifetime(..)
| token::Interpolated(_) => true,
_ => token.can_begin_type(), _ => token.can_begin_type(),
}, },
NonterminalKind::Block => match &token.kind { NonterminalKind::Block => match &token.kind {
token::OpenDelim(Delimiter::Brace) => true, token::OpenDelim(Delimiter::Brace) => true,
token::NtLifetime(..) => true,
token::Interpolated(nt) => match &**nt { token::Interpolated(nt) => match &**nt {
NtBlock(_) | NtLifetime(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true, NtBlock(_) | NtStmt(_) | NtExpr(_) | NtLiteral(_) => true,
NtItem(_) | NtPat(_) | NtTy(_) | NtIdent(..) | NtMeta(_) | NtPath(_) NtItem(_) | NtPat(_) | NtTy(_) | NtMeta(_) | NtPath(_) | NtVis(_) => false,
| NtVis(_) => false,
}, },
_ => false, _ => false,
}, },
NonterminalKind::Path | NonterminalKind::Meta => match &token.kind { NonterminalKind::Path | NonterminalKind::Meta => match &token.kind {
token::PathSep | token::Ident(..) => true, token::PathSep | token::Ident(..) | token::NtIdent(..) => true,
token::Interpolated(nt) => may_be_ident(nt), token::Interpolated(nt) => may_be_ident(nt),
_ => false, _ => false,
}, },
NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => match &token.kind { NonterminalKind::PatParam { .. } | NonterminalKind::PatWithOr => match &token.kind {
token::Ident(..) | // box, ref, mut, and other identifiers (can stricten) // box, ref, mut, and other identifiers (can stricten)
token::Ident(..) | token::NtIdent(..) |
token::OpenDelim(Delimiter::Parenthesis) | // tuple pattern token::OpenDelim(Delimiter::Parenthesis) | // tuple pattern
token::OpenDelim(Delimiter::Bracket) | // slice pattern token::OpenDelim(Delimiter::Bracket) | // slice pattern
token::BinOp(token::And) | // reference token::BinOp(token::And) | // reference
@ -86,10 +89,7 @@ impl<'a> Parser<'a> {
_ => false, _ => false,
}, },
NonterminalKind::Lifetime => match &token.kind { NonterminalKind::Lifetime => match &token.kind {
token::Lifetime(_) => true, token::Lifetime(_) | token::NtLifetime(..) => true,
token::Interpolated(nt) => {
matches!(&**nt, NtLifetime(_))
}
_ => false, _ => false,
}, },
NonterminalKind::TT | NonterminalKind::Item | NonterminalKind::Stmt => { NonterminalKind::TT | NonterminalKind::Item | NonterminalKind::Stmt => {
@ -154,15 +154,16 @@ impl<'a> Parser<'a> {
} }
// this could be handled like a token, since it is one // this could be handled like a token, since it is one
NonterminalKind::Ident if let Some((ident, is_raw)) = get_macro_ident(&self.token) => {
self.bump();
NtIdent(ident, is_raw)
}
NonterminalKind::Ident => { NonterminalKind::Ident => {
return Err(self.dcx().create_err(UnexpectedNonterminal::Ident { return if let Some((ident, is_raw)) = get_macro_ident(&self.token) {
span: self.token.span, self.bump();
token: self.token.clone(), Ok(ParseNtResult::Ident(ident, is_raw))
})); } else {
Err(self.dcx().create_err(UnexpectedNonterminal::Ident {
span: self.token.span,
token: self.token.clone(),
}))
};
} }
NonterminalKind::Path => { NonterminalKind::Path => {
NtPath(P(self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?)) NtPath(P(self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?))
@ -173,14 +174,14 @@ impl<'a> Parser<'a> {
.collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?)) .collect_tokens_no_attrs(|this| this.parse_visibility(FollowedByType::Yes))?))
} }
NonterminalKind::Lifetime => { NonterminalKind::Lifetime => {
if self.check_lifetime() { return if self.check_lifetime() {
NtLifetime(self.expect_lifetime().ident) Ok(ParseNtResult::Lifetime(self.expect_lifetime().ident))
} else { } else {
return Err(self.dcx().create_err(UnexpectedNonterminal::Lifetime { Err(self.dcx().create_err(UnexpectedNonterminal::Lifetime {
span: self.token.span, span: self.token.span,
token: self.token.clone(), token: self.token.clone(),
})); }))
} };
} }
}; };