From 097c40cf6e1defc2fc49d521374254ee27f5f1fb Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Sun, 12 Aug 2018 20:15:59 +0300 Subject: [PATCH] syntax: Enforce attribute grammar in the parser --- src/libsyntax/attr/mod.rs | 4 +- src/libsyntax/config.rs | 2 +- src/libsyntax/feature_gate.rs | 38 ++++++++++--------- src/libsyntax/parse/attr.rs | 34 ++++++++++++++--- src/libsyntax/parse/parser.rs | 2 +- src/test/compile-fail-fulldeps/issue-48941.rs | 5 +-- .../proc-macro/proc-macro-attributes.rs | 2 +- .../proc-macro/proc-macro-gates.rs | 6 --- src/test/parse-fail/attr-bad-meta-2.rs | 12 ++++++ src/test/parse-fail/attr-bad-meta-3.rs | 12 ++++++ src/test/parse-fail/attr-bad-meta.rs | 5 ++- .../proc-macro/auxiliary/derive-b.rs | 2 +- .../run-pass-fulldeps/proc-macro/derive-b.rs | 2 +- .../proc-macro/attribute-spans-preserved.rs | 2 +- .../attribute-spans-preserved.stderr | 6 +-- .../attribute-spans-preserved.stdout | 2 +- src/test/ui/attr-eq-token-tree.rs | 16 ++++++++ src/test/ui/macros/macro-attribute.rs | 4 +- src/test/ui/macros/macro-attribute.stderr | 11 ++---- src/test/ui/unrestricted-attribute-tokens.rs | 16 ++++++++ 20 files changed, 128 insertions(+), 55 deletions(-) create mode 100644 src/test/parse-fail/attr-bad-meta-2.rs create mode 100644 src/test/parse-fail/attr-bad-meta-3.rs create mode 100644 src/test/ui/attr-eq-token-tree.rs create mode 100644 src/test/ui/unrestricted-attribute-tokens.rs diff --git a/src/libsyntax/attr/mod.rs b/src/libsyntax/attr/mod.rs index 879f555ba03..5857bd282f0 100644 --- a/src/libsyntax/attr/mod.rs +++ b/src/libsyntax/attr/mod.rs @@ -607,7 +607,7 @@ impl NestedMetaItemKind { } impl Lit { - fn tokens(&self) -> TokenStream { + crate fn tokens(&self) -> TokenStream { TokenTree::Token(self.span, self.node.token()).into() } } @@ -794,7 +794,7 @@ pub fn inject(mut krate: ast::Crate, parse_sess: &ParseSess, attrs: &[String]) - ); let start_span = parser.span; - let (path, tokens) = panictry!(parser.parse_path_and_tokens()); + let (path, tokens) = panictry!(parser.parse_meta_item_unrestricted()); let end_span = parser.span; if parser.token != token::Eof { parse_sess.span_diagnostic diff --git a/src/libsyntax/config.rs b/src/libsyntax/config.rs index 4fe78bf829a..b4e35a9d564 100644 --- a/src/libsyntax/config.rs +++ b/src/libsyntax/config.rs @@ -90,7 +90,7 @@ impl<'a> StripUnconfigured<'a> { let cfg = parser.parse_meta_item()?; parser.expect(&token::Comma)?; let lo = parser.span.lo(); - let (path, tokens) = parser.parse_path_and_tokens()?; + let (path, tokens) = parser.parse_meta_item_unrestricted()?; parser.expect(&token::CloseDelim(token::Paren))?; Ok((cfg, path, tokens, parser.prev_span.with_lo(lo))) }) { diff --git a/src/libsyntax/feature_gate.rs b/src/libsyntax/feature_gate.rs index 56e69b9df9e..f837bead6a0 100644 --- a/src/libsyntax/feature_gate.rs +++ b/src/libsyntax/feature_gate.rs @@ -1526,28 +1526,30 @@ impl<'a> Visitor<'a> for PostExpansionVisitor<'a> { } } - match attr.parse_meta(self.context.parse_sess) { - Ok(meta) => { - // allow attr_literals in #[repr(align(x))] and #[repr(packed(n))] - let mut allow_attr_literal = false; - if attr.path == "repr" { - if let Some(content) = meta.meta_item_list() { - allow_attr_literal = content.iter().any( - |c| c.check_name("align") || c.check_name("packed")); + if !self.context.features.unrestricted_attribute_tokens { + // Unfortunately, `parse_meta` cannot be called speculatively because it can report + // errors by itself, so we have to call it only if the feature is disabled. + match attr.parse_meta(self.context.parse_sess) { + Ok(meta) => { + // allow attr_literals in #[repr(align(x))] and #[repr(packed(n))] + let mut allow_attr_literal = false; + if attr.path == "repr" { + if let Some(content) = meta.meta_item_list() { + allow_attr_literal = content.iter().any( + |c| c.check_name("align") || c.check_name("packed")); + } + } + + if !allow_attr_literal && contains_novel_literal(&meta) { + gate_feature_post!(&self, attr_literals, attr.span, + "non-string literals in attributes, or string \ + literals in top-level positions, are experimental"); } } - - if !allow_attr_literal && contains_novel_literal(&meta) { - gate_feature_post!(&self, attr_literals, attr.span, - "non-string literals in attributes, or string \ - literals in top-level positions, are experimental"); + Err(mut err) => { + err.help("try enabling `#![feature(unrestricted_attribute_tokens)]`").emit() } } - Err(mut err) => { - err.cancel(); - gate_feature_post!(&self, unrestricted_attribute_tokens, attr.span, - "arbitrary tokens in non-macro attributes are unstable"); - } } } diff --git a/src/libsyntax/parse/attr.rs b/src/libsyntax/parse/attr.rs index 4d59f64bb6b..b0136c3e18b 100644 --- a/src/libsyntax/parse/attr.rs +++ b/src/libsyntax/parse/attr.rs @@ -12,9 +12,9 @@ use attr; use ast; use codemap::respan; use parse::{SeqSep, PResult}; -use parse::token::{self, Nonterminal}; +use parse::token::{self, Nonterminal, DelimToken}; use parse::parser::{Parser, TokenType, PathStyle}; -use tokenstream::TokenStream; +use tokenstream::{TokenStream, TokenTree}; #[derive(Debug)] enum InnerAttributeParsePolicy<'a> { @@ -116,7 +116,7 @@ impl<'a> Parser<'a> { }; self.expect(&token::OpenDelim(token::Bracket))?; - let (path, tokens) = self.parse_path_and_tokens()?; + let (path, tokens) = self.parse_meta_item_unrestricted()?; self.expect(&token::CloseDelim(token::Bracket))?; let hi = self.prev_span; @@ -138,7 +138,16 @@ impl<'a> Parser<'a> { }) } - crate fn parse_path_and_tokens(&mut self) -> PResult<'a, (ast::Path, TokenStream)> { + /// Parse an inner part of attribute - path and following tokens. + /// The tokens must be either a delimited token stream, or empty token stream, + /// or the "legacy" key-value form. + /// PATH `(` TOKEN_STREAM `)` + /// PATH `[` TOKEN_STREAM `]` + /// PATH `{` TOKEN_STREAM `}` + /// PATH + /// PATH `=` TOKEN_TREE + /// The delimiters or `=` are still put into the resulting token stream. + crate fn parse_meta_item_unrestricted(&mut self) -> PResult<'a, (ast::Path, TokenStream)> { let meta = match self.token { token::Interpolated(ref nt) => match nt.0 { Nonterminal::NtMeta(ref meta) => Some(meta.clone()), @@ -150,7 +159,22 @@ impl<'a> Parser<'a> { self.bump(); (meta.ident, meta.node.tokens(meta.span)) } else { - (self.parse_path(PathStyle::Mod)?, self.parse_tokens()) + let path = self.parse_path(PathStyle::Mod)?; + let tokens = if self.check(&token::OpenDelim(DelimToken::Paren)) || + self.check(&token::OpenDelim(DelimToken::Bracket)) || + self.check(&token::OpenDelim(DelimToken::Brace)) { + self.parse_token_tree().into() + } else if self.eat(&token::Eq) { + let eq = TokenTree::Token(self.prev_span, token::Eq); + let tree = match self.token { + token::CloseDelim(_) | token::Eof => self.unexpected()?, + _ => self.parse_token_tree(), + }; + TokenStream::concat(vec![eq.into(), tree.into()]) + } else { + TokenStream::empty() + }; + (path, tokens) }) } diff --git a/src/libsyntax/parse/parser.rs b/src/libsyntax/parse/parser.rs index 0e45cacaf38..b1e1cdee2ee 100644 --- a/src/libsyntax/parse/parser.rs +++ b/src/libsyntax/parse/parser.rs @@ -825,7 +825,7 @@ impl<'a> Parser<'a> { /// /// This method will automatically add `tok` to `expected_tokens` if `tok` is not /// encountered. - fn check(&mut self, tok: &token::Token) -> bool { + crate fn check(&mut self, tok: &token::Token) -> bool { let is_present = self.token == *tok; if !is_present { self.expected_tokens.push(TokenType::Token(tok.clone())); } is_present diff --git a/src/test/compile-fail-fulldeps/issue-48941.rs b/src/test/compile-fail-fulldeps/issue-48941.rs index 4be2874ed4f..baeb019df1c 100644 --- a/src/test/compile-fail-fulldeps/issue-48941.rs +++ b/src/test/compile-fail-fulldeps/issue-48941.rs @@ -17,10 +17,7 @@ #![feature(plugin)] #![plugin(macro_crate_test)] -#[noop_attribute"x"] //~ ERROR expected one of -fn night() { } - -#[noop_attribute("hi"), rank = 2] //~ ERROR unexpected token +#[noop_attribute("hi", rank = a)] //~ ERROR expected unsuffixed literal or identifier, found a fn knight() { } #[noop_attribute("/user", data= = " or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#[path =] //~ ERROR unexpected token: `]` +mod m {} diff --git a/src/test/parse-fail/attr-bad-meta-3.rs b/src/test/parse-fail/attr-bad-meta-3.rs new file mode 100644 index 00000000000..92e2a59d25d --- /dev/null +++ b/src/test/parse-fail/attr-bad-meta-3.rs @@ -0,0 +1,12 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +#[path() token] //~ ERROR expected `]`, found `token` +mod m {} diff --git a/src/test/parse-fail/attr-bad-meta.rs b/src/test/parse-fail/attr-bad-meta.rs index 41db88121cb..6f9d794dc2d 100644 --- a/src/test/parse-fail/attr-bad-meta.rs +++ b/src/test/parse-fail/attr-bad-meta.rs @@ -8,6 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -// asterisk is bogus -#[path*] //~ ERROR arbitrary tokens in non-macro attributes are unstable +#![feature(unrestricted_attribute_tokens)] + +#[path*] //~ ERROR expected one of `(`, `::`, `=`, `[`, `]`, or `{`, found `*` mod m {} diff --git a/src/test/run-pass-fulldeps/proc-macro/auxiliary/derive-b.rs b/src/test/run-pass-fulldeps/proc-macro/auxiliary/derive-b.rs index 7b521f2b913..e1aabad4142 100644 --- a/src/test/run-pass-fulldeps/proc-macro/auxiliary/derive-b.rs +++ b/src/test/run-pass-fulldeps/proc-macro/auxiliary/derive-b.rs @@ -19,7 +19,7 @@ use proc_macro::TokenStream; #[proc_macro_derive(B, attributes(B, C))] pub fn derive(input: TokenStream) -> TokenStream { let input = input.to_string(); - assert!(input.contains("#[B arbitrary tokens]")); + assert!(input.contains("#[B [ arbitrary tokens ]]")); assert!(input.contains("struct B {")); assert!(input.contains("#[C]")); "".parse().unwrap() diff --git a/src/test/run-pass-fulldeps/proc-macro/derive-b.rs b/src/test/run-pass-fulldeps/proc-macro/derive-b.rs index 918d2c17123..1de6496e29f 100644 --- a/src/test/run-pass-fulldeps/proc-macro/derive-b.rs +++ b/src/test/run-pass-fulldeps/proc-macro/derive-b.rs @@ -16,7 +16,7 @@ extern crate derive_b; #[derive(Debug, PartialEq, derive_b::B, Eq, Copy, Clone)] -#[cfg_attr(all(), B arbitrary tokens)] +#[cfg_attr(all(), B[arbitrary tokens])] struct B { #[C] a: u64 diff --git a/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.rs b/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.rs index e1401653ba3..18ca34b117d 100644 --- a/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.rs +++ b/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.rs @@ -17,6 +17,6 @@ extern crate attribute_spans_preserved as foo; use foo::foo; #[ foo ( let y: u32 = "z"; ) ] //~ ERROR: mismatched types -#[ bar let x: u32 = "y"; ] //~ ERROR: mismatched types +#[ bar { let x: u32 = "y"; } ] //~ ERROR: mismatched types fn main() { } diff --git a/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stderr b/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stderr index fe62bd23b87..a6cbf79209e 100644 --- a/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stderr +++ b/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stderr @@ -8,10 +8,10 @@ LL | #[ foo ( let y: u32 = "z"; ) ] //~ ERROR: mismatched types found type `&'static str` error[E0308]: mismatched types - --> $DIR/attribute-spans-preserved.rs:20:21 + --> $DIR/attribute-spans-preserved.rs:20:23 | -LL | #[ bar let x: u32 = "y"; ] //~ ERROR: mismatched types - | ^^^ expected u32, found reference +LL | #[ bar { let x: u32 = "y"; } ] //~ ERROR: mismatched types + | ^^^ expected u32, found reference | = note: expected type `u32` found type `&'static str` diff --git a/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stdout b/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stdout index 33dc064ef68..b1487fcd5ed 100644 --- a/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stdout +++ b/src/test/ui-fulldeps/proc-macro/attribute-spans-preserved.stdout @@ -1 +1 @@ -fn main ( ) { let y : u32 = "z" ; let x : u32 = "y" ; } +fn main ( ) { let y : u32 = "z" ; { let x : u32 = "y" ; } } diff --git a/src/test/ui/attr-eq-token-tree.rs b/src/test/ui/attr-eq-token-tree.rs new file mode 100644 index 00000000000..c759e62dba0 --- /dev/null +++ b/src/test/ui/attr-eq-token-tree.rs @@ -0,0 +1,16 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// compile-pass + +#![feature(custom_attribute, unrestricted_attribute_tokens)] + +#[my_attr = !] // OK under feature gate +fn main() {} diff --git a/src/test/ui/macros/macro-attribute.rs b/src/test/ui/macros/macro-attribute.rs index a77b1724876..111375b3693 100644 --- a/src/test/ui/macros/macro-attribute.rs +++ b/src/test/ui/macros/macro-attribute.rs @@ -8,5 +8,7 @@ // option. This file may not be copied, modified, or distributed // except according to those terms. -#[doc = $not_there] //~ ERROR arbitrary tokens in non-macro attributes are unstable +#![feature(unrestricted_attribute_tokens)] + +#[doc = $not_there] //~ ERROR expected `]`, found `not_there` fn main() { } diff --git a/src/test/ui/macros/macro-attribute.stderr b/src/test/ui/macros/macro-attribute.stderr index 48132dddf1a..c403872ecb3 100644 --- a/src/test/ui/macros/macro-attribute.stderr +++ b/src/test/ui/macros/macro-attribute.stderr @@ -1,11 +1,8 @@ -error[E0658]: arbitrary tokens in non-macro attributes are unstable (see issue #44690) - --> $DIR/macro-attribute.rs:11:1 +error: expected `]`, found `not_there` + --> $DIR/macro-attribute.rs:13:10 | -LL | #[doc = $not_there] //~ ERROR arbitrary tokens in non-macro attributes are unstable - | ^^^^^^^^^^^^^^^^^^^ - | - = help: add #![feature(unrestricted_attribute_tokens)] to the crate attributes to enable +LL | #[doc = $not_there] //~ ERROR expected `]`, found `not_there` + | ^^^^^^^^^ expected `]` error: aborting due to previous error -For more information about this error, try `rustc --explain E0658`. diff --git a/src/test/ui/unrestricted-attribute-tokens.rs b/src/test/ui/unrestricted-attribute-tokens.rs new file mode 100644 index 00000000000..2971b504369 --- /dev/null +++ b/src/test/ui/unrestricted-attribute-tokens.rs @@ -0,0 +1,16 @@ +// Copyright 2018 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +// compile-pass + +#![feature(custom_attribute, unrestricted_attribute_tokens)] + +#[my_attr(a b c d)] +fn main() {}