dnrops
/
rust
mirror of https://github.com/rust-lang/rust.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Markers API

This commit is contained in:
Aleksey Kladov 2018-01-20 23:25:34 +03:00
parent be60d5aa66
commit 0b5d39f2a2
10 changed files with 189 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/parser/event_parser/grammar/attributes.rs
View File

@ -19,13 +19,13 @@ fn attribute(p: &mut Parser, kind: AttrKind) -> bool {
if kind == AttrKind::Inner && p.raw_lookahead(1) != EXCL {
return false;
}
p.start(ATTR);
let attr = p.start();
p.bump();
if kind == AttrKind::Inner {
p.bump();
}
p.expect(L_BRACK) && meta_item(p) && p.expect(R_BRACK);
p.finish();
attr.complete(p, ATTR);
true
} else {
false
@ -34,7 +34,7 @@ fn attribute(p: &mut Parser, kind: AttrKind) -> bool {
fn meta_item(p: &mut Parser) -> bool {
if p.at(IDENT) {
p.start(META_ITEM);
let meta_item = p.start();
p.bump();
if p.eat(EQ) {
if !expressions::literal(p) {
@ -46,7 +46,7 @@ fn meta_item(p: &mut Parser) -> bool {
comma_list(p, R_PAREN, meta_item_inner);
p.expect(R_PAREN);
}
p.finish();
meta_item.complete(p, META_ITEM);
true
} else {
false

4
src/parser/event_parser/grammar/expressions.rs
View File

@ -6,9 +6,9 @@ pub(super) fn literal(p: &mut Parser) -> bool {
INT_NUMBER | FLOAT_NUMBER |
BYTE | CHAR |
STRING | RAW_STRING | BYTE_STRING | RAW_BYTE_STRING => {
p.start(LITERAL);
let lit = p.start();
p.bump();
p.finish();
lit.complete(p, LITERAL);
true
}
_ => false

151
src/parser/event_parser/grammar/items.rs
View File

@ -8,19 +8,34 @@ pub(super) fn mod_contents(p: &mut Parser) {
}
fn item(p: &mut Parser) {
let attrs_start = p.mark();
let item = p.start();
attributes::outer_attributes(p);
visibility(p);
let la = p.raw_lookahead(1);
let item_start = p.mark();
match p.current() {
EXTERN_KW if la == CRATE_KW => extern_crate_item(p),
MOD_KW => mod_item(p),
USE_KW => use_item(p),
STRUCT_KW => struct_item(p),
FN_KW => fn_item(p),
let item_kind = match p.current() {
EXTERN_KW if la == CRATE_KW => {
extern_crate_item(p);
EXTERN_CRATE_ITEM
}
MOD_KW => {
mod_item(p);
MOD_ITEM
}
USE_KW => {
use_item(p);
USE_ITEM
}
STRUCT_KW => {
struct_item(p);
STRUCT_ITEM
}
FN_KW => {
fn_item(p);
FN_ITEM
}
err_token => {
p.start(ERROR);
item.abandon(p);
let err = p.start();
let message = if err_token == SEMI {
//TODO: if the item is incomplete, this message is misleading
"expected item, found `;`\n\
@ -32,60 +47,52 @@ fn item(p: &mut Parser) {
.message(message)
.emit();
p.bump();
p.finish();
err.complete(p, ERROR);
return;
}
};
p.forward_parent(attrs_start, item_start);
item.complete(p, item_kind);
}
fn struct_item(p: &mut Parser) {
p.start(STRUCT_ITEM);
assert!(p.at(STRUCT_KW));
p.bump();
struct_inner(p);
p.finish();
fn struct_inner(p: &mut Parser) {
if !p.expect(IDENT) {
p.finish();
return;
}
generic_parameters(p);
match p.current() {
WHERE_KW => {
where_clause(p);
match p.current() {
SEMI => {
p.bump();
return;
}
L_CURLY => named_fields(p),
_ => { //TODO: special case `(` error message
p.error()
.message("expected `;` or `{`")
.emit();
return;
}
if !p.expect(IDENT) {
return;
}
generic_parameters(p);
match p.current() {
WHERE_KW => {
where_clause(p);
match p.current() {
SEMI => {
p.bump();
return;
}
L_CURLY => named_fields(p),
_ => { //TODO: special case `(` error message
p.error()
.message("expected `;` or `{`")
.emit();
return;
}
}
SEMI => {
p.bump();
return;
}
L_CURLY => named_fields(p),
L_PAREN => {
tuple_fields(p);
p.expect(SEMI);
}
_ => {
p.error()
.message("expected `;`, `{`, or `(`")
.emit();
return;
}
}
SEMI => {
p.bump();
return;
}
L_CURLY => named_fields(p),
L_PAREN => {
pos_fields(p);
p.expect(SEMI);
}
_ => {
p.error()
.message("expected `;`, `{`, or `(`")
.emit();
return;
}
}
}
@ -97,30 +104,30 @@ fn named_fields(p: &mut Parser) {
}));
fn named_field(p: &mut Parser) {
p.start(NAMED_FIELD);
let field = p.start();
visibility(p);
if p.expect(IDENT) && p.expect(COLON) {
types::type_ref(p);
};
p.finish()
field.complete(p, NAMED_FIELD);
}
}
fn tuple_fields(p: &mut Parser) {
fn pos_fields(p: &mut Parser) {
if !p.expect(L_PAREN) {
return;
}
comma_list(p, R_PAREN, |p| {
tuple_field(p);
pos_field(p);
true
});
p.expect(R_PAREN);
fn tuple_field(p: &mut Parser) {
p.start(POS_FIELD);
fn pos_field(p: &mut Parser) {
let pos_field = p.start();
visibility(p);
types::type_ref(p);
p.finish();
pos_field.complete(p, POS_FIELD);
}
}
@ -129,28 +136,21 @@ fn generic_parameters(_: &mut Parser) {}
fn where_clause(_: &mut Parser) {}
fn extern_crate_item(p: &mut Parser) {
p.start(EXTERN_CRATE_ITEM);
assert!(p.at(EXTERN_KW));
p.bump();
assert!(p.at(CRATE_KW));
p.bump();
p.expect(IDENT) && alias(p) && p.expect(SEMI);
p.finish();
}
fn mod_item(p: &mut Parser) {
p.start(MOD_ITEM);
assert!(p.at(MOD_KW));
p.bump();
if p.expect(IDENT) && !p.eat(SEMI) {
p.curly_block(mod_contents);
}
p.finish()
}
pub(super) fn is_use_tree_start(kind: SyntaxKind) -> bool {
@ -158,28 +158,24 @@ pub(super) fn is_use_tree_start(kind: SyntaxKind) -> bool {
}
fn use_item(p: &mut Parser) {
p.start(USE_ITEM);
assert!(p.at(USE_KW));
p.bump();
use_tree(p);
p.expect(SEMI);
p.finish();
fn use_tree(p: &mut Parser) -> bool {
let la = p.raw_lookahead(1);
let m = p.start();
match (p.current(), la) {
(STAR, _) => {
p.start(USE_TREE);
p.bump();
}
(COLONCOLON, STAR) => {
p.start(USE_TREE);
p.bump();
p.bump();
}
(L_CURLY, _) | (COLONCOLON, L_CURLY) => {
p.start(USE_TREE);
if p.at(COLONCOLON) {
p.bump();
}
@ -188,7 +184,6 @@ fn use_item(p: &mut Parser) {
});
}
_ if paths::is_path_start(p) => {
p.start(USE_TREE);
paths::use_path(p);
match p.current() {
AS_KW => {
@ -216,23 +211,23 @@ fn use_item(p: &mut Parser) {
_ => (),
}
}
_ => return false,
_ => {
m.abandon(p);
return false
},
}
p.finish();
m.complete(p, USE_TREE);
return true;
}
}
fn fn_item(p: &mut Parser) {
p.start(FN_ITEM);
assert!(p.at(FN_KW));
p.bump();
p.expect(IDENT) && p.expect(L_PAREN) && p.expect(R_PAREN)
&& p.curly_block(|_| ());
p.finish();
}

12
src/parser/event_parser/grammar/mod.rs
View File

@ -10,15 +10,15 @@ mod types;
mod paths;
pub(crate) fn file(p: &mut Parser) {
p.start(FILE);
let file = p.start();
p.eat(SHEBANG);
items::mod_contents(p);
p.finish()
file.complete(p, FILE);
}
fn visibility(p: &mut Parser) {
if p.at(PUB_KW) {
p.start(VISIBILITY);
let vis = p.start();
p.bump();
if p.at(L_PAREN) {
match p.raw_lookahead(1) {
@ -32,16 +32,16 @@ fn visibility(p: &mut Parser) {
_ => ()
}
}
p.finish();
vis.complete(p, VISIBILITY);
}
}
fn alias(p: &mut Parser) -> bool {
if p.at(AS_KW) {
p.start(ALIAS);
let alias = p.start();
p.bump();
p.expect(IDENT);
p.finish();
alias.complete(p, ALIAS);
}
true //FIXME: return false if three are errors
}

17
src/parser/event_parser/grammar/paths.rs
View File

@ -8,19 +8,16 @@ pub(crate) fn use_path(p: &mut Parser) {
if !is_path_start(p) {
return;
}
let mut prev = p.mark();
p.start(PATH);
let path = p.start();
path_segment(p, true);
p.finish();
let mut qual = path.complete(p, PATH);
loop {
let curr = p.mark();
if p.at(COLONCOLON) && !items::is_use_tree_start(p.raw_lookahead(1)) {
p.start(PATH);
let path = qual.precede(p);
p.bump();
path_segment(p, false);
p.forward_parent(prev, curr);
prev = curr;
p.finish();
let path = path.complete(p, PATH);
qual = path;
} else {
break;
}
@ -28,7 +25,7 @@ pub(crate) fn use_path(p: &mut Parser) {
}
fn path_segment(p: &mut Parser, first: bool) {
p.start(PATH_SEGMENT);
let segment = p.start();
if first {
p.eat(COLONCOLON);
}
@ -42,5 +39,5 @@ fn path_segment(p: &mut Parser, first: bool) {
.emit();
}
};
p.finish();
segment.complete(p, PATH_SEGMENT);
}

2
src/parser/event_parser/mod.rs
View File

@ -23,4 +23,4 @@ pub(crate) fn parse<'t>(text: &'t str, raw_tokens: &'t [Token]) -> Vec<Event> {
let mut parser = parser::Parser::new(text, raw_tokens);
grammar::file(&mut parser);
parser.into_events()
}
}

112
src/parser/event_parser/parser.rs
View File

@ -1,8 +1,71 @@
use {Token, SyntaxKind, TextUnit};
use super::{Event};
use super::Event;
use super::super::is_insignificant;
use syntax_kinds::{L_CURLY, R_CURLY, ERROR};
use tree::EOF;
use tree::{EOF, TOMBSTONE};
pub(crate) struct Marker {
pos: u32
}
impl Marker {
pub fn complete(self, p: &mut Parser, kind: SyntaxKind) -> CompleteMarker {
match self.event(p) {
&mut Event::Start { kind: ref mut slot, ..} => {
*slot = kind;
}
_ => unreachable!(),
}
p.event(Event::Finish);
let result = CompleteMarker { pos: self.pos };
::std::mem::forget(self);
result
}
pub fn abandon(self, p: &mut Parser) {
let idx = self.pos as usize;
if idx == p.events.len() - 1 {
match p.events.pop() {
Some(Event::Start { kind: TOMBSTONE, forward_parent: None }) => (),
_ => unreachable!()
}
}
::std::mem::forget(self);
}
fn event<'p>(&self, p: &'p mut Parser) -> &'p mut Event {
&mut p.events[self.idx()]
}
fn idx(&self) -> usize {
self.pos as usize
}
}
impl Drop for Marker {
fn drop(&mut self) {
if !::std::thread::panicking() {
panic!("Each marker should be eithe completed or abandoned");
}
}
}
pub(crate) struct CompleteMarker {
pos: u32
}
impl CompleteMarker {
pub(crate) fn precede(self, p: &mut Parser) -> Marker {
let m = p.start();
match p.events[self.pos as usize] {
Event::Start { ref mut forward_parent, ..} => {
*forward_parent = Some(m.pos - self.pos);
}
_ => unreachable!(),
}
m
}
}
pub(crate) struct Parser<'t> {
@ -19,12 +82,9 @@ pub(crate) struct Parser<'t> {
curly_limit: Option<i32>,
}
#[derive(Debug, Clone, Copy,PartialEq, Eq)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct Pos(u32);
#[derive(Debug, Clone, Copy,PartialEq, Eq)]
pub(crate) struct Mark(u32);
impl<'t> Parser<'t> {
pub(crate) fn new(text: &'t str, raw_tokens: &'t [Token]) -> Parser<'t> {
let mut tokens = Vec::new();
@ -50,31 +110,13 @@ impl<'t> Parser<'t> {
}
}
pub(crate) fn mark(&self) -> Mark {
Mark(self.events.len() as u32)
}
pub(crate) fn forward_parent(&mut self, child: Mark, parent: Mark) {
if child == parent || parent == self.mark() {
return
}
assert!(child.0 < parent.0);
let diff = parent.0 - child.0;
match self.events[child.0 as usize] {
Event::Start { ref mut forward_parent, .. } => {
*forward_parent = Some(diff);
}
_ => unreachable!()
}
}
pub(crate) fn pos(&self) -> Pos {
Pos(self.pos as u32)
}
pub(crate) fn into_events(self) -> Vec<Event> {
assert!(self.curly_limit.is_none());
assert!(self.current() == EOF);
assert_eq!(self.current(), EOF);
self.events
}
@ -85,18 +127,16 @@ impl<'t> Parser<'t> {
let token = self.tokens[self.pos];
if let Some(limit) = self.curly_limit {
if limit == self.curly_level && token.kind == R_CURLY {
return EOF
return EOF;
}
}
token.kind
}
pub(crate) fn start(&mut self, kind: SyntaxKind) {
self.event(Event::Start { kind, forward_parent: None });
}
pub(crate) fn finish(&mut self) {
self.event(Event::Finish);
pub(crate) fn start(&mut self) -> Marker {
let m = Marker { pos: self.events.len() as u32 };
self.event(Event::Start { kind: TOMBSTONE, forward_parent: None });
m
}
pub(crate) fn error<'p>(&'p mut self) -> ErrorBuilder<'p, 't> {
@ -124,20 +164,20 @@ impl<'t> Parser<'t> {
let old_level = self.curly_level;
let old_limit = self.curly_limit;
if !self.expect(L_CURLY) {
return false
return false;
}
self.curly_limit = Some(self.curly_level);
f(self);
assert!(self.curly_level > old_level);
self.curly_limit = old_limit;
if !self.expect(R_CURLY) {
self.start(ERROR);
let err = self.start();
while self.curly_level > old_level {
if self.bump() == EOF {
break;
}
}
self.finish();
err.complete(self, ERROR);
}
true
}
@ -149,7 +189,7 @@ impl<'t> Parser<'t> {
pub(crate) struct ErrorBuilder<'p, 't: 'p> {
message: Option<String>,
parser: &'p mut Parser<'t>
parser: &'p mut Parser<'t>,
}
impl<'t, 'p> ErrorBuilder<'p, 't> {

5
src/parser/mod.rs
View File

@ -1,6 +1,7 @@
use {Token, File, FileBuilder, Sink, SyntaxKind};
use syntax_kinds::*;
use tree::TOMBSTONE;
mod event_parser;
use self::event_parser::Event;
@ -29,6 +30,8 @@ fn from_events_to_file(
}
match event {
&Event::Start { kind: TOMBSTONE, .. } => (),
&Event::Start { .. } => {
forward_parents.clear();
let mut idx = i;
@ -62,7 +65,7 @@ fn from_events_to_file(
}
builder.finish_internal()
},
&Event::Token { kind, mut n_raw_tokens } => loop {
&Event::Token { kind: _, mut n_raw_tokens } => loop {
let token = tokens[idx];
if !is_insignificant(token.kind) {
n_raw_tokens -= 1;

8
src/tree/file_builder.rs
View File

@ -48,7 +48,9 @@ impl Sink for FileBuilder {
}
fn finish_internal(&mut self) {
let (id, _) = self.in_progress.pop().unwrap();
let (id, _) = self.in_progress.pop().expect(
"trying to complete a node, but there are no in-progress nodes"
);
if !self.in_progress.is_empty() {
self.add_len(id);
}
@ -77,8 +79,8 @@ impl FileBuilder {
self.in_progress.iter().map(|&(idx, _)| self.nodes[idx].kind)
.collect::<Vec<_>>()
);
assert!(
self.pos == (self.text.len() as u32).into(),
assert_eq!(
self.pos, (self.text.len() as u32).into(),
"nodes in FileBuilder do not cover the whole file"
);
File {

15
src/tree/mod.rs
View File

@ -10,17 +10,24 @@ pub use self::file_builder::{FileBuilder, Sink};
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct SyntaxKind(pub(crate) u32);
pub(crate) const EOF: SyntaxKind = SyntaxKind(10000);
pub(crate) const EOF: SyntaxKind = SyntaxKind(!0);
pub(crate) const EOF_INFO: SyntaxInfo = SyntaxInfo {
name: "EOF"
};
pub(crate) const TOMBSTONE: SyntaxKind = SyntaxKind(!0 - 1);
pub(crate) const TOMBSTONE_INFO: SyntaxInfo = SyntaxInfo {
name: "TOMBSTONE"
};
impl SyntaxKind {
fn info(self) -> &'static SyntaxInfo {
if self == EOF {
return &EOF_INFO;
match self {
EOF => &EOF_INFO,
TOMBSTONE => &TOMBSTONE_INFO,
_ => syntax_info(self),
}
syntax_info(self)
}
}