mirror of https://github.com/rust-lang/rust.git
kill old tree
This commit is contained in:
Aleksey Kladov
parent
c12450fb4e
commit
ad188d4c3d
45
src/lib.rs
45
src/lib.rs
|
|
@ -26,52 +26,21 @@ mod yellow;
|
|||
|
||||
pub mod syntax_kinds;
|
||||
pub use text_unit::{TextRange, TextUnit};
|
||||
pub use tree::{File, Node, SyntaxKind, Token};
|
||||
pub(crate) use tree::{ErrorMsg, FileBuilder, Sink, GreenBuilder};
|
||||
pub use tree::{SyntaxKind, Token};
|
||||
pub(crate) use tree::{Sink, GreenBuilder};
|
||||
pub use lexer::{next_token, tokenize};
|
||||
pub use yellow::SyntaxNode;
|
||||
pub(crate) use yellow::SError;
|
||||
pub use parser::{parse, parse_green};
|
||||
pub use parser::{parse_green};
|
||||
|
||||
/// Utilities for simple uses of the parser.
|
||||
pub mod utils {
|
||||
use std::fmt::Write;
|
||||
|
||||
use {File, Node, SyntaxNode};
|
||||
use {SyntaxNode};
|
||||
use std::collections::BTreeSet;
|
||||
use SError;
|
||||
|
||||
/// Parse a file and create a string representation of the resulting parse tree.
|
||||
pub fn dump_tree(file: &File) -> String {
|
||||
let mut result = String::new();
|
||||
go(file.root(), &mut result, 0);
|
||||
return result;
|
||||
|
||||
fn go(node: Node, buff: &mut String, level: usize) {
|
||||
buff.push_str(&String::from(" ").repeat(level));
|
||||
write!(buff, "{:?}\n", node).unwrap();
|
||||
let my_errors = node.errors().filter(|e| e.after_child().is_none());
|
||||
let parent_errors = node.parent()
|
||||
.into_iter()
|
||||
.flat_map(|n| n.errors())
|
||||
.filter(|e| e.after_child() == Some(node));
|
||||
|
||||
for err in my_errors {
|
||||
buff.push_str(&String::from(" ").repeat(level));
|
||||
write!(buff, "err: `{}`\n", err.message()).unwrap();
|
||||
}
|
||||
|
||||
for child in node.children() {
|
||||
go(child, buff, level + 1)
|
||||
}
|
||||
|
||||
for err in parent_errors {
|
||||
buff.push_str(&String::from(" ").repeat(level));
|
||||
write!(buff, "err: `{}`\n", err.message()).unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parse a file and create a string representation of the resulting parse tree.
|
||||
pub fn dump_tree_green(syntax: &SyntaxNode) -> String {
|
||||
let mut errors: BTreeSet<_> = syntax.root.errors.iter().cloned().collect();
|
||||
|
|
@ -82,12 +51,6 @@ pub mod utils {
|
|||
fn go(node: &SyntaxNode, buff: &mut String, level: usize, errors: &mut BTreeSet<SError>) {
|
||||
buff.push_str(&String::from(" ").repeat(level));
|
||||
write!(buff, "{:?}\n", node).unwrap();
|
||||
// let my_errors = node.errors().filter(|e| e.after_child().is_none());
|
||||
// let parent_errors = node.parent()
|
||||
// .into_iter()
|
||||
// .flat_map(|n| n.errors())
|
||||
// .filter(|e| e.after_child() == Some(node));
|
||||
//
|
||||
let my_errors: Vec<_> = errors.iter().filter(|e| e.offset == node.range().start())
|
||||
.cloned().collect();
|
||||
for err in my_errors {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use {
|
||||
ErrorMsg, File, FileBuilder, Sink, SyntaxKind, Token, GreenBuilder,
|
||||
Sink, SyntaxKind, Token,
|
||||
syntax_kinds::TOMBSTONE,
|
||||
};
|
||||
use super::is_insignificant;
|
||||
|
|
@ -67,12 +67,6 @@ pub(crate) enum Event {
|
|||
},
|
||||
}
|
||||
|
||||
pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> File {
|
||||
let mut builder = FileBuilder::new(text);
|
||||
process(&mut builder, tokens, events);
|
||||
builder.finish()
|
||||
}
|
||||
|
||||
pub(super) fn process(builder: &mut Sink, tokens: &[Token], events: Vec<Event>) {
|
||||
let mut idx = 0;
|
||||
|
||||
|
|
@ -147,7 +141,7 @@ pub(super) fn process(builder: &mut Sink, tokens: &[Token], events: Vec<Event>)
|
|||
}
|
||||
builder.leaf(kind, len);
|
||||
}
|
||||
&Event::Error { ref msg } => builder.error(ErrorMsg { msg: msg.clone() }),
|
||||
&Event::Error { ref msg } => builder.error(msg.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ mod grammar;
|
|||
|
||||
use std::sync::Arc;
|
||||
use {
|
||||
File, SyntaxKind, Token,
|
||||
Token,
|
||||
yellow::SyntaxNode,
|
||||
syntax_kinds::*
|
||||
};
|
||||
|
|
@ -15,18 +15,6 @@ use GreenBuilder;
|
|||
use parser::event::process;
|
||||
|
||||
|
||||
/// Parse a sequence of tokens into the representative node tree
|
||||
pub fn parse(text: String, tokens: &[Token]) -> File {
|
||||
let events = {
|
||||
let input = input::ParserInput::new(&text, tokens);
|
||||
let parser_impl = parser::imp::ParserImpl::new(&input);
|
||||
let mut parser = parser::Parser(parser_impl);
|
||||
grammar::file(&mut parser);
|
||||
parser.0.into_events()
|
||||
};
|
||||
event::to_file(text, tokens, events)
|
||||
}
|
||||
|
||||
/// Parse a sequence of tokens into the representative node tree
|
||||
pub fn parse_green(text: String, tokens: &[Token]) -> SyntaxNode {
|
||||
let events = {
|
||||
|
|
|
|||
|
|
@ -12,157 +12,13 @@ use {
|
|||
SyntaxKind, TextRange, TextUnit,
|
||||
yellow::GreenNode
|
||||
};
|
||||
use super::{File, NodeData, NodeIdx, SyntaxErrorData};
|
||||
use SError;
|
||||
|
||||
pub(crate) trait Sink {
|
||||
fn leaf(&mut self, kind: SyntaxKind, len: TextUnit);
|
||||
fn start_internal(&mut self, kind: SyntaxKind);
|
||||
fn finish_internal(&mut self);
|
||||
fn error(&mut self, err: ErrorMsg);
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct FileBuilder {
|
||||
text: String,
|
||||
nodes: Vec<NodeData>,
|
||||
errors: Vec<SyntaxErrorData>,
|
||||
in_progress: Vec<(NodeIdx, Option<NodeIdx>)>,
|
||||
// (parent, last_child)
|
||||
pos: TextUnit,
|
||||
}
|
||||
|
||||
impl Sink for FileBuilder {
|
||||
fn leaf(&mut self, kind: SyntaxKind, len: TextUnit) {
|
||||
let leaf = NodeData {
|
||||
kind,
|
||||
range: TextRange::offset_len(self.pos, len),
|
||||
parent: None,
|
||||
first_child: None,
|
||||
next_sibling: None,
|
||||
};
|
||||
self.pos += len;
|
||||
let id = self.push_child(leaf);
|
||||
self.add_len(id);
|
||||
}
|
||||
|
||||
fn start_internal(&mut self, kind: SyntaxKind) {
|
||||
let node = NodeData {
|
||||
kind,
|
||||
range: TextRange::offset_len(self.pos, 0.into()),
|
||||
parent: None,
|
||||
first_child: None,
|
||||
next_sibling: None,
|
||||
};
|
||||
let id = if self.in_progress.is_empty() {
|
||||
self.new_node(node)
|
||||
} else {
|
||||
self.push_child(node)
|
||||
};
|
||||
self.in_progress.push((id, None))
|
||||
}
|
||||
|
||||
fn finish_internal(&mut self) {
|
||||
let (id, _) = self.in_progress
|
||||
.pop()
|
||||
.expect("trying to complete a node, but there are no in-progress nodes");
|
||||
if !self.in_progress.is_empty() {
|
||||
self.add_len(id);
|
||||
}
|
||||
}
|
||||
|
||||
fn error(&mut self, err: ErrorMsg) {
|
||||
let &(node, after_child) = self.in_progress.last().unwrap();
|
||||
self.errors.push(SyntaxErrorData {
|
||||
node,
|
||||
message: err.msg,
|
||||
after_child,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl FileBuilder {
|
||||
pub fn new(text: String) -> FileBuilder {
|
||||
FileBuilder {
|
||||
text,
|
||||
nodes: Vec::new(),
|
||||
errors: Vec::new(),
|
||||
in_progress: Vec::new(),
|
||||
pos: 0.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn finish(self) -> File {
|
||||
assert!(
|
||||
self.in_progress.is_empty(),
|
||||
"some nodes in FileBuilder are unfinished: {:?}",
|
||||
self.in_progress
|
||||
.iter()
|
||||
.map(|&(idx, _)| self.nodes[idx].kind)
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
assert_eq!(
|
||||
self.pos,
|
||||
(self.text.len() as u32).into(),
|
||||
"nodes in FileBuilder do not cover the whole file"
|
||||
);
|
||||
File {
|
||||
text: self.text,
|
||||
nodes: self.nodes,
|
||||
errors: self.errors,
|
||||
}
|
||||
}
|
||||
|
||||
fn new_node(&mut self, data: NodeData) -> NodeIdx {
|
||||
let id = NodeIdx(self.nodes.len() as u32);
|
||||
self.nodes.push(data);
|
||||
id
|
||||
}
|
||||
|
||||
fn push_child(&mut self, mut child: NodeData) -> NodeIdx {
|
||||
child.parent = Some(self.current_id());
|
||||
let id = self.new_node(child);
|
||||
{
|
||||
let (parent, sibling) = *self.in_progress.last().unwrap();
|
||||
let slot = if let Some(idx) = sibling {
|
||||
&mut self.nodes[idx].next_sibling
|
||||
} else {
|
||||
&mut self.nodes[parent].first_child
|
||||
};
|
||||
fill(slot, id);
|
||||
}
|
||||
self.in_progress.last_mut().unwrap().1 = Some(id);
|
||||
id
|
||||
}
|
||||
|
||||
fn add_len(&mut self, child: NodeIdx) {
|
||||
let range = self.nodes[child].range;
|
||||
grow(&mut self.current_parent().range, range);
|
||||
}
|
||||
|
||||
fn current_id(&self) -> NodeIdx {
|
||||
self.in_progress.last().unwrap().0
|
||||
}
|
||||
|
||||
fn current_parent(&mut self) -> &mut NodeData {
|
||||
let idx = self.current_id();
|
||||
&mut self.nodes[idx]
|
||||
}
|
||||
}
|
||||
|
||||
fn fill<T>(slot: &mut Option<T>, value: T) {
|
||||
assert!(slot.is_none());
|
||||
*slot = Some(value);
|
||||
}
|
||||
|
||||
fn grow(left: &mut TextRange, right: TextRange) {
|
||||
assert_eq!(left.end(), right.start());
|
||||
*left = TextRange::from_to(left.start(), right.end())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct ErrorMsg {
|
||||
pub(crate) msg: String,
|
||||
fn error(&mut self, err: String);
|
||||
}
|
||||
|
||||
pub(crate) struct GreenBuilder {
|
||||
|
|
@ -216,8 +72,8 @@ impl Sink for GreenBuilder {
|
|||
}
|
||||
}
|
||||
|
||||
fn error(&mut self, err: ErrorMsg) {
|
||||
self.errors.push(SError { message: err.msg, offset: self.pos })
|
||||
fn error(&mut self, message: String) {
|
||||
self.errors.push(SError { message, offset: self.pos })
|
||||
}
|
||||
}
|
||||
impl SyntaxKind {
|
||||
|
|
|
|||
221
src/tree/mod.rs
221
src/tree/mod.rs
|
|
@ -1,8 +1,8 @@
|
|||
mod file_builder;
|
||||
|
||||
use ::{TextRange, TextUnit};
|
||||
use std::{fmt, cmp};
|
||||
pub(crate) use self::file_builder::{ErrorMsg, FileBuilder, Sink, GreenBuilder};
|
||||
use ::{TextUnit};
|
||||
use std::{fmt};
|
||||
pub(crate) use self::file_builder::{Sink, GreenBuilder};
|
||||
|
||||
pub use syntax_kinds::SyntaxKind;
|
||||
|
||||
|
|
@ -25,218 +25,3 @@ pub struct Token {
|
|||
/// The length of the token.
|
||||
pub len: TextUnit,
|
||||
}
|
||||
|
||||
/// The contents of a Rust source file.
|
||||
#[derive(Debug)]
|
||||
pub struct File {
|
||||
text: String,
|
||||
nodes: Vec<NodeData>,
|
||||
errors: Vec<SyntaxErrorData>,
|
||||
}
|
||||
|
||||
impl File {
|
||||
/// The root node of this source file.
|
||||
pub fn root<'f>(&'f self) -> Node<'f> {
|
||||
assert!(!self.nodes.is_empty());
|
||||
Node {
|
||||
file: self,
|
||||
idx: NodeIdx(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A reference to a token in a Rust source file.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct Node<'f> {
|
||||
file: &'f File,
|
||||
idx: NodeIdx,
|
||||
}
|
||||
|
||||
impl<'f> Node<'f> {
|
||||
/// The kind of the token at this node.
|
||||
pub fn kind(&self) -> SyntaxKind {
|
||||
self.data().kind
|
||||
}
|
||||
|
||||
/// The text range covered by the token at this node.
|
||||
pub fn range(&self) -> TextRange {
|
||||
self.data().range
|
||||
}
|
||||
|
||||
/// The text at this node.
|
||||
pub fn text(&self) -> &'f str {
|
||||
&self.file.text.as_str()[self.range()]
|
||||
}
|
||||
|
||||
/// The parent node to this node.
|
||||
pub fn parent(&self) -> Option<Node<'f>> {
|
||||
self.as_node(self.data().parent)
|
||||
}
|
||||
|
||||
/// The children nodes of this node.
|
||||
pub fn children(&self) -> Children<'f> {
|
||||
Children {
|
||||
next: self.as_node(self.data().first_child),
|
||||
}
|
||||
}
|
||||
|
||||
/// Any errors contained in this node.
|
||||
pub fn errors(&self) -> SyntaxErrors<'f> {
|
||||
let pos = self.file.errors.iter().position(|e| e.node == self.idx);
|
||||
let next = pos.map(|i| ErrorIdx(i as u32)).map(|idx| SyntaxError {
|
||||
file: self.file,
|
||||
idx,
|
||||
});
|
||||
SyntaxErrors { next }
|
||||
}
|
||||
|
||||
fn data(&self) -> &'f NodeData {
|
||||
&self.file.nodes[self.idx]
|
||||
}
|
||||
|
||||
fn as_node(&self, idx: Option<NodeIdx>) -> Option<Node<'f>> {
|
||||
idx.map(|idx| Node {
|
||||
file: self.file,
|
||||
idx,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<'f> fmt::Debug for Node<'f> {
|
||||
fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
|
||||
write!(fmt, "{:?}@{:?}", self.kind(), self.range())?;
|
||||
if has_short_text(self.kind()) {
|
||||
write!(fmt, " \"{}\"", self.text())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn has_short_text(kind: SyntaxKind) -> bool {
|
||||
use syntax_kinds::*;
|
||||
match kind {
|
||||
IDENT | LIFETIME => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
impl<'f> cmp::PartialEq<Node<'f>> for Node<'f> {
|
||||
fn eq(&self, other: &Node<'f>) -> bool {
|
||||
self.idx == other.idx && ::std::ptr::eq(self.file, other.file)
|
||||
}
|
||||
}
|
||||
|
||||
impl<'f> cmp::Eq for Node<'f> {}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct SyntaxError<'f> {
|
||||
file: &'f File,
|
||||
idx: ErrorIdx,
|
||||
}
|
||||
|
||||
impl<'f> SyntaxError<'f> {
|
||||
pub fn message(&self) -> &'f str {
|
||||
self.data().message.as_str()
|
||||
}
|
||||
|
||||
pub fn after_child(&self) -> Option<Node<'f>> {
|
||||
let idx = self.data().after_child?;
|
||||
Some(Node {
|
||||
file: self.file,
|
||||
idx,
|
||||
})
|
||||
}
|
||||
|
||||
fn data(&self) -> &'f SyntaxErrorData {
|
||||
&self.file.errors[self.idx]
|
||||
}
|
||||
|
||||
fn next(&self) -> Option<SyntaxError<'f>> {
|
||||
let next_idx = self.idx.0 + 1;
|
||||
if !((next_idx as usize) < self.file.errors.len()) {
|
||||
return None;
|
||||
}
|
||||
let result = SyntaxError {
|
||||
file: self.file,
|
||||
idx: ErrorIdx(next_idx),
|
||||
};
|
||||
if result.data().node != self.data().node {
|
||||
return None;
|
||||
}
|
||||
Some(result)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Children<'f> {
|
||||
next: Option<Node<'f>>,
|
||||
}
|
||||
|
||||
impl<'f> Iterator for Children<'f> {
|
||||
type Item = Node<'f>;
|
||||
|
||||
fn next(&mut self) -> Option<Node<'f>> {
|
||||
let next = self.next;
|
||||
self.next = next.and_then(|node| node.as_node(node.data().next_sibling));
|
||||
next
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct SyntaxErrors<'f> {
|
||||
next: Option<SyntaxError<'f>>,
|
||||
}
|
||||
|
||||
impl<'f> Iterator for SyntaxErrors<'f> {
|
||||
type Item = SyntaxError<'f>;
|
||||
|
||||
fn next(&mut self) -> Option<SyntaxError<'f>> {
|
||||
let next = self.next;
|
||||
self.next = next.as_ref().and_then(SyntaxError::next);
|
||||
next
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
struct NodeIdx(u32);
|
||||
|
||||
#[derive(Debug)]
|
||||
struct NodeData {
|
||||
kind: SyntaxKind,
|
||||
range: TextRange,
|
||||
parent: Option<NodeIdx>,
|
||||
first_child: Option<NodeIdx>,
|
||||
next_sibling: Option<NodeIdx>,
|
||||
}
|
||||
|
||||
impl ::std::ops::Index<NodeIdx> for Vec<NodeData> {
|
||||
type Output = NodeData;
|
||||
|
||||
fn index(&self, NodeIdx(idx): NodeIdx) -> &NodeData {
|
||||
&self[idx as usize]
|
||||
}
|
||||
}
|
||||
|
||||
impl ::std::ops::IndexMut<NodeIdx> for Vec<NodeData> {
|
||||
fn index_mut(&mut self, NodeIdx(idx): NodeIdx) -> &mut NodeData {
|
||||
&mut self[idx as usize]
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct ErrorIdx(u32);
|
||||
|
||||
#[derive(Debug)]
|
||||
struct SyntaxErrorData {
|
||||
node: NodeIdx,
|
||||
message: String,
|
||||
after_child: Option<NodeIdx>,
|
||||
}
|
||||
|
||||
impl ::std::ops::Index<ErrorIdx> for Vec<SyntaxErrorData> {
|
||||
type Output = SyntaxErrorData;
|
||||
|
||||
fn index(&self, ErrorIdx(idx): ErrorIdx) -> &SyntaxErrorData {
|
||||
&self[idx as usize]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,13 +116,6 @@ impl GreenNode {
|
|||
GreenNodeData::Branch(_) => false
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn leaf_text(&self) -> &str {
|
||||
match &self.data {
|
||||
GreenNodeData::Leaf(l) => l.text.as_str(),
|
||||
GreenNodeData::Branch(_) => panic!("not a leaf")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ mod syntax;
|
|||
|
||||
use std::{
|
||||
sync::{Arc, Weak},
|
||||
ops::Deref,
|
||||
mem
|
||||
};
|
||||
pub(crate) use self::{
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use std::sync::{Arc, Weak, RwLock};
|
||||
use std::sync::{Arc, RwLock};
|
||||
use {
|
||||
TextUnit, SyntaxKind, TextRange,
|
||||
TextUnit,
|
||||
yellow::{Ptr, GreenNode, TextLen}
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ use {
|
|||
TextRange, TextUnit, SyntaxKind,
|
||||
yellow::{Ptr, RedNode, GreenNode, TextLen},
|
||||
};
|
||||
use yellow::green::GreenTrivia;
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct SyntaxNode {
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
extern crate libsyntax2;
|
||||
extern crate testutils;
|
||||
|
||||
use libsyntax2::{parse, tokenize, parse_green};
|
||||
use libsyntax2::utils::{dump_tree, dump_tree_green};
|
||||
use libsyntax2::{tokenize, parse_green};
|
||||
use libsyntax2::utils::{dump_tree_green};
|
||||
use testutils::dir_tests;
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -2,14 +2,14 @@ extern crate libsyntax2;
|
|||
|
||||
use std::io::Read;
|
||||
|
||||
use libsyntax2::{parse, tokenize};
|
||||
use libsyntax2::utils::dump_tree;
|
||||
use libsyntax2::{parse_green, tokenize};
|
||||
use libsyntax2::utils::dump_tree_green;
|
||||
|
||||
fn main() {
|
||||
let text = read_input();
|
||||
let tokens = tokenize(&text);
|
||||
let file = parse(text, &tokens);
|
||||
let tree = dump_tree(&file);
|
||||
let file = parse_green(text, &tokens);
|
||||
let tree = dump_tree_green(&file);
|
||||
println!("{}", tree);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue