From 966e9db2b83802dfb55d55bd3a26e69dced1bbd7 Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Wed, 1 Aug 2018 10:40:07 +0300 Subject: [PATCH] Extract libeditor --- Cargo.toml | 2 +- cli/src/main.rs | 4 +-- code/native/Cargo.toml | 2 +- code/native/src/lib.rs | 40 +++++----------------- libeditor/Cargo.toml | 9 +++++ libeditor/src/lib.rs | 53 ++++++++++++++++++++++++++++++ src/algo/walk.rs | 1 - src/ast.rs | 2 +- src/lib.rs | 2 +- src/parser_impl/event.rs | 2 +- src/parser_impl/mod.rs | 8 ++--- src/syntax_kinds/generated.rs | 41 +++++++++++++++++++++++ src/syntax_kinds/generated.rs.tera | 10 ++++++ src/yellow/builder.rs | 10 +++--- tests/parser.rs | 2 +- 15 files changed, 138 insertions(+), 50 deletions(-) create mode 100644 libeditor/Cargo.toml create mode 100644 libeditor/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 0a9d1815ec2..d616133528a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Aleksey Kladov "] license = "MIT OR Apache-2.0" [workspace] -members = [ "tools", "cli" ] +members = [ "tools", "cli", "libeditor" ] [dependencies] unicode-xid = "0.1.0" diff --git a/cli/src/main.rs b/cli/src/main.rs index f6e7e2fedf1..0044841ed34 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -52,7 +52,7 @@ fn main() -> Result<()> { fn parse() -> Result { let text = read_stdin()?; let start = Instant::now(); - let file = libsyntax2::parse(text); + let file = libsyntax2::parse(&text); eprintln!("elapsed {:?}", start.elapsed()); let tree = libsyntax2::utils::dump_tree(&file); Ok(tree) @@ -74,7 +74,7 @@ fn render_test(file: &Path, line: usize) -> Result<(String, String)> { None => bail!("No test found at line {} at {}", line, file.display()), Some((_start_line, test)) => test, }; - let file = libsyntax2::parse(test.text.clone()); + let file = libsyntax2::parse(&test.text); let tree = libsyntax2::utils::dump_tree(&file); Ok((test.text, tree)) } diff --git a/code/native/Cargo.toml b/code/native/Cargo.toml index 1648dfb3382..3a27051b2f2 100644 --- a/code/native/Cargo.toml +++ b/code/native/Cargo.toml @@ -16,4 +16,4 @@ neon-build = "0.2.0" [dependencies] neon = "0.2.0" -libsyntax2 = { path = "../../" } +libeditor = { path = "../../libeditor" } diff --git a/code/native/src/lib.rs b/code/native/src/lib.rs index 068767fabd6..aae7ad2f394 100644 --- a/code/native/src/lib.rs +++ b/code/native/src/lib.rs @@ -1,42 +1,20 @@ #[macro_use] extern crate neon; -extern crate libsyntax2; +extern crate libeditor; -use libsyntax2::{ - TextRange, - File, - utils::dump_tree, - SyntaxKind::*, - algo, -}; use neon::prelude::*; pub struct Wrapper { - inner: File, + inner: libeditor::File, } -impl Wrapper { - fn highlight(&self) -> Vec<(TextRange, &'static str)> { - let mut res = Vec::new(); - let syntax = self.inner.syntax(); - for node in algo::walk::preorder(syntax.as_ref()) { - if node.kind() == ERROR { - res.push((node.range(), "error")) - } - } - res - } -} - - - declare_types! { /// A class for generating greeting strings. pub class RustFile for Wrapper { init(mut cx) { let text = cx.argument::(0)?.value(); Ok(Wrapper { - inner: File::parse(&text) + inner: libeditor::File::new(&text) }) } @@ -45,7 +23,7 @@ declare_types! { let tree = { let guard = cx.lock(); let wrapper = this.borrow(&guard); - dump_tree(&wrapper.inner.syntax()) + wrapper.inner.syntax_tree() }; Ok(cx.string(tree.as_str()).upcast()) } @@ -55,15 +33,15 @@ declare_types! { let highlights = { let guard = cx.lock(); let wrapper = this.borrow(&guard); - wrapper.highlight() + wrapper.inner.highlight() }; let res = cx.empty_array(); - for (i, (range, tag)) in highlights.into_iter().enumerate() { - let start: u32 = range.start().into(); - let end: u32 = range.end().into(); + for (i, hl) in highlights.into_iter().enumerate() { + let start: u32 = hl.range.start().into(); + let end: u32 = hl.range.end().into(); let start = cx.number(start); let end = cx.number(end); - let tag = cx.string(tag); + let tag = cx.string(hl.tag); let hl = cx.empty_array(); hl.set(&mut cx, 0, start)?; hl.set(&mut cx, 1, end)?; diff --git a/libeditor/Cargo.toml b/libeditor/Cargo.toml new file mode 100644 index 00000000000..1a532ce2ff4 --- /dev/null +++ b/libeditor/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "libeditor" +version = "0.1.0" +authors = ["Aleksey Kladov "] +publish = false + +[dependencies] +libsyntax2 = { path = "../" } +text_unit = "0.1.2" diff --git a/libeditor/src/lib.rs b/libeditor/src/lib.rs new file mode 100644 index 00000000000..119bdb2d6fb --- /dev/null +++ b/libeditor/src/lib.rs @@ -0,0 +1,53 @@ +extern crate libsyntax2; +extern crate text_unit; + +use libsyntax2::{ + algo::walk, + SyntaxKind::*, +}; +use text_unit::TextRange; + +pub struct File { + inner: libsyntax2::File +} + +pub struct HighlightedRange { + pub range: TextRange, + pub tag: &'static str, +} + +impl File { + pub fn new(text: &str) -> File { + File { + inner: libsyntax2::File::parse(text) + } + } + + pub fn highlight(&self) -> Vec { + let syntax = self.inner.syntax(); + let mut res = Vec::new(); + for node in walk::preorder(syntax.as_ref()) { + let tag = match node.kind() { + ERROR => "error", + COMMENT | DOC_COMMENT => "comment", + STRING | RAW_STRING | RAW_BYTE_STRING | BYTE_STRING => "string", + ATTR => "attribute", + NAME_REF => "text", + NAME => "function", + INT_NUMBER | FLOAT_NUMBER | CHAR | BYTE => "literal", + LIFETIME => "parameter", + k if k.is_keyword() => "keyword", + _ => continue, + }; + res.push(HighlightedRange { + range: node.range(), + tag + }) + } + res + } + + pub fn syntax_tree(&self) -> String { + ::libsyntax2::utils::dump_tree(&self.inner.syntax()) + } +} diff --git a/src/algo/walk.rs b/src/algo/walk.rs index a50ec2a0930..ad0f2d8fb5d 100644 --- a/src/algo/walk.rs +++ b/src/algo/walk.rs @@ -6,7 +6,6 @@ pub fn preorder<'a>(root: SyntaxNodeRef<'a>) -> impl Iterator None, }) } - #[derive(Debug, Copy, Clone)] pub enum WalkEvent<'a> { Enter(SyntaxNodeRef<'a>), diff --git a/src/ast.rs b/src/ast.rs index 48e1d23ac1e..caf5fb7ef4f 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -9,7 +9,7 @@ pub struct File> { impl File> { pub fn parse(text: &str) -> Self { File { - syntax: ::parse(text.to_owned()), + syntax: ::parse(text), } } } diff --git a/src/lib.rs b/src/lib.rs index 8f25de9a441..611b8749286 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,7 +45,7 @@ pub use { yellow::{SyntaxNode, SyntaxNodeRef, SyntaxRoot, TreeRoot}, }; -pub fn parse(text: String) -> SyntaxNode { +pub fn parse(text: &str) -> SyntaxNode { let tokens = tokenize(&text); parser_impl::parse::(text, &tokens) } diff --git a/src/parser_impl/event.rs b/src/parser_impl/event.rs index eb5d0a4be40..66a0b6fc05a 100644 --- a/src/parser_impl/event.rs +++ b/src/parser_impl/event.rs @@ -76,7 +76,7 @@ pub(crate) enum Event { }, } -pub(super) fn process(builder: &mut impl Sink, tokens: &[Token], events: Vec) { +pub(super) fn process<'a>(builder: &mut impl Sink<'a>, tokens: &[Token], events: Vec) { let mut idx = 0; let mut holes = Vec::new(); diff --git a/src/parser_impl/mod.rs b/src/parser_impl/mod.rs index b58094be38e..2791c8da559 100644 --- a/src/parser_impl/mod.rs +++ b/src/parser_impl/mod.rs @@ -14,10 +14,10 @@ use { use SyntaxKind::{self, EOF, TOMBSTONE}; -pub(crate) trait Sink { +pub(crate) trait Sink<'a> { type Tree; - fn new(text: String) -> Self; + fn new(text: &'a str) -> Self; fn leaf(&mut self, kind: SyntaxKind, len: TextUnit); fn start_internal(&mut self, kind: SyntaxKind); @@ -27,9 +27,9 @@ pub(crate) trait Sink { } /// Parse a sequence of tokens into the representative node tree -pub(crate) fn parse(text: String, tokens: &[Token]) -> S::Tree { +pub(crate) fn parse<'a, S: Sink<'a>>(text: &'a str, tokens: &[Token]) -> S::Tree { let events = { - let input = input::ParserInput::new(&text, tokens); + let input = input::ParserInput::new(text, tokens); let parser_impl = ParserImpl::new(&input); let mut parser_api = Parser(parser_impl); grammar::file(&mut parser_api); diff --git a/src/syntax_kinds/generated.rs b/src/syntax_kinds/generated.rs index 435be781c18..cbcd0c4e144 100644 --- a/src/syntax_kinds/generated.rs +++ b/src/syntax_kinds/generated.rs @@ -169,6 +169,47 @@ pub enum SyntaxKind { use self::SyntaxKind::*; impl SyntaxKind { + pub fn is_keyword(self) -> bool { + match self { + | USE_KW + | FN_KW + | STRUCT_KW + | ENUM_KW + | TRAIT_KW + | IMPL_KW + | TRUE_KW + | FALSE_KW + | AS_KW + | EXTERN_KW + | CRATE_KW + | MOD_KW + | PUB_KW + | SELF_KW + | SUPER_KW + | IN_KW + | WHERE_KW + | FOR_KW + | LOOP_KW + | WHILE_KW + | IF_KW + | ELSE_KW + | MATCH_KW + | CONST_KW + | STATIC_KW + | MUT_KW + | UNSAFE_KW + | TYPE_KW + | REF_KW + | LET_KW + | MOVE_KW + | AUTO_KW + | DEFAULT_KW + | UNION_KW + => true, + _ => false + } + } + pub(crate) fn info(self) -> &'static SyntaxInfo { match self { SEMI => &SyntaxInfo { name: "SEMI" }, diff --git a/src/syntax_kinds/generated.rs.tera b/src/syntax_kinds/generated.rs.tera index d719c8312f9..4e2ee56c3b3 100644 --- a/src/syntax_kinds/generated.rs.tera +++ b/src/syntax_kinds/generated.rs.tera @@ -24,6 +24,16 @@ pub enum SyntaxKind { use self::SyntaxKind::*; impl SyntaxKind { + pub fn is_keyword(self) -> bool { + match self { +{%- for kw in concat(a=keywords, b=contextual_keywords) %} + | {{kw | upper}}_KW +{%- endfor %} + => true, + _ => false + } + } + pub(crate) fn info(self) -> &'static SyntaxInfo { match self { {%- for t in concat(a=single_byte_tokens, b=multi_byte_tokens) %} diff --git a/src/yellow/builder.rs b/src/yellow/builder.rs index b68ba789ebc..dbe646cd749 100644 --- a/src/yellow/builder.rs +++ b/src/yellow/builder.rs @@ -4,20 +4,18 @@ use { SyntaxKind, TextRange, TextUnit, }; -pub(crate) struct GreenBuilder { - text: String, +pub(crate) struct GreenBuilder<'a> { + text: &'a str, stack: Vec, pos: TextUnit, root: Option, errors: Vec, } -impl GreenBuilder {} - -impl Sink for GreenBuilder { +impl<'a> Sink<'a> for GreenBuilder<'a> { type Tree = SyntaxNode; - fn new(text: String) -> Self { + fn new(text: &'a str) -> Self { GreenBuilder { text, stack: Vec::new(), diff --git a/tests/parser.rs b/tests/parser.rs index 770610974eb..af2ae11bb37 100644 --- a/tests/parser.rs +++ b/tests/parser.rs @@ -8,7 +8,7 @@ use testutils::dir_tests; #[test] fn parser_tests() { dir_tests(&["parser/inline", "parser/ok", "parser/err"], |text| { - let file = parse(text.to_string()); + let file = parse(text); dump_tree(&file) }) }