Migrate to text-unit

This commit is contained in:
Aleksey Kladov 2018-07-28 13:07:10 +03:00
parent 52dc0ddc84
commit 8d9961b753
8 changed files with 18 additions and 153 deletions

View File

@ -9,6 +9,7 @@ members = [ "tools" ]
[dependencies]
unicode-xid = "0.1.0"
text_unit = "0.1.1"
[dev-dependencies]
testutils = { path = "./tests/testutils" }

View File

@ -11,7 +11,7 @@ impl<'s> Ptr<'s> {
pub fn new(text: &'s str) -> Ptr<'s> {
Ptr {
text,
len: TextUnit::new(0),
len: 0.into(),
}
}
@ -47,7 +47,7 @@ impl<'s> Ptr<'s> {
pub fn bump(&mut self) -> Option<char> {
let ch = self.chars().next()?;
self.len += TextUnit::len_of_char(ch);
self.len += TextUnit::of_char(ch);
Some(ch)
}

View File

@ -16,14 +16,14 @@
//#![warn(unreachable_pub)] // rust-lang/rust#47816
extern crate unicode_xid;
extern crate text_unit;
mod text;
mod tree;
mod lexer;
mod parser;
pub mod syntax_kinds;
pub use text::{TextRange, TextUnit};
pub use text_unit::{TextRange, TextUnit};
pub use tree::{File, Node, SyntaxKind, Token};
pub(crate) use tree::{ErrorMsg, FileBuilder, Sink};
pub use lexer::{next_token, tokenize};

View File

@ -1,5 +1,7 @@
use {ErrorMsg, File, FileBuilder, Sink, SyntaxKind, TextUnit, Token};
use syntax_kinds::TOMBSTONE;
use {
ErrorMsg, File, FileBuilder, Sink, SyntaxKind, Token,
syntax_kinds::TOMBSTONE,
};
use super::is_insignificant;
/// `Parser` produces a flat list of `Event`s.
@ -133,7 +135,7 @@ pub(super) fn to_file(text: String, tokens: &[Token], events: Vec<Event>) -> Fil
builder.leaf(token.kind, token.len);
idx += 1
}
let mut len = TextUnit::new(0);
let mut len = 0.into();
for _ in 0..n_raw_tokens {
len += tokens[idx].len;
idx += 1;

View File

@ -14,7 +14,7 @@ impl<'t> ParserInput<'t> {
pub fn new(text: &'t str, raw_tokens: &'t [Token]) -> ParserInput<'t> {
let mut tokens = Vec::new();
let mut start_offsets = Vec::new();
let mut len = TextUnit::new(0);
let mut len = 0.into();
for &token in raw_tokens.iter() {
if !is_insignificant(token.kind) {
tokens.push(token);
@ -44,7 +44,7 @@ impl<'t> ParserInput<'t> {
if !(idx < self.tokens.len()) {
return "";
}
let range = TextRange::from_len(self.start_offsets[idx], self.tokens[idx].len);
let range = TextRange::offset_len(self.start_offsets[idx], self.tokens[idx].len);
&self.text[range]
}
}

View File

@ -1,136 +0,0 @@
use std::fmt;
use std::ops;
/// An text position in a source file
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct TextUnit(u32);
impl TextUnit {
/// The positional offset required for one character
pub fn len_of_char(c: char) -> TextUnit {
TextUnit(c.len_utf8() as u32)
}
#[allow(missing_docs)]
pub fn new(val: u32) -> TextUnit {
TextUnit(val)
}
}
impl fmt::Debug for TextUnit {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
<Self as fmt::Display>::fmt(self, f)
}
}
impl fmt::Display for TextUnit {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
self.0.fmt(f)
}
}
impl From<TextUnit> for u32 {
fn from(tu: TextUnit) -> u32 {
tu.0
}
}
impl From<u32> for TextUnit {
fn from(tu: u32) -> TextUnit {
TextUnit::new(tu)
}
}
impl ops::Add<TextUnit> for TextUnit {
type Output = TextUnit;
fn add(self, rhs: TextUnit) -> TextUnit {
TextUnit(self.0 + rhs.0)
}
}
impl ops::AddAssign<TextUnit> for TextUnit {
fn add_assign(&mut self, rhs: TextUnit) {
self.0 += rhs.0
}
}
impl ops::Sub<TextUnit> for TextUnit {
type Output = TextUnit;
fn sub(self, rhs: TextUnit) -> TextUnit {
TextUnit(self.0 - rhs.0)
}
}
impl ops::SubAssign<TextUnit> for TextUnit {
fn sub_assign(&mut self, rhs: TextUnit) {
self.0 -= rhs.0
}
}
/// A range of text in a source file
#[derive(Clone, Copy, PartialEq, Eq)]
pub struct TextRange {
start: TextUnit,
end: TextUnit,
}
impl fmt::Debug for TextRange {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
<Self as fmt::Display>::fmt(self, f)
}
}
impl fmt::Display for TextRange {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(f, "[{}; {})", self.start(), self.end())
}
}
impl TextRange {
/// An length-0 range of text
pub fn empty() -> TextRange {
TextRange::from_to(TextUnit::new(0), TextUnit::new(0))
}
/// The left-inclusive range (`[from..to)`) between to points in the text
pub fn from_to(from: TextUnit, to: TextUnit) -> TextRange {
assert!(from <= to, "Invalid text range [{}; {})", from, to);
TextRange {
start: from,
end: to,
}
}
/// The range from some point over some length
pub fn from_len(from: TextUnit, len: TextUnit) -> TextRange {
TextRange::from_to(from, from + len)
}
/// The starting position of this range
pub fn start(&self) -> TextUnit {
self.start
}
/// The end position of this range
pub fn end(&self) -> TextUnit {
self.end
}
/// The length of this range
pub fn len(&self) -> TextUnit {
self.end - self.start
}
/// Is this range empty of any content?
pub fn is_empty(&self) -> bool {
self.start() == self.end()
}
}
impl ops::Index<TextRange> for str {
type Output = str;
fn index(&self, index: TextRange) -> &str {
&self[index.start().0 as usize..index.end().0 as usize]
}
}

View File

@ -31,7 +31,7 @@ impl Sink for FileBuilder {
fn leaf(&mut self, kind: SyntaxKind, len: TextUnit) {
let leaf = NodeData {
kind,
range: TextRange::from_len(self.pos, len),
range: TextRange::offset_len(self.pos, len),
parent: None,
first_child: None,
next_sibling: None,
@ -44,7 +44,7 @@ impl Sink for FileBuilder {
fn start_internal(&mut self, kind: SyntaxKind) {
let node = NodeData {
kind,
range: TextRange::from_len(self.pos, 0.into()),
range: TextRange::offset_len(self.pos, 0.into()),
parent: None,
first_child: None,
next_sibling: None,
@ -83,7 +83,7 @@ impl FileBuilder {
nodes: Vec::new(),
errors: Vec::new(),
in_progress: Vec::new(),
pos: TextUnit::new(0),
pos: 0.into(),
}
}

View File

@ -1,9 +1,7 @@
use text::{TextRange, TextUnit};
use std::fmt;
use std::cmp;
mod file_builder;
use ::{TextRange, TextUnit};
use std::{fmt, cmp};
pub(crate) use self::file_builder::{ErrorMsg, FileBuilder, Sink};
pub use syntax_kinds::SyntaxKind;