diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index d81ea033950..d66046ca1df 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs @@ -11,7 +11,7 @@ mod numbers; use self::numbers::scan_number; mod strings; -use self::strings::{string_literal_start, scan_char, scan_byte_char_or_string}; +use self::strings::{string_literal_start, scan_char, scan_byte_char_or_string, scan_string, scan_raw_string}; pub fn next_token(text: &str) -> Token { assert!(!text.is_empty()); @@ -128,6 +128,16 @@ fn next_token_inner(c: char, ptr: &mut Ptr) -> SyntaxKind { scan_literal_suffix(ptr); return kind }, + '"' => { + scan_string(ptr); + scan_literal_suffix(ptr); + return STRING; + } + 'r' => { + scan_raw_string(ptr); + scan_literal_suffix(ptr); + return RAW_STRING; + } _ => (), } ERROR diff --git a/src/lexer/strings.rs b/src/lexer/strings.rs index 283ce8feb37..2c1d863746d 100644 --- a/src/lexer/strings.rs +++ b/src/lexer/strings.rs @@ -47,6 +47,27 @@ pub(crate) fn scan_byte_char_or_string(ptr: &mut Ptr) -> SyntaxKind { } } +pub(crate) fn scan_string(ptr: &mut Ptr) { + while let Some(c) = ptr.bump() { + if c == '"' { + return + } + } +} + +pub(crate) fn scan_raw_string(ptr: &mut Ptr) { + if !ptr.next_is('"') { + return + } + ptr.bump(); + + while let Some(c) = ptr.bump() { + if c == '"' { + return + } + } +} + fn scan_byte(ptr: &mut Ptr) { if ptr.next_is('\'') { ptr.bump(); @@ -83,4 +104,5 @@ fn scan_raw_byte_string(ptr: &mut Ptr) { fn scan_char_or_byte(ptr: &mut Ptr) { //FIXME: deal with escape sequencies ptr.bump(); -} \ No newline at end of file +} + diff --git a/tests/data/lexer/0008_strings.rs b/tests/data/lexer/0008_byte_strings.rs similarity index 100% rename from tests/data/lexer/0008_strings.rs rename to tests/data/lexer/0008_byte_strings.rs diff --git a/tests/data/lexer/0008_strings.txt b/tests/data/lexer/0008_byte_strings.txt similarity index 100% rename from tests/data/lexer/0008_strings.txt rename to tests/data/lexer/0008_byte_strings.txt diff --git a/tests/data/lexer/0009_strings.rs b/tests/data/lexer/0009_strings.rs new file mode 100644 index 00000000000..7b7faa5d8f8 --- /dev/null +++ b/tests/data/lexer/0009_strings.rs @@ -0,0 +1 @@ +"hello" r"world" diff --git a/tests/data/lexer/0009_strings.txt b/tests/data/lexer/0009_strings.txt new file mode 100644 index 00000000000..7fb6b7b368c --- /dev/null +++ b/tests/data/lexer/0009_strings.txt @@ -0,0 +1,4 @@ +STRING 7 "\"hello\"" +WHITESPACE 1 " " +RAW_STRING 8 "r\"world\"" +WHITESPACE 1 "\n"