From 78593f1667b031365439e7e4c9a4d8d719b80369 Mon Sep 17 00:00:00 2001 From: Charlotte Meyer Date: Sat, 24 Sep 2022 17:17:56 +0000 Subject: [PATCH] feat(parser): parse double quoted strings --- crates/oyster_parser/src/ast.rs | 36 ++++++++++++-- crates/oyster_parser/src/lexer.rs | 40 +++++++++++++-- crates/oyster_parser/src/lib.rs | 2 +- crates/oyster_parser/src/parser.rs | 42 ++++++++++++---- crates/oyster_parser/tests/it/ast.rs | 27 ++++++++++ crates/oyster_parser/tests/it/cst.rs | 27 ++++++++++ crates/oyster_parser/tests/it/lexer.rs | 27 ++++++++++ crates/oyster_parser/tests/it/parser.rs | 27 ++++++++++ .../it__ast__double_quote_string.snap | 28 +++++++++++ ...t__ast__escaped_char_in_double_quotes.snap | 34 +++++++++++++ .../it__ast__unterminated_double_quotes.snap | 7 +++ .../it__cst__comment_in_pipeline.snap | 6 +-- .../it__cst__double_quote_string.snap | 41 ++++++++++++++++ ...t__cst__escaped_char_in_double_quotes.snap | 49 +++++++++++++++++++ .../it/snapshots/it__cst__inline_comment.snap | 2 +- .../it__cst__multiline_pipeline.snap | 6 +-- .../tests/it/snapshots/it__cst__pipeline.snap | 4 +- .../it__cst__reject_double_pipe.snap | 4 +- .../it__cst__reject_leading_pipe.snap | 4 +- .../it__cst__reject_pipe_semicolon.snap | 4 +- .../it__cst__reject_trailing_pipe.snap | 2 +- .../it/snapshots/it__cst__semicolon.snap | 4 +- .../it/snapshots/it__cst__simple_command.snap | 4 +- .../tests/it/snapshots/it__cst__unicode.snap | 2 +- .../it__cst__unterminated_double_quotes.snap | 41 ++++++++++++++++ .../tests/it/snapshots/it__cst__word.snap | 2 +- .../snapshots/it__cst__word_with_escape.snap | 4 +- .../it__lexer__double_quotes_in_command.snap | 8 +++ .../it__lexer__double_quotes_in_string.snap | 8 +++ .../it__lexer__escaped_quotes_in_string.snap | 8 +++ .../it/snapshots/it__lexer__plain_word.snap | 2 +- .../snapshots/it__lexer__word_with_hash.snap | 2 +- .../it__parser__comment_in_pipeline.snap | 6 +-- .../it__parser__double_quote_string.snap | 34 +++++++++++++ ...parser__escaped_char_in_double_quotes.snap | 42 ++++++++++++++++ .../snapshots/it__parser__inline_comment.snap | 2 +- .../it__parser__multiline_pipeline.snap | 6 +-- .../it/snapshots/it__parser__pipeline.snap | 4 +- .../it__parser__reject_double_pipe.snap | 4 +- .../it__parser__reject_leading_pipe.snap | 4 +- .../it__parser__reject_pipe_semicolon.snap | 4 +- .../it__parser__reject_trailing_pipe.snap | 2 +- .../it/snapshots/it__parser__semicolon.snap | 4 +- .../snapshots/it__parser__simple_command.snap | 4 +- .../it/snapshots/it__parser__unicode.snap | 2 +- ...t__parser__unterminated_double_quotes.snap | 34 +++++++++++++ .../tests/it/snapshots/it__parser__word.snap | 2 +- .../it__parser__word_with_escape.snap | 4 +- 48 files changed, 594 insertions(+), 68 deletions(-) create mode 100644 crates/oyster_parser/tests/it/snapshots/it__ast__double_quote_string.snap create mode 100644 crates/oyster_parser/tests/it/snapshots/it__ast__escaped_char_in_double_quotes.snap create mode 100644 crates/oyster_parser/tests/it/snapshots/it__ast__unterminated_double_quotes.snap create mode 100644 crates/oyster_parser/tests/it/snapshots/it__cst__double_quote_string.snap create mode 100644 crates/oyster_parser/tests/it/snapshots/it__cst__escaped_char_in_double_quotes.snap create mode 100644 crates/oyster_parser/tests/it/snapshots/it__cst__unterminated_double_quotes.snap create mode 100644 crates/oyster_parser/tests/it/snapshots/it__lexer__double_quotes_in_command.snap create mode 100644 crates/oyster_parser/tests/it/snapshots/it__lexer__double_quotes_in_string.snap create mode 100644 crates/oyster_parser/tests/it/snapshots/it__lexer__escaped_quotes_in_string.snap create mode 100644 crates/oyster_parser/tests/it/snapshots/it__parser__double_quote_string.snap create mode 100644 crates/oyster_parser/tests/it/snapshots/it__parser__escaped_char_in_double_quotes.snap create mode 100644 crates/oyster_parser/tests/it/snapshots/it__parser__unterminated_double_quotes.snap diff --git a/crates/oyster_parser/src/ast.rs b/crates/oyster_parser/src/ast.rs index 83b915d..a713781 100644 --- a/crates/oyster_parser/src/ast.rs +++ b/crates/oyster_parser/src/ast.rs @@ -73,11 +73,11 @@ fn build_tree_pipeline<'a>( ParseEvent::StartNode(NodeKind::Command) => { children.push(build_tree_command(parser, source, pos)?) } - ParseEvent::EndNode => break, ParseEvent::NewLeaf(NodeKind::Pipe, _) => { *pos += 1; children.last_mut().unwrap().1 = Redirect::Stdout; } + ParseEvent::EndNode => break, ParseEvent::NewLeaf(_, len) => *pos += len, ParseEvent::Error(err, _) => return Err(err), _ => unreachable!(), @@ -116,9 +116,38 @@ fn build_tree_word<'a>( ) -> Result, ParseError> { let mut children = vec![]; + while let Some(ev) = parser.next() { + match ev { + ParseEvent::NewLeaf(NodeKind::PlainText, len) => { + children.push(WordPart::Text(&source[*pos..*pos + len])); + *pos += len + } + ParseEvent::NewLeaf(NodeKind::EscapedChar, len) => { + children.push(WordPart::Text(&source[*pos + 1..*pos + len])); + *pos += len + } + ParseEvent::StartNode(NodeKind::DQuotedString) => { + build_tree_string(parser, source, pos, &mut children)? + } + ParseEvent::EndNode => break, + ParseEvent::NewLeaf(_, len) => *pos += len, + ParseEvent::Error(err, _) => return Err(err), + _ => unreachable!(), + } + } + + Ok(Word(children)) +} + +fn build_tree_string<'a>( + parser: &mut Parser, + source: &'a str, + pos: &mut usize, + children: &mut Vec>, +) -> Result<(), ParseError> { for ev in parser { match ev { - ParseEvent::NewLeaf(NodeKind::PlainWord, len) => { + ParseEvent::NewLeaf(NodeKind::PlainText, len) => { children.push(WordPart::Text(&source[*pos..*pos + len])); *pos += len } @@ -127,10 +156,11 @@ fn build_tree_word<'a>( *pos += len } ParseEvent::EndNode => break, + ParseEvent::NewLeaf(_, len) => *pos += len, ParseEvent::Error(err, _) => return Err(err), _ => unreachable!(), } } - Ok(Word(children)) + Ok(()) } diff --git a/crates/oyster_parser/src/lexer.rs b/crates/oyster_parser/src/lexer.rs index 14091e9..03f1097 100644 --- a/crates/oyster_parser/src/lexer.rs +++ b/crates/oyster_parser/src/lexer.rs @@ -12,8 +12,10 @@ pub enum TokenKind { Semicolon, /// A pipe. Pipe, - /// A plain, unquoted word. - PlainWord, + /// Double quotes. + DoubleQuote, + /// Plain text. Either outside or inside of quotes. + PlainText, /// A backslash followed by another character. EscapedChar, /// A line comment, from # to newline. @@ -101,6 +103,7 @@ impl Lexer<'_> { let kind = match c { ';' => TokenKind::Semicolon, '|' => TokenKind::Pipe, + '"' => TokenKind::DoubleQuote, c if is_whitespace(c) => { self.eat_while(is_whitespace); @@ -123,8 +126,37 @@ impl Lexer<'_> { } _ => { - self.eat_while(|c| ![' ', '\t', '\n', ';', '|', '\\'].contains(&c)); - TokenKind::PlainWord + self.eat_while(|c| ![' ', '\t', '\n', ';', '|', '\\', '"'].contains(&c)); + TokenKind::PlainText + } + }; + + let len = self.token_len(); + + Token { kind, len } + } + } + } + + /// Get the next token in the string sublanguage. + pub fn next_string_token(&mut self) -> Token { + match self.next_char() { + None => Token { + kind: TokenKind::Eof, + len: 0, + }, + Some(c) => { + let kind = match c { + '"' => TokenKind::DoubleQuote, + + '\\' => { + self.next_char(); + TokenKind::EscapedChar + } + + _ => { + self.eat_while(|c| !['"', '\\'].contains(&c)); + TokenKind::PlainText } }; diff --git a/crates/oyster_parser/src/lib.rs b/crates/oyster_parser/src/lib.rs index 9b9bc8e..12311ec 100644 --- a/crates/oyster_parser/src/lib.rs +++ b/crates/oyster_parser/src/lib.rs @@ -8,7 +8,7 @@ //! _terminator ::= SEMICOLON | NEWLINES | EOF //! pipeline ::= command (PIPE NEWLINES? command)* //! command ::= word+ -//! word ::= (PLAIN_WORD)+ +//! word ::= (PLAIN_TEXT|DQUOTES PLAIN_TEXT DQUOTES)+ //! //! extras ::= COMMENT | WHITESPACE | BACKSLASH_N //! ``` diff --git a/crates/oyster_parser/src/parser.rs b/crates/oyster_parser/src/parser.rs index 7135c19..e784a04 100644 --- a/crates/oyster_parser/src/parser.rs +++ b/crates/oyster_parser/src/parser.rs @@ -20,7 +20,8 @@ pub enum NodeKind { Newlines, Semicolon, Pipe, - PlainWord, + DoubleQuote, + PlainText, EscapedChar, Comment, @@ -28,6 +29,7 @@ pub enum NodeKind { Pipeline, Command, Word, + DQuotedString, /// Read a pipe but didn't start word yet. PipelineCont, @@ -118,23 +120,35 @@ impl Iterator for Parser<'_> { } macro_rules! leaf { - ($type:expr) => {{ + ($type:expr $(, $mode:ident)?) => {{ use NodeKind::*; let len = self.lookahead.len; - self.lookahead = self.lex.next_command_token(); + self.lookahead = next_token!($($mode)?); Some(ParseEvent::NewLeaf($type, len)) }}; } macro_rules! error { - ($type:expr) => {{ + ($type:expr $(, $mode:ident)?) => {{ use ParseError::*; let len = self.lookahead.len; - self.lookahead = self.lex.next_command_token(); + self.lookahead = next_token!($($mode)?); Some(ParseEvent::Error($type, len)) }}; } + macro_rules! next_token { + () => { + self.lex.next_command_token() + }; + (Command) => { + self.lex.next_command_token() + }; + (String) => { + self.lex.next_string_token() + }; + } + use TokenKind::*; if let Some(ev) = self.buffer.take() { @@ -149,7 +163,7 @@ impl Iterator for Parser<'_> { Newlines => leaf!(Newlines), Semicolon => leaf!(Semicolon), Comment => leaf!(Comment), - PlainWord | EscapedChar => call!(Pipeline), + PlainText | DoubleQuote | EscapedChar => call!(Pipeline), Pipe => error!(UnexpectedPipe), Eof => chain!(None, ret!()), // return silently }, @@ -157,14 +171,14 @@ impl Iterator for Parser<'_> { Whitespace => leaf!(Whitespace), Comment => leaf!(Comment), Pipe => chain!(leaf!(Pipe), call!(PipelineCont)), - PlainWord | EscapedChar => call!(Command), + PlainText | DoubleQuote | EscapedChar => call!(Command), Newlines | Semicolon | Eof => ret!(), }, NodeKind::PipelineCont => match self.lookahead.kind { Whitespace => leaf!(Whitespace), Newlines => leaf!(Newlines), Comment => leaf!(Comment), - PlainWord | EscapedChar => tailcall!(Command), + PlainText | DoubleQuote | EscapedChar => tailcall!(Command), Semicolon => chain_buf!(chain!(error!(UnexpectedSemicolon), ret!()), ret!()), Pipe => chain!(error!(UnexpectedPipe), ret!()), Eof => chain!(error!(UnexpectedEof), ret!()), @@ -172,15 +186,23 @@ impl Iterator for Parser<'_> { NodeKind::Command => match self.lookahead.kind { Whitespace => leaf!(Whitespace), Comment => leaf!(Comment), - PlainWord | EscapedChar => call!(Word), + PlainText | DoubleQuote | EscapedChar => call!(Word), Newlines | Semicolon | Eof => ret!(), Pipe => ret!(), }, NodeKind::Word => match self.lookahead.kind { - PlainWord => leaf!(PlainWord), + PlainText => leaf!(PlainText), EscapedChar => leaf!(EscapedChar), + DoubleQuote => chain_buf!(call!(DQuotedString), leaf!(DoubleQuote, String)), Comment | Whitespace | Newlines | Semicolon | Pipe | Eof => ret!(), }, + NodeKind::DQuotedString => match self.lookahead.kind { + PlainText => leaf!(PlainText, String), + EscapedChar => leaf!(EscapedChar, String), + DoubleQuote => chain_buf!(leaf!(DoubleQuote, Command), ret!()), + Eof => chain_buf!(error!(UnexpectedEof, Command), ret!()), + _ => unreachable!(), + }, _ => unreachable!(), }, } diff --git a/crates/oyster_parser/tests/it/ast.rs b/crates/oyster_parser/tests/it/ast.rs index c1bc542..436b73f 100644 --- a/crates/oyster_parser/tests/it/ast.rs +++ b/crates/oyster_parser/tests/it/ast.rs @@ -143,3 +143,30 @@ fn reject_double_pipe() { assert_snapshot!(actual); } + +#[test] +fn double_quote_string() { + let source = r#""hello world""#; + + let actual = parse(source); + + assert_snapshot!(actual); +} + +#[test] +fn escaped_char_in_double_quotes() { + let source = r#""hello \" world""#; + + let actual = parse(source); + + assert_snapshot!(actual); +} + +#[test] +fn unterminated_double_quotes() { + let source = r#""hello world"#; + + let actual = parse(source); + + assert_snapshot!(actual); +} diff --git a/crates/oyster_parser/tests/it/cst.rs b/crates/oyster_parser/tests/it/cst.rs index f75d039..b1af315 100644 --- a/crates/oyster_parser/tests/it/cst.rs +++ b/crates/oyster_parser/tests/it/cst.rs @@ -143,3 +143,30 @@ fn reject_double_pipe() { assert_snapshot!(actual); } + +#[test] +fn double_quote_string() { + let source = r#""hello world""#; + + let actual = parse(source); + + assert_snapshot!(actual); +} + +#[test] +fn escaped_char_in_double_quotes() { + let source = r#""hello \" world""#; + + let actual = parse(source); + + assert_snapshot!(actual); +} + +#[test] +fn unterminated_double_quotes() { + let source = r#""hello world"#; + + let actual = parse(source); + + assert_snapshot!(actual); +} diff --git a/crates/oyster_parser/tests/it/lexer.rs b/crates/oyster_parser/tests/it/lexer.rs index 98c78e4..0834534 100644 --- a/crates/oyster_parser/tests/it/lexer.rs +++ b/crates/oyster_parser/tests/it/lexer.rs @@ -81,3 +81,30 @@ fn comment() { assert_snapshot!(actual); } + +#[test] +fn double_quotes_in_command() { + let source = r#"""#; + + let actual = Lexer::new(source).next_command_token(); + + assert_snapshot!(actual); +} + +#[test] +fn double_quotes_in_string() { + let source = r#"""#; + + let actual = Lexer::new(source).next_string_token(); + + assert_snapshot!(actual); +} + +#[test] +fn escaped_quotes_in_string() { + let source = r#"\""#; + + let actual = Lexer::new(source).next_string_token(); + + assert_snapshot!(actual); +} diff --git a/crates/oyster_parser/tests/it/parser.rs b/crates/oyster_parser/tests/it/parser.rs index f81cdb0..b456b24 100644 --- a/crates/oyster_parser/tests/it/parser.rs +++ b/crates/oyster_parser/tests/it/parser.rs @@ -143,3 +143,30 @@ fn reject_double_pipe() { assert_snapshot!(actual); } + +#[test] +fn double_quote_string() { + let source = r#""hello world""#; + + let actual = parse(source); + + assert_snapshot!(actual); +} + +#[test] +fn escaped_char_in_double_quotes() { + let source = r#""hello \" world""#; + + let actual = parse(source); + + assert_snapshot!(actual); +} + +#[test] +fn unterminated_double_quotes() { + let source = r#""hello world"#; + + let actual = parse(source); + + assert_snapshot!(actual); +} diff --git a/crates/oyster_parser/tests/it/snapshots/it__ast__double_quote_string.snap b/crates/oyster_parser/tests/it/snapshots/it__ast__double_quote_string.snap new file mode 100644 index 0000000..1aac750 --- /dev/null +++ b/crates/oyster_parser/tests/it/snapshots/it__ast__double_quote_string.snap @@ -0,0 +1,28 @@ +--- +source: crates/oyster_parser/tests/it/ast.rs +expression: actual +--- +Ok( + Code( + [ + Pipeline( + Pipeline( + [ + Command( + [ + Word( + [ + Text( + "hello world", + ), + ], + ), + ], + None, + ), + ], + ), + ), + ], + ), +) diff --git a/crates/oyster_parser/tests/it/snapshots/it__ast__escaped_char_in_double_quotes.snap b/crates/oyster_parser/tests/it/snapshots/it__ast__escaped_char_in_double_quotes.snap new file mode 100644 index 0000000..bf3f7eb --- /dev/null +++ b/crates/oyster_parser/tests/it/snapshots/it__ast__escaped_char_in_double_quotes.snap @@ -0,0 +1,34 @@ +--- +source: crates/oyster_parser/tests/it/ast.rs +expression: actual +--- +Ok( + Code( + [ + Pipeline( + Pipeline( + [ + Command( + [ + Word( + [ + Text( + "hello ", + ), + Text( + "\"", + ), + Text( + " world", + ), + ], + ), + ], + None, + ), + ], + ), + ), + ], + ), +) diff --git a/crates/oyster_parser/tests/it/snapshots/it__ast__unterminated_double_quotes.snap b/crates/oyster_parser/tests/it/snapshots/it__ast__unterminated_double_quotes.snap new file mode 100644 index 0000000..9254c8f --- /dev/null +++ b/crates/oyster_parser/tests/it/snapshots/it__ast__unterminated_double_quotes.snap @@ -0,0 +1,7 @@ +--- +source: crates/oyster_parser/tests/it/ast.rs +expression: actual +--- +Err( + UnexpectedEof, +) diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__comment_in_pipeline.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__comment_in_pipeline.snap index 1b37eb7..e4be687 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__cst__comment_in_pipeline.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__comment_in_pipeline.snap @@ -15,7 +15,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 6, }, ], @@ -49,7 +49,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 2, }, ], @@ -62,7 +62,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 2, }, ], diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__double_quote_string.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__double_quote_string.snap new file mode 100644 index 0000000..37ac2c7 --- /dev/null +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__double_quote_string.snap @@ -0,0 +1,41 @@ +--- +source: crates/oyster_parser/tests/it/cst.rs +expression: actual +--- +Tree { + kind: Program, + children: [ + Tree { + kind: Pipeline, + children: [ + Tree { + kind: Command, + children: [ + Tree { + kind: Word, + children: [ + Tree { + kind: DQuotedString, + children: [ + Leaf { + kind: DoubleQuote, + len: 1, + }, + Leaf { + kind: PlainText, + len: 11, + }, + Leaf { + kind: DoubleQuote, + len: 1, + }, + ], + }, + ], + }, + ], + }, + ], + }, + ], +} diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__escaped_char_in_double_quotes.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__escaped_char_in_double_quotes.snap new file mode 100644 index 0000000..f7e7e49 --- /dev/null +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__escaped_char_in_double_quotes.snap @@ -0,0 +1,49 @@ +--- +source: crates/oyster_parser/tests/it/cst.rs +expression: actual +--- +Tree { + kind: Program, + children: [ + Tree { + kind: Pipeline, + children: [ + Tree { + kind: Command, + children: [ + Tree { + kind: Word, + children: [ + Tree { + kind: DQuotedString, + children: [ + Leaf { + kind: DoubleQuote, + len: 1, + }, + Leaf { + kind: PlainText, + len: 6, + }, + Leaf { + kind: EscapedChar, + len: 2, + }, + Leaf { + kind: PlainText, + len: 6, + }, + Leaf { + kind: DoubleQuote, + len: 1, + }, + ], + }, + ], + }, + ], + }, + ], + }, + ], +} diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__inline_comment.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__inline_comment.snap index c89743f..086718c 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__cst__inline_comment.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__inline_comment.snap @@ -15,7 +15,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 6, }, ], diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__multiline_pipeline.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__multiline_pipeline.snap index bc7aa64..cb1061b 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__cst__multiline_pipeline.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__multiline_pipeline.snap @@ -15,7 +15,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 6, }, ], @@ -41,7 +41,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 2, }, ], @@ -54,7 +54,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 2, }, ], diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__pipeline.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__pipeline.snap index 68bf0c1..b628535 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__cst__pipeline.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__pipeline.snap @@ -15,7 +15,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 6, }, ], @@ -41,7 +41,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 3, }, ], diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__reject_double_pipe.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__reject_double_pipe.snap index e78c9dd..61ffaec 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__cst__reject_double_pipe.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__reject_double_pipe.snap @@ -15,7 +15,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 6, }, ], @@ -49,7 +49,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 3, }, ], diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__reject_leading_pipe.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__reject_leading_pipe.snap index d625082..6fd9e16 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__cst__reject_leading_pipe.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__reject_leading_pipe.snap @@ -15,7 +15,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 6, }, ], @@ -46,7 +46,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 3, }, ], diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__reject_pipe_semicolon.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__reject_pipe_semicolon.snap index 4a2da24..4c02141 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__cst__reject_pipe_semicolon.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__reject_pipe_semicolon.snap @@ -15,7 +15,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 6, }, ], @@ -54,7 +54,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 3, }, ], diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__reject_trailing_pipe.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__reject_trailing_pipe.snap index fc08fce..9412ebf 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__cst__reject_trailing_pipe.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__reject_trailing_pipe.snap @@ -15,7 +15,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 6, }, ], diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__semicolon.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__semicolon.snap index 28bf307..fb16a3a 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__cst__semicolon.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__semicolon.snap @@ -15,7 +15,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 5, }, ], @@ -42,7 +42,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 2, }, ], diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__simple_command.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__simple_command.snap index a4f5be9..cc042e5 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__cst__simple_command.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__simple_command.snap @@ -15,7 +15,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 4, }, ], @@ -28,7 +28,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 2, }, ], diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__unicode.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__unicode.snap index 525fc37..1b1d890 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__cst__unicode.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__unicode.snap @@ -15,7 +15,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 6, }, ], diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__unterminated_double_quotes.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__unterminated_double_quotes.snap new file mode 100644 index 0000000..7378ebb --- /dev/null +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__unterminated_double_quotes.snap @@ -0,0 +1,41 @@ +--- +source: crates/oyster_parser/tests/it/cst.rs +expression: actual +--- +Tree { + kind: Program, + children: [ + Tree { + kind: Pipeline, + children: [ + Tree { + kind: Command, + children: [ + Tree { + kind: Word, + children: [ + Tree { + kind: DQuotedString, + children: [ + Leaf { + kind: DoubleQuote, + len: 1, + }, + Leaf { + kind: PlainText, + len: 11, + }, + Error { + kind: UnexpectedEof, + len: 0, + }, + ], + }, + ], + }, + ], + }, + ], + }, + ], +} diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__word.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__word.snap index d3f9fcc..82c2595 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__cst__word.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__word.snap @@ -15,7 +15,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 4, }, ], diff --git a/crates/oyster_parser/tests/it/snapshots/it__cst__word_with_escape.snap b/crates/oyster_parser/tests/it/snapshots/it__cst__word_with_escape.snap index dea8584..ceb21f2 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__cst__word_with_escape.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__cst__word_with_escape.snap @@ -15,7 +15,7 @@ Tree { kind: Word, children: [ Leaf { - kind: PlainWord, + kind: PlainText, len: 5, }, Leaf { @@ -23,7 +23,7 @@ Tree { len: 2, }, Leaf { - kind: PlainWord, + kind: PlainText, len: 5, }, ], diff --git a/crates/oyster_parser/tests/it/snapshots/it__lexer__double_quotes_in_command.snap b/crates/oyster_parser/tests/it/snapshots/it__lexer__double_quotes_in_command.snap new file mode 100644 index 0000000..01bfb24 --- /dev/null +++ b/crates/oyster_parser/tests/it/snapshots/it__lexer__double_quotes_in_command.snap @@ -0,0 +1,8 @@ +--- +source: crates/oyster_parser/tests/it/lexer.rs +expression: actual +--- +Token { + kind: DoubleQuote, + len: 1, +} diff --git a/crates/oyster_parser/tests/it/snapshots/it__lexer__double_quotes_in_string.snap b/crates/oyster_parser/tests/it/snapshots/it__lexer__double_quotes_in_string.snap new file mode 100644 index 0000000..01bfb24 --- /dev/null +++ b/crates/oyster_parser/tests/it/snapshots/it__lexer__double_quotes_in_string.snap @@ -0,0 +1,8 @@ +--- +source: crates/oyster_parser/tests/it/lexer.rs +expression: actual +--- +Token { + kind: DoubleQuote, + len: 1, +} diff --git a/crates/oyster_parser/tests/it/snapshots/it__lexer__escaped_quotes_in_string.snap b/crates/oyster_parser/tests/it/snapshots/it__lexer__escaped_quotes_in_string.snap new file mode 100644 index 0000000..b482977 --- /dev/null +++ b/crates/oyster_parser/tests/it/snapshots/it__lexer__escaped_quotes_in_string.snap @@ -0,0 +1,8 @@ +--- +source: crates/oyster_parser/tests/it/lexer.rs +expression: actual +--- +Token { + kind: EscapedChar, + len: 2, +} diff --git a/crates/oyster_parser/tests/it/snapshots/it__lexer__plain_word.snap b/crates/oyster_parser/tests/it/snapshots/it__lexer__plain_word.snap index 68dd11a..98a8540 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__lexer__plain_word.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__lexer__plain_word.snap @@ -3,6 +3,6 @@ source: crates/oyster_parser/tests/it/lexer.rs expression: actual --- Token { - kind: PlainWord, + kind: PlainText, len: 6, } diff --git a/crates/oyster_parser/tests/it/snapshots/it__lexer__word_with_hash.snap b/crates/oyster_parser/tests/it/snapshots/it__lexer__word_with_hash.snap index ee41b05..48a6b74 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__lexer__word_with_hash.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__lexer__word_with_hash.snap @@ -3,6 +3,6 @@ source: crates/oyster_parser/tests/it/lexer.rs expression: actual --- Token { - kind: PlainWord, + kind: PlainText, len: 13, } diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__comment_in_pipeline.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__comment_in_pipeline.snap index e625cdd..d294b4d 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__parser__comment_in_pipeline.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__comment_in_pipeline.snap @@ -13,7 +13,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 6, ), EndNode, @@ -45,7 +45,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 2, ), EndNode, @@ -57,7 +57,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 2, ), EndNode, diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__double_quote_string.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__double_quote_string.snap new file mode 100644 index 0000000..06d445f --- /dev/null +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__double_quote_string.snap @@ -0,0 +1,34 @@ +--- +source: crates/oyster_parser/tests/it/parser.rs +expression: actual +--- +[ + StartNode( + Pipeline, + ), + StartNode( + Command, + ), + StartNode( + Word, + ), + StartNode( + DQuotedString, + ), + NewLeaf( + DoubleQuote, + 1, + ), + NewLeaf( + PlainText, + 11, + ), + NewLeaf( + DoubleQuote, + 1, + ), + EndNode, + EndNode, + EndNode, + EndNode, +] diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__escaped_char_in_double_quotes.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__escaped_char_in_double_quotes.snap new file mode 100644 index 0000000..b7d4053 --- /dev/null +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__escaped_char_in_double_quotes.snap @@ -0,0 +1,42 @@ +--- +source: crates/oyster_parser/tests/it/parser.rs +expression: actual +--- +[ + StartNode( + Pipeline, + ), + StartNode( + Command, + ), + StartNode( + Word, + ), + StartNode( + DQuotedString, + ), + NewLeaf( + DoubleQuote, + 1, + ), + NewLeaf( + PlainText, + 6, + ), + NewLeaf( + EscapedChar, + 2, + ), + NewLeaf( + PlainText, + 6, + ), + NewLeaf( + DoubleQuote, + 1, + ), + EndNode, + EndNode, + EndNode, + EndNode, +] diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__inline_comment.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__inline_comment.snap index 22ec3dd..bfab0d5 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__parser__inline_comment.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__inline_comment.snap @@ -13,7 +13,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 6, ), EndNode, diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__multiline_pipeline.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__multiline_pipeline.snap index 271b05b..ee6ebf5 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__parser__multiline_pipeline.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__multiline_pipeline.snap @@ -13,7 +13,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 6, ), EndNode, @@ -37,7 +37,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 2, ), EndNode, @@ -49,7 +49,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 2, ), EndNode, diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__pipeline.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__pipeline.snap index 9e2485b..07a65ec 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__parser__pipeline.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__pipeline.snap @@ -13,7 +13,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 6, ), EndNode, @@ -37,7 +37,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 3, ), EndNode, diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__reject_double_pipe.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__reject_double_pipe.snap index 43a9897..481e2a9 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__parser__reject_double_pipe.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__reject_double_pipe.snap @@ -13,7 +13,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 6, ), EndNode, @@ -45,7 +45,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 3, ), EndNode, diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__reject_leading_pipe.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__reject_leading_pipe.snap index 2aa98d8..cbf7e00 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__parser__reject_leading_pipe.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__reject_leading_pipe.snap @@ -13,7 +13,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 6, ), EndNode, @@ -41,7 +41,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 3, ), EndNode, diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__reject_pipe_semicolon.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__reject_pipe_semicolon.snap index 11abca2..e1a585e 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__parser__reject_pipe_semicolon.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__reject_pipe_semicolon.snap @@ -13,7 +13,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 6, ), EndNode, @@ -49,7 +49,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 3, ), EndNode, diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__reject_trailing_pipe.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__reject_trailing_pipe.snap index 83917fa..ab1c8b6 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__parser__reject_trailing_pipe.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__reject_trailing_pipe.snap @@ -13,7 +13,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 6, ), EndNode, diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__semicolon.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__semicolon.snap index 8cb4932..fed87e9 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__parser__semicolon.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__semicolon.snap @@ -13,7 +13,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 5, ), EndNode, @@ -37,7 +37,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 2, ), EndNode, diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__simple_command.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__simple_command.snap index 59e1d21..977e8f8 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__parser__simple_command.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__simple_command.snap @@ -13,7 +13,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 4, ), EndNode, @@ -25,7 +25,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 2, ), EndNode, diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__unicode.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__unicode.snap index 1feddf1..ac786d8 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__parser__unicode.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__unicode.snap @@ -13,7 +13,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 6, ), EndNode, diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__unterminated_double_quotes.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__unterminated_double_quotes.snap new file mode 100644 index 0000000..4727dc1 --- /dev/null +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__unterminated_double_quotes.snap @@ -0,0 +1,34 @@ +--- +source: crates/oyster_parser/tests/it/parser.rs +expression: actual +--- +[ + StartNode( + Pipeline, + ), + StartNode( + Command, + ), + StartNode( + Word, + ), + StartNode( + DQuotedString, + ), + NewLeaf( + DoubleQuote, + 1, + ), + NewLeaf( + PlainText, + 11, + ), + Error( + UnexpectedEof, + 0, + ), + EndNode, + EndNode, + EndNode, + EndNode, +] diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__word.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__word.snap index 59ba7d0..d05178c 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__parser__word.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__word.snap @@ -13,7 +13,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 4, ), EndNode, diff --git a/crates/oyster_parser/tests/it/snapshots/it__parser__word_with_escape.snap b/crates/oyster_parser/tests/it/snapshots/it__parser__word_with_escape.snap index f6b6ea6..dfc09d0 100644 --- a/crates/oyster_parser/tests/it/snapshots/it__parser__word_with_escape.snap +++ b/crates/oyster_parser/tests/it/snapshots/it__parser__word_with_escape.snap @@ -13,7 +13,7 @@ expression: actual Word, ), NewLeaf( - PlainWord, + PlainText, 5, ), NewLeaf( @@ -21,7 +21,7 @@ expression: actual 2, ), NewLeaf( - PlainWord, + PlainText, 4, ), EndNode,