diff --git a/parser/src/fstring.rs b/parser/src/fstring.rs deleted file mode 100644 index 58e8f98..0000000 --- a/parser/src/fstring.rs +++ /dev/null @@ -1,479 +0,0 @@ -// We no longer need this file -use self::FStringErrorType::*; -use crate::{ - ast::{Constant, ConversionFlag, Expr, ExprKind, Location}, - error::{FStringError, FStringErrorType, ParseError}, - parser::parse_expression, -}; -use std::{iter, mem, str}; - -struct FStringParser<'a> { - chars: iter::Peekable>, - str_start: Location, - str_end: Location, -} - -impl<'a> FStringParser<'a> { - fn new(source: &'a str, str_start: Location, str_end: Location) -> Self { - Self { - chars: source.chars().peekable(), - str_start, - str_end, - } - } - - fn next_char(&mut self) -> Option { - self.chars.next() - } - - fn peek(&mut self) -> Option<&char> { - self.chars.peek() - } - - #[inline] - fn expr(&self, node: ExprKind) -> Expr { - Expr::new(self.str_start, self.str_end, node) - } - - fn parse_formatted_value(&mut self, nested: u8) -> Result, FStringErrorType> { - let mut expression = String::new(); - let mut spec = None; - let mut delims = Vec::new(); - let mut conversion = ConversionFlag::None; - let mut self_documenting = false; - let mut trailing_seq = String::new(); - - while let Some(ch) = self.next_char() { - match ch { - // can be integrated better with the remainign code, but as a starting point ok - // in general I would do here a tokenizing of the fstrings to omit this peeking. - '!' if self.peek() == Some(&'=') => { - expression.push_str("!="); - self.next_char(); - } - - '=' if self.peek() == Some(&'=') => { - expression.push_str("=="); - self.next_char(); - } - - '>' if self.peek() == Some(&'=') => { - expression.push_str(">="); - self.next_char(); - } - - '<' if self.peek() == Some(&'=') => { - expression.push_str("<="); - self.next_char(); - } - - '!' if delims.is_empty() && self.peek() != Some(&'=') => { - if expression.trim().is_empty() { - return Err(EmptyExpression); - } - - conversion = match self.next_char() { - Some('s') => ConversionFlag::Str, - Some('a') => ConversionFlag::Ascii, - Some('r') => ConversionFlag::Repr, - Some(_) => { - return Err(if expression.trim().is_empty() { - EmptyExpression - } else { - InvalidConversionFlag - }); - } - None => { - return Err(if expression.trim().is_empty() { - EmptyExpression - } else { - UnclosedLbrace - }); - } - }; - - if let Some(&peek) = self.peek() { - if peek != '}' && peek != ':' { - return Err(if expression.trim().is_empty() { - EmptyExpression - } else { - UnclosedLbrace - }); - } - } else { - return Err(if expression.trim().is_empty() { - EmptyExpression - } else { - UnclosedLbrace - }); - } - } - - // match a python 3.8 self documenting expression - // format '{' PYTHON_EXPRESSION '=' FORMAT_SPECIFIER? '}' - '=' if self.peek() != Some(&'=') && delims.is_empty() => { - self_documenting = true; - } - - ':' if delims.is_empty() => { - let parsed_spec = self.parse_spec(nested)?; - - spec = Some(Box::new(self.expr(ExprKind::JoinedStr { - values: parsed_spec, - }))); - } - '(' | '{' | '[' => { - expression.push(ch); - delims.push(ch); - } - ')' => { - let last_delim = delims.pop(); - match last_delim { - Some('(') => { - expression.push(ch); - } - Some(c) => { - return Err(MismatchedDelimiter(c, ')')); - } - None => { - return Err(Unmatched(')')); - } - } - } - ']' => { - let last_delim = delims.pop(); - match last_delim { - Some('[') => { - expression.push(ch); - } - Some(c) => { - return Err(MismatchedDelimiter(c, ']')); - } - None => { - return Err(Unmatched(']')); - } - } - } - '}' if !delims.is_empty() => { - let last_delim = delims.pop(); - match last_delim { - Some('{') => { - expression.push(ch); - } - Some(c) => return Err(MismatchedDelimiter(c, '}')), - None => {} - } - } - '}' => { - if expression.trim().is_empty() { - return Err(EmptyExpression); - } - - let ret = if !self_documenting { - vec![self.expr(ExprKind::FormattedValue { - value: Box::new( - parse_fstring_expr(&expression) - .map_err(|e| InvalidExpression(Box::new(e.error)))?, - ), - conversion: conversion as _, - format_spec: spec, - })] - } else { - vec![ - self.expr(ExprKind::Constant { - value: Constant::Str(expression.to_owned() + "="), - kind: None, - }), - self.expr(ExprKind::Constant { - value: trailing_seq.into(), - kind: None, - }), - self.expr(ExprKind::FormattedValue { - value: Box::new( - parse_fstring_expr(&expression) - .map_err(|e| InvalidExpression(Box::new(e.error)))?, - ), - conversion: (if conversion == ConversionFlag::None && spec.is_none() - { - ConversionFlag::Repr - } else { - conversion - }) as _, - format_spec: spec, - }), - ] - }; - return Ok(ret); - } - '"' | '\'' => { - expression.push(ch); - loop { - let Some(c) = self.next_char() else { - return Err(UnterminatedString); - }; - expression.push(c); - if c == ch { - break; - } - } - } - ' ' if self_documenting => { - trailing_seq.push(ch); - } - '\\' => return Err(ExpressionCannotInclude('\\')), - _ => { - if self_documenting { - return Err(UnclosedLbrace); - } - - expression.push(ch); - } - } - } - Err(if expression.trim().is_empty() { - EmptyExpression - } else { - UnclosedLbrace - }) - } - - fn parse_spec(&mut self, nested: u8) -> Result, FStringErrorType> { - let mut spec_constructor = Vec::new(); - let mut constant_piece = String::new(); - while let Some(&next) = self.peek() { - match next { - '{' => { - if !constant_piece.is_empty() { - spec_constructor.push(self.expr(ExprKind::Constant { - value: constant_piece.to_owned().into(), - kind: None, - })); - constant_piece.clear(); - } - let parsed_expr = self.parse(nested + 1)?; - spec_constructor.extend(parsed_expr); - continue; - } - '}' => { - break; - } - _ => { - constant_piece.push(next); - } - } - self.next_char(); - } - if !constant_piece.is_empty() { - spec_constructor.push(self.expr(ExprKind::Constant { - value: constant_piece.to_owned().into(), - kind: None, - })); - constant_piece.clear(); - } - Ok(spec_constructor) - } - - fn parse(&mut self, nested: u8) -> Result, FStringErrorType> { - if nested >= 2 { - return Err(ExpressionNestedTooDeeply); - } - - let mut content = String::new(); - let mut values = vec![]; - - while let Some(&ch) = self.peek() { - match ch { - '{' => { - self.next_char(); - if nested == 0 { - match self.peek() { - Some('{') => { - self.next_char(); - content.push('{'); - continue; - } - None => return Err(UnclosedLbrace), - _ => {} - } - } - if !content.is_empty() { - values.push(self.expr(ExprKind::Constant { - value: mem::take(&mut content).into(), - kind: None, - })); - } - - let parsed_values = self.parse_formatted_value(nested)?; - values.extend(parsed_values); - } - '}' => { - if nested > 0 { - break; - } - self.next_char(); - if let Some('}') = self.peek() { - self.next_char(); - content.push('}'); - } else { - return Err(SingleRbrace); - } - } - _ => { - content.push(ch); - self.next_char(); - } - } - } - - if !content.is_empty() { - values.push(self.expr(ExprKind::Constant { - value: content.into(), - kind: None, - })) - } - - Ok(values) - } -} - -fn parse_fstring_expr(source: &str) -> Result { - let fstring_body = format!("({source})"); - parse_expression(&fstring_body, "") -} - -/// Parse an fstring from a string, located at a certain position in the sourcecode. -/// In case of errors, we will get the location and the error returned. -pub fn parse_located_fstring( - source: &str, - start: Location, - end: Location, -) -> Result, FStringError> { - FStringParser::new(source, start, end) - .parse(0) - .map_err(|error| FStringError { - error, - location: start, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn parse_fstring(source: &str) -> Result, FStringErrorType> { - FStringParser::new(source, Location::default(), Location::default()).parse(0) - } - - #[test] - fn test_parse_fstring() { - let source = "{a}{ b }{{foo}}"; - let parse_ast = parse_fstring(source).unwrap(); - - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_fstring_nested_spec() { - let source = "{foo:{spec}}"; - let parse_ast = parse_fstring(source).unwrap(); - - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_fstring_not_nested_spec() { - let source = "{foo:spec}"; - let parse_ast = parse_fstring(source).unwrap(); - - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_empty_fstring() { - insta::assert_debug_snapshot!(parse_fstring("").unwrap()); - } - - #[test] - fn test_fstring_parse_selfdocumenting_base() { - let src = "{user=}"; - let parse_ast = parse_fstring(src).unwrap(); - - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_fstring_parse_selfdocumenting_base_more() { - let src = "mix {user=} with text and {second=}"; - let parse_ast = parse_fstring(src).unwrap(); - - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_fstring_parse_selfdocumenting_format() { - let src = "{user=:>10}"; - let parse_ast = parse_fstring(src).unwrap(); - - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_invalid_fstring() { - assert_eq!(parse_fstring("{5!a"), Err(UnclosedLbrace)); - assert_eq!(parse_fstring("{5!a1}"), Err(UnclosedLbrace)); - assert_eq!(parse_fstring("{5!"), Err(UnclosedLbrace)); - assert_eq!(parse_fstring("abc{!a 'cat'}"), Err(EmptyExpression)); - assert_eq!(parse_fstring("{!a"), Err(EmptyExpression)); - assert_eq!(parse_fstring("{ !a}"), Err(EmptyExpression)); - - assert_eq!(parse_fstring("{5!}"), Err(InvalidConversionFlag)); - assert_eq!(parse_fstring("{5!x}"), Err(InvalidConversionFlag)); - - assert_eq!(parse_fstring("{a:{a:{b}}}"), Err(ExpressionNestedTooDeeply)); - - assert_eq!(parse_fstring("{a:b}}"), Err(SingleRbrace)); - assert_eq!(parse_fstring("}"), Err(SingleRbrace)); - assert_eq!(parse_fstring("{a:{b}"), Err(UnclosedLbrace)); - assert_eq!(parse_fstring("{"), Err(UnclosedLbrace)); - - assert_eq!(parse_fstring("{}"), Err(EmptyExpression)); - - // TODO: check for InvalidExpression enum? - assert!(parse_fstring("{class}").is_err()); - } - - #[test] - fn test_parse_fstring_not_equals() { - let source = "{1 != 2}"; - let parse_ast = parse_fstring(source).unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_fstring_equals() { - let source = "{42 == 42}"; - let parse_ast = parse_fstring(source).unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_fstring_selfdoc_prec_space() { - let source = "{x =}"; - let parse_ast = parse_fstring(source).unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_fstring_selfdoc_trailing_space() { - let source = "{x= }"; - let parse_ast = parse_fstring(source).unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } - - #[test] - fn test_parse_fstring_yield_expr() { - let source = "{yield}"; - let parse_ast = parse_fstring(source).unwrap(); - insta::assert_debug_snapshot!(parse_ast); - } -} diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap similarity index 80% rename from parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap rename to parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap index 0941873..de77be8 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base.snap +++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base.snap @@ -1,17 +1,17 @@ --- -source: compiler/parser/src/fstring.rs +source: compiler/parser/src/string_parser.rs expression: parse_ast --- [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 10, }, ), custom: (), @@ -24,13 +24,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 10, }, ), custom: (), @@ -43,13 +43,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 10, }, ), custom: (), @@ -57,12 +57,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 1, + column: 3, }, end_location: Some( Location { row: 1, - column: 5, + column: 7, }, ), custom: (), diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap similarity index 81% rename from parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap rename to parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap index 92a05f2..e7f5bff 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_base_more.snap +++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_base_more.snap @@ -1,17 +1,17 @@ --- -source: compiler/parser/src/fstring.rs +source: compiler/parser/src/string_parser.rs expression: parse_ast --- [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 38, }, ), custom: (), @@ -24,13 +24,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 38, }, ), custom: (), @@ -43,13 +43,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 38, }, ), custom: (), @@ -62,13 +62,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 38, }, ), custom: (), @@ -76,12 +76,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 1, + column: 7, }, end_location: Some( Location { row: 1, - column: 5, + column: 11, }, ), custom: (), @@ -96,13 +96,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 38, }, ), custom: (), @@ -115,13 +115,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 38, }, ), custom: (), @@ -134,13 +134,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 38, }, ), custom: (), @@ -153,13 +153,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 38, }, ), custom: (), @@ -167,12 +167,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 1, + column: 29, }, end_location: Some( Location { row: 1, - column: 7, + column: 35, }, ), custom: (), diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap similarity index 80% rename from parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap rename to parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap index ab5ea8c..5efa6f6 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__fstring_parse_selfdocumenting_format.snap +++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__fstring_parse_selfdocumenting_format.snap @@ -1,17 +1,17 @@ --- -source: compiler/parser/src/fstring.rs +source: compiler/parser/src/string_parser.rs expression: parse_ast --- [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 14, }, ), custom: (), @@ -24,13 +24,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 14, }, ), custom: (), @@ -43,13 +43,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 14, }, ), custom: (), @@ -57,12 +57,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 1, + column: 3, }, end_location: Some( Location { row: 1, - column: 5, + column: 7, }, ), custom: (), @@ -75,13 +75,13 @@ expression: parse_ast format_spec: Some( Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 14, }, ), custom: (), @@ -89,13 +89,13 @@ expression: parse_ast values: [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 14, }, ), custom: (), diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_empty_fstring.snap similarity index 54% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_empty_fstring.snap index e561a7f..fbc08b4 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_empty_fstring.snap +++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_empty_fstring.snap @@ -1,5 +1,5 @@ --- -source: compiler/parser/src/fstring.rs +source: compiler/parser/src/string_parser.rs expression: "parse_fstring(\"\").unwrap()" --- [] diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring.snap similarity index 81% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring.snap index 92428aa..32b63f9 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring.snap +++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring.snap @@ -1,17 +1,17 @@ --- -source: compiler/parser/src/fstring.rs +source: compiler/parser/src/string_parser.rs expression: parse_ast --- [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 18, }, ), custom: (), @@ -19,12 +19,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 1, + column: 3, }, end_location: Some( Location { row: 1, - column: 2, + column: 4, }, ), custom: (), @@ -39,13 +39,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 18, }, ), custom: (), @@ -53,12 +53,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 2, + column: 7, }, end_location: Some( Location { row: 1, - column: 3, + column: 8, }, ), custom: (), @@ -73,13 +73,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 18, }, ), custom: (), diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_equals.snap similarity index 85% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_equals.snap index fbbe849..e9d8120 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_equals.snap +++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_equals.snap @@ -1,17 +1,17 @@ --- -source: compiler/parser/src/fstring.rs +source: compiler/parser/src/string_parser.rs expression: parse_ast --- [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 13, }, ), custom: (), @@ -19,12 +19,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 1, + column: 3, }, end_location: Some( Location { row: 1, - column: 9, + column: 11, }, ), custom: (), @@ -32,12 +32,12 @@ expression: parse_ast left: Located { location: Location { row: 1, - column: 1, + column: 3, }, end_location: Some( Location { row: 1, - column: 3, + column: 5, }, ), custom: (), @@ -55,12 +55,12 @@ expression: parse_ast Located { location: Location { row: 1, - column: 7, + column: 9, }, end_location: Some( Location { row: 1, - column: 9, + column: 11, }, ), custom: (), diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap similarity index 82% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap index ce41337..268a16e 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_nested_spec.snap +++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_nested_spec.snap @@ -1,17 +1,17 @@ --- -source: compiler/parser/src/fstring.rs +source: compiler/parser/src/string_parser.rs expression: parse_ast --- [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 15, }, ), custom: (), @@ -19,12 +19,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 1, + column: 3, }, end_location: Some( Location { row: 1, - column: 4, + column: 6, }, ), custom: (), @@ -37,13 +37,13 @@ expression: parse_ast format_spec: Some( Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 15, }, ), custom: (), @@ -51,13 +51,13 @@ expression: parse_ast values: [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 15, }, ), custom: (), @@ -65,12 +65,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 1, + column: 8, }, end_location: Some( Location { row: 1, - column: 5, + column: 12, }, ), custom: (), diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap similarity index 85% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap index 8f66749..aaa2ad1 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_equals.snap +++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_equals.snap @@ -1,17 +1,17 @@ --- -source: compiler/parser/src/fstring.rs +source: compiler/parser/src/string_parser.rs expression: parse_ast --- [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 11, }, ), custom: (), @@ -19,12 +19,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 1, + column: 3, }, end_location: Some( Location { row: 1, - column: 7, + column: 9, }, ), custom: (), @@ -32,12 +32,12 @@ expression: parse_ast left: Located { location: Location { row: 1, - column: 1, + column: 3, }, end_location: Some( Location { row: 1, - column: 2, + column: 4, }, ), custom: (), @@ -55,12 +55,12 @@ expression: parse_ast Located { location: Location { row: 1, - column: 6, + column: 8, }, end_location: Some( Location { row: 1, - column: 7, + column: 9, }, ), custom: (), diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap similarity index 80% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap index b76f7aa..0d1107e 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_not_nested_spec.snap +++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_not_nested_spec.snap @@ -1,17 +1,17 @@ --- -source: compiler/parser/src/fstring.rs +source: compiler/parser/src/string_parser.rs expression: parse_ast --- [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 13, }, ), custom: (), @@ -19,12 +19,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 1, + column: 3, }, end_location: Some( Location { row: 1, - column: 4, + column: 6, }, ), custom: (), @@ -37,13 +37,13 @@ expression: parse_ast format_spec: Some( Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 13, }, ), custom: (), @@ -51,13 +51,13 @@ expression: parse_ast values: [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 13, }, ), custom: (), diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap similarity index 80% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap index c40f7e3..7851495 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_prec_space.snap +++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_prec_space.snap @@ -1,17 +1,17 @@ --- -source: compiler/parser/src/fstring.rs +source: compiler/parser/src/string_parser.rs expression: parse_ast --- [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 10, }, ), custom: (), @@ -24,13 +24,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 10, }, ), custom: (), @@ -43,13 +43,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 10, }, ), custom: (), @@ -57,12 +57,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 1, + column: 3, }, end_location: Some( Location { row: 1, - column: 2, + column: 4, }, ), custom: (), diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap similarity index 80% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap index d73746a..621e499 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_selfdoc_trailing_space.snap +++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_selfdoc_trailing_space.snap @@ -1,17 +1,17 @@ --- -source: compiler/parser/src/fstring.rs +source: compiler/parser/src/string_parser.rs expression: parse_ast --- [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 10, }, ), custom: (), @@ -24,13 +24,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 10, }, ), custom: (), @@ -43,13 +43,13 @@ expression: parse_ast }, Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 10, }, ), custom: (), @@ -57,12 +57,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 1, + column: 3, }, end_location: Some( Location { row: 1, - column: 2, + column: 4, }, ), custom: (), diff --git a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap similarity index 79% rename from parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap rename to parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap index 767ed51..62a0cf4 100644 --- a/parser/src/snapshots/rustpython_parser__fstring__tests__parse_fstring_yield_expr.snap +++ b/parser/src/snapshots/rustpython_parser__string_parser__tests__parse_fstring_yield_expr.snap @@ -1,17 +1,17 @@ --- -source: compiler/parser/src/fstring.rs +source: compiler/parser/src/string_parser.rs expression: parse_ast --- [ Located { location: Location { - row: 0, + row: 1, column: 0, }, end_location: Some( Location { - row: 0, - column: 0, + row: 1, + column: 10, }, ), custom: (), @@ -19,12 +19,12 @@ expression: parse_ast value: Located { location: Location { row: 1, - column: 1, + column: 3, }, end_location: Some( Location { row: 1, - column: 6, + column: 8, }, ), custom: (), diff --git a/parser/src/string_parser.rs b/parser/src/string_parser.rs index 610951a..1889b3f 100644 --- a/parser/src/string_parser.rs +++ b/parser/src/string_parser.rs @@ -36,6 +36,7 @@ impl<'a> StringParser<'a> { } } + #[inline] fn next_char(&mut self) -> Option { let Some(c) = self.chars.next() else { return None @@ -48,10 +49,12 @@ impl<'a> StringParser<'a> { Some(c) } + #[inline] fn peek(&mut self) -> Option<&char> { self.chars.peek() } + #[inline] fn get_pos(&self) -> Location { self.location } @@ -560,3 +563,136 @@ pub fn parse_string( ) -> Result, LexicalError> { StringParser::new(source, kind, triple_quoted, start, end).parse() } + +#[cfg(test)] +mod tests { + use super::*; + + fn parse_fstring(source: &str) -> Result, FStringErrorType> { + StringParser::new( + source, + StringKind::FString, + false, + Location::new(1, 0), + Location::new(1, source.len() + 3), // 3 for prefix and quotes + ) + .parse() + .map_err(|e| match e.error { + LexicalErrorType::FStringError(e) => e, + e => unreachable!("Unexpected error type {:?}", e), + }) + } + + #[test] + fn test_parse_fstring() { + let source = "{a}{ b }{{foo}}"; + let parse_ast = parse_fstring(source).unwrap(); + + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_fstring_nested_spec() { + let source = "{foo:{spec}}"; + let parse_ast = parse_fstring(source).unwrap(); + + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_fstring_not_nested_spec() { + let source = "{foo:spec}"; + let parse_ast = parse_fstring(source).unwrap(); + + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_empty_fstring() { + insta::assert_debug_snapshot!(parse_fstring("").unwrap()); + } + + #[test] + fn test_fstring_parse_selfdocumenting_base() { + let src = "{user=}"; + let parse_ast = parse_fstring(src).unwrap(); + + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_fstring_parse_selfdocumenting_base_more() { + let src = "mix {user=} with text and {second=}"; + let parse_ast = parse_fstring(src).unwrap(); + + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_fstring_parse_selfdocumenting_format() { + let src = "{user=:>10}"; + let parse_ast = parse_fstring(src).unwrap(); + + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_invalid_fstring() { + assert_eq!(parse_fstring("{5!a"), Err(UnclosedLbrace)); + assert_eq!(parse_fstring("{5!a1}"), Err(UnclosedLbrace)); + assert_eq!(parse_fstring("{5!"), Err(UnclosedLbrace)); + assert_eq!(parse_fstring("abc{!a 'cat'}"), Err(EmptyExpression)); + assert_eq!(parse_fstring("{!a"), Err(EmptyExpression)); + assert_eq!(parse_fstring("{ !a}"), Err(EmptyExpression)); + + assert_eq!(parse_fstring("{5!}"), Err(InvalidConversionFlag)); + assert_eq!(parse_fstring("{5!x}"), Err(InvalidConversionFlag)); + + assert_eq!(parse_fstring("{a:{a:{b}}}"), Err(ExpressionNestedTooDeeply)); + + assert_eq!(parse_fstring("{a:b}}"), Err(SingleRbrace)); + assert_eq!(parse_fstring("}"), Err(SingleRbrace)); + assert_eq!(parse_fstring("{a:{b}"), Err(UnclosedLbrace)); + assert_eq!(parse_fstring("{"), Err(UnclosedLbrace)); + + assert_eq!(parse_fstring("{}"), Err(EmptyExpression)); + + // TODO: check for InvalidExpression enum? + assert!(parse_fstring("{class}").is_err()); + } + + #[test] + fn test_parse_fstring_not_equals() { + let source = "{1 != 2}"; + let parse_ast = parse_fstring(source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_fstring_equals() { + let source = "{42 == 42}"; + let parse_ast = parse_fstring(source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_fstring_selfdoc_prec_space() { + let source = "{x =}"; + let parse_ast = parse_fstring(source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_fstring_selfdoc_trailing_space() { + let source = "{x= }"; + let parse_ast = parse_fstring(source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } + + #[test] + fn test_parse_fstring_yield_expr() { + let source = "{yield}"; + let parse_ast = parse_fstring(source).unwrap(); + insta::assert_debug_snapshot!(parse_ast); + } +}