Implement template strings (#17851)

This PR implements template strings (t-strings) in the parser and
formatter for Ruff.

Minimal changes necessary to compile were made in other parts of the code (e.g. ty, the linter, etc.). These will be covered properly in follow-up PRs.

crates/ruff_python_ast_integration_tests/tests/comparable.rs

@@ -1,19 +1,36 @@
 use ruff_python_ast::comparable::ComparableExpr;
 use ruff_python_parser::{ParseError, parse_expression};
 
+#[track_caller]
+fn assert_comparable(left: &str, right: &str) -> Result<(), ParseError> {
+    let left_parsed = parse_expression(left)?;
+    let right_parsed = parse_expression(right)?;
+
+    let left_compr = ComparableExpr::from(left_parsed.expr());
+    let right_compr = ComparableExpr::from(right_parsed.expr());
+
+    assert_eq!(left_compr, right_compr);
+    Ok(())
+}
+
+#[track_caller]
+fn assert_noncomparable(left: &str, right: &str) -> Result<(), ParseError> {
+    let left_parsed = parse_expression(left)?;
+    let right_parsed = parse_expression(right)?;
+
+    let left_compr = ComparableExpr::from(left_parsed.expr());
+    let right_compr = ComparableExpr::from(right_parsed.expr());
+
+    assert_ne!(left_compr, right_compr);
+    Ok(())
+}
+
 #[test]
 fn concatenated_strings_compare_equal() -> Result<(), ParseError> {
     let split_contents = r#"'a' 'b' r'\n raw'"#;
     let value_contents = r#"'ab\\n raw'"#;
 
-    let split_parsed = parse_expression(split_contents)?;
-    let value_parsed = parse_expression(value_contents)?;
-
-    let split_compr = ComparableExpr::from(split_parsed.expr());
-    let value_compr = ComparableExpr::from(value_parsed.expr());
-
-    assert_eq!(split_compr, value_compr);
-    Ok(())
+    assert_comparable(split_contents, value_contents)
 }
 
 #[test]
@@ -21,14 +38,7 @@ fn concatenated_bytes_compare_equal() -> Result<(), ParseError> {
     let split_contents = r#"b'a' b'b'"#;
     let value_contents = r#"b'ab'"#;
 
-    let split_parsed = parse_expression(split_contents)?;
-    let value_parsed = parse_expression(value_contents)?;
-
-    let split_compr = ComparableExpr::from(split_parsed.expr());
-    let value_compr = ComparableExpr::from(value_parsed.expr());
-
-    assert_eq!(split_compr, value_compr);
-    Ok(())
+    assert_comparable(split_contents, value_contents)
 }
 
 #[test]
@@ -36,12 +46,45 @@ fn concatenated_fstrings_compare_equal() -> Result<(), ParseError> {
     let split_contents = r#"f"{foo!r} this" r"\n raw" f" and {bar!s} that""#;
     let value_contents = r#"f"{foo!r} this\\n raw and {bar!s} that""#;
 
-    let split_parsed = parse_expression(split_contents)?;
-    let value_parsed = parse_expression(value_contents)?;
-
-    let split_compr = ComparableExpr::from(split_parsed.expr());
-    let value_compr = ComparableExpr::from(value_parsed.expr());
-
-    assert_eq!(split_compr, value_compr);
-    Ok(())
+    assert_comparable(split_contents, value_contents)
+}
+
+#[test]
+fn concatenated_tstrings_compare_equal() -> Result<(), ParseError> {
+    let split_contents = r#"t"{foo!r} this" r"\n raw" t" and {bar!s} that""#;
+    let value_contents = r#"t"{foo!r} this\\n raw and {bar!s} that""#;
+
+    assert_comparable(split_contents, value_contents)
+}
+
+#[test]
+fn concatenated_f_and_t_strings_interwoven_compare_equal() -> Result<(), ParseError> {
+    let split_contents = r#"f"{foo} this " t"{bar}" "baz""#;
+    let value_contents = r#"f"{foo}" t" this {bar}" "baz""#;
+
+    assert_comparable(split_contents, value_contents)
+}
+
+#[test]
+fn concatenated_f_and_t_strings_compare_unequal_when_swapped() -> Result<(), ParseError> {
+    let f_then_t_contents = r#"f"{foo!r} this" r"\n raw" t" and {bar!s} that""#;
+    let t_then_f_contents = r#"t"{foo!r} this" r"\n raw" f" and {bar!s} that""#;
+
+    assert_noncomparable(f_then_t_contents, t_then_f_contents)
+}
+
+#[test]
+fn t_strings_literal_order_matters_compare_unequal() -> Result<(), ParseError> {
+    let interp_then_literal_contents = r#"t"{foo}bar""#;
+    let literal_then_interp_contents = r#"t"bar{foo}""#;
+
+    assert_noncomparable(interp_then_literal_contents, literal_then_interp_contents)
+}
+
+#[test]
+fn t_strings_empty_concat_equal() -> Result<(), ParseError> {
+    let empty_literal = r#""" t"hey{foo}""#;
+    let empty_f_string = r#"f""t"hey{foo}""#;
+
+    assert_comparable(empty_literal, empty_f_string)
 }

crates/ruff_python_ast_integration_tests/tests/snapshots/source_order__f_strings.snap

@@ -1,18 +1,17 @@
 ---
 source: crates/ruff_python_ast_integration_tests/tests/source_order.rs
 expression: trace
-snapshot_kind: text
 ---
 - ModModule
   - StmtExpr
     - ExprFString
       - StringLiteral
       - FString
-        - FStringLiteralElement
-        - FStringExpressionElement
+        - InterpolatedStringLiteralElement
+        - InterpolatedElement
           - ExprName
-          - FStringLiteralElement
-          - FStringExpressionElement
+          - InterpolatedStringLiteralElement
+          - InterpolatedElement
             - ExprName
-          - FStringLiteralElement
-        - FStringLiteralElement
+          - InterpolatedStringLiteralElement
+        - InterpolatedStringLiteralElement

crates/ruff_python_ast_integration_tests/tests/snapshots/source_order__t_strings.snap

@@ -0,0 +1,17 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/source_order.rs
+expression: trace
+---
+- ModModule
+  - StmtExpr
+    - ExprTString
+      - StringLiteral
+      - TString
+        - InterpolatedStringLiteralElement
+        - InterpolatedElement
+          - ExprName
+          - InterpolatedStringLiteralElement
+          - InterpolatedElement
+            - ExprName
+          - InterpolatedStringLiteralElement
+        - InterpolatedStringLiteralElement

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__f_strings.snap

@@ -1,17 +1,16 @@
 ---
 source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
 expression: trace
-snapshot_kind: text
 ---
 - StmtExpr
   - ExprFString
     - StringLiteral
     - FString
-      - FStringLiteralElement
-      - FStringExpressionElement
+      - InterpolatedStringLiteralElement
+      - InterpolatedElement
         - ExprName
-        - FStringLiteralElement
-        - FStringExpressionElement
+        - InterpolatedStringLiteralElement
+        - InterpolatedElement
           - ExprName
-        - FStringLiteralElement
-      - FStringLiteralElement
+        - InterpolatedStringLiteralElement
+      - InterpolatedStringLiteralElement

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__t_strings.snap

@@ -0,0 +1,16 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtExpr
+  - ExprTString
+    - StringLiteral
+    - TString
+      - InterpolatedStringLiteralElement
+      - InterpolatedElement
+        - ExprName
+        - InterpolatedStringLiteralElement
+        - InterpolatedElement
+          - ExprName
+        - InterpolatedStringLiteralElement
+      - InterpolatedStringLiteralElement

crates/ruff_python_ast_integration_tests/tests/source_order.rs

@@ -146,6 +146,15 @@ fn f_strings() {
     assert_snapshot!(trace);
 }
 
+#[test]
+fn t_strings() {
+    let source = r"'pre' t'foo {bar:.{x}f} baz'";
+
+    let trace = trace_source_order_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
 fn trace_source_order_visitation(source: &str) -> String {
     let parsed = parse(source, ParseOptions::from(Mode::Module)).unwrap();
 

crates/ruff_python_ast_integration_tests/tests/visitor.rs

@@ -4,13 +4,14 @@ use insta::assert_snapshot;
 
 use ruff_python_ast::visitor::{
     Visitor, walk_alias, walk_bytes_literal, walk_comprehension, walk_except_handler, walk_expr,
-    walk_f_string, walk_f_string_element, walk_keyword, walk_match_case, walk_parameter,
-    walk_parameters, walk_pattern, walk_stmt, walk_string_literal, walk_type_param, walk_with_item,
+    walk_f_string, walk_interpolated_string_element, walk_keyword, walk_match_case, walk_parameter,
+    walk_parameters, walk_pattern, walk_stmt, walk_string_literal, walk_t_string, walk_type_param,
+    walk_with_item,
 };
 use ruff_python_ast::{
     self as ast, Alias, AnyNodeRef, BoolOp, BytesLiteral, CmpOp, Comprehension, ExceptHandler,
-    Expr, FString, FStringElement, Keyword, MatchCase, Operator, Parameter, Parameters, Pattern,
-    Stmt, StringLiteral, TypeParam, UnaryOp, WithItem,
+    Expr, FString, InterpolatedStringElement, Keyword, MatchCase, Operator, Parameter, Parameters,
+    Pattern, Stmt, StringLiteral, TString, TypeParam, UnaryOp, WithItem,
 };
 use ruff_python_parser::{Mode, ParseOptions, parse};
 
@@ -154,6 +155,15 @@ fn f_strings() {
     assert_snapshot!(trace);
 }
 
+#[test]
+fn t_strings() {
+    let source = r"'pre' t'foo {bar:.{x}f} baz'";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
 fn trace_visitation(source: &str) -> String {
     let parsed = parse(source, ParseOptions::from(Mode::Module)).unwrap();
 
@@ -318,9 +328,15 @@ impl Visitor<'_> for RecordVisitor {
         self.exit_node();
     }
 
-    fn visit_f_string_element(&mut self, f_string_element: &FStringElement) {
+    fn visit_interpolated_string_element(&mut self, f_string_element: &InterpolatedStringElement) {
         self.enter_node(f_string_element);
-        walk_f_string_element(self, f_string_element);
+        walk_interpolated_string_element(self, f_string_element);
+        self.exit_node();
+    }
+
+    fn visit_t_string(&mut self, t_string: &TString) {
+        self.enter_node(t_string);
+        walk_t_string(self, t_string);
         self.exit_node();
     }
 }