Implement template strings (#17851)

This PR implements template strings (t-strings) in the parser and
formatter for Ruff.

Minimal changes necessary to compile were made in other parts of the code (e.g. ty, the linter, etc.). These will be covered properly in follow-up PRs.

crates/ruff_python_formatter/tests/snapshots/format@expression__binary.py.snap

@@ -437,6 +437,19 @@ f"hellooooooooooooooooooooooo \
         worlddddddddddddddddddddddddddddddddd" + (aaaaaaaaaaaaaaaaaaaaaaaaaa + bbbbbbbbbbbbbbbbbbb)
 aaaaaaaaaaa = f"hellooooooooooooooooooooooo \
         worlddddddddddddddddddddddddddddddddd" + (aaaaaaaaaaaaaaaaaaaaaaaaaa + bbbbbbbbbbbbbbbbbbb)
+
+# This t-string should be flattened
+xxxxxxxxxxxxxxxx = t"aaaaaaaaaaaaaaaaaaaaa {
+    expression } bbbbbbbbbbbbbbbbbbbbbbbb" + (
+    yyyyyyyyyyyyyy + zzzzzzzzzzz
+)
+
+# This is not a multiline t-string, but the expression is too long so it should be
+# wrapped in parentheses.
+t"hellooooooooooooooooooooooo \
+        worlddddddddddddddddddddddddddddddddd" + (aaaaaaaaaaaaaaaaaaaaaaaaaa + bbbbbbbbbbbbbbbbbbb)
+aaaaaaaaaaa = t"hellooooooooooooooooooooooo \
+        worlddddddddddddddddddddddddddddddddd" + (aaaaaaaaaaaaaaaaaaaaaaaaaa + bbbbbbbbbbbbbbbbbbb)
 ```
 
 ## Output
@@ -927,4 +940,22 @@ aaaaaaaaaaa = (
         worlddddddddddddddddddddddddddddddddd"
     + (aaaaaaaaaaaaaaaaaaaaaaaaaa + bbbbbbbbbbbbbbbbbbb)
 )
+
+# This t-string should be flattened
+xxxxxxxxxxxxxxxx = t"aaaaaaaaaaaaaaaaaaaaa {expression} bbbbbbbbbbbbbbbbbbbbbbbb" + (
+    yyyyyyyyyyyyyy + zzzzzzzzzzz
+)
+
+# This is not a multiline t-string, but the expression is too long so it should be
+# wrapped in parentheses.
+(
+    t"hellooooooooooooooooooooooo \
+        worlddddddddddddddddddddddddddddddddd"
+    + (aaaaaaaaaaaaaaaaaaaaaaaaaa + bbbbbbbbbbbbbbbbbbb)
+)
+aaaaaaaaaaa = (
+    t"hellooooooooooooooooooooooo \
+        worlddddddddddddddddddddddddddddddddd"
+    + (aaaaaaaaaaaaaaaaaaaaaaaaaa + bbbbbbbbbbbbbbbbbbb)
+)
 ```

crates/ruff_python_formatter/tests/snapshots/format@expression__join_implicit_concatenated_string.py.snap

@@ -106,6 +106,55 @@ f"{10 + len('bar')=}" f'no debug{10}' f"{10 + len('bar')=}"
 f"{10 + len('bar')=}" f'{10 + len("bar")=}'
 
 
+##############################################################################
+# T-strings
+##############################################################################
+
+# Escape `{` and `}` when merging a t-string with a string
+"a {not_a_variable}" t"b {10}" "c"
+
+# Join, and break expressions
+t"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa{
+expression
+}bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" t"cccccccccccccccccccc {20999}" "more"
+
+# Join, but don't break the expressions
+t"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa{expression}bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" t"cccccccccccccccccccc {20999}" "more"
+
+t"test{
+expression
+}flat" t"can be {
+joined
+} together"
+
+aaaaaaaaaaa = t"test{
+expression
+}flat" t"cean beeeeeeee {
+joined
+} eeeeeeeeeeeeeeeeeeeeeeeeeeeee" # inline
+
+
+t"single quoted '{x}'" t'double quoted "{x}"' # Same number of quotes => use preferred quote style
+t"single quote ' {x}" t'double quoted "{x}"'  # More double quotes => use single quotes
+t"single quoted '{x}'" t'double quote " {x}"'  # More single quotes => use double quotes
+
+# Different triple quoted strings
+t"{'''test'''}" t'{"""other"""}'
+
+# Now with inner quotes
+t"{'''test ' '''}" t'{"""other " """}'
+t"{some_where_nested('''test ' ''')}" t'{"""other " """ + "more"}'
+t"{b'''test ' '''}" t'{b"""other " """}'
+t"{t'''test ' '''}" t'{t"""other " """}'
+
+# debug expressions containing quotes
+t"{10 + len('bar')=}" t"{10 + len('bar')=}"
+t"{10 + len('bar')=}" t'no debug{10}' t"{10 + len('bar')=}"
+
+# We can't safely merge this pre Python 3.12 without altering the debug expression.
+t"{10 + len('bar')=}" t'{10 + len("bar")=}'
+
+
 ##############################################################################
 # Don't join raw strings
 ##############################################################################
@@ -116,6 +165,9 @@ R"a" "normal"
 f"test" fr"test"
 f"test" fR"test"
 
+t"test" tr"test"
+t"test" tR"test"
+
 
 ##############################################################################
 # Don't join triple quoted strings
@@ -125,9 +177,22 @@ f"test" fR"test"
 
 "single" f""""single"""
 
+"single" t""""single"""
+
 b"single" b"""triple"""
 
 
+##############################################################################
+# Don't join t-strings and f-strings
+##############################################################################
+
+t"{interp}" f"{expr}"
+
+f"{expr}" t"{interp}"
+
+f"{expr}" "string" t"{interp}"
+
+
 ##############################################################################
 # Join strings in with statements
 ##############################################################################
@@ -452,6 +517,50 @@ f"{10 + len('bar')=}no debug{10}{10 + len('bar')=}"
 f"{10 + len('bar')=}" f'{10 + len("bar")=}'
 
 
+##############################################################################
+# T-strings
+##############################################################################
+
+# Escape `{` and `}` when merging a t-string with a string
+t"a {{not_a_variable}}b {10}c"
+
+# Join, and break expressions
+t"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa{
+    expression
+}bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccc {20999}more"
+
+# Join, but don't break the expressions
+t"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa{expression}bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbcccccccccccccccccccc {20999}more"
+
+t"test{expression}flatcan be {joined} together"
+
+aaaaaaaaaaa = (
+    t"test{expression}flat"
+    t"cean beeeeeeee {joined} eeeeeeeeeeeeeeeeeeeeeeeeeeeee"
+)  # inline
+
+
+t"single quoted '{x}'double quoted \"{x}\""  # Same number of quotes => use preferred quote style
+t'single quote \' {x}double quoted "{x}"'  # More double quotes => use single quotes
+t"single quoted '{x}'double quote \" {x}\""  # More single quotes => use double quotes
+
+# Different triple quoted strings
+t"{'''test'''}{'''other'''}"
+
+# Now with inner quotes
+t"{'''test ' '''}{'''other " '''}"
+t"{some_where_nested('''test ' ''')}{'''other " ''' + 'more'}"
+t"{b'''test ' '''}{b'''other " '''}"
+t"{t'''test ' '''}{t'''other " '''}"
+
+# debug expressions containing quotes
+t"{10 + len('bar')=}{10 + len('bar')=}"
+t"{10 + len('bar')=}no debug{10}{10 + len('bar')=}"
+
+# We can't safely merge this pre Python 3.12 without altering the debug expression.
+t"{10 + len('bar')=}{10 + len("bar")=}"
+
+
 ##############################################################################
 # Don't join raw strings
 ##############################################################################
@@ -462,6 +571,9 @@ R"a" "normal"
 f"test" rf"test"
 f"test" Rf"test"
 
+t"test" rt"test"
+t"test" Rt"test"
+
 
 ##############################################################################
 # Don't join triple quoted strings
@@ -471,9 +583,22 @@ f"test" Rf"test"
 
 "single" f""""single"""
 
+"single" t""""single"""
+
 b"single" b"""triple"""
 
 
+##############################################################################
+# Don't join t-strings and f-strings
+##############################################################################
+
+t"{interp}" f"{expr}"
+
+f"{expr}" t"{interp}"
+
+f"{expr}" "string" t"{interp}"
+
+
 ##############################################################################
 # Join strings in with statements
 ##############################################################################
@@ -780,7 +905,7 @@ f"aaaaaaaaaaaaaaaa \
 ```diff
 --- Stable
 +++ Preview
-@@ -242,9 +242,12 @@
+@@ -302,9 +302,12 @@
  ##############################################################################
  # Use can_omit_optional_parentheses layout to avoid an instability where the formatter
  # picks the can_omit_optional_parentheses layout when the strings are joined.

crates/ruff_python_formatter/tests/snapshots/format@expression__join_implicit_concatenated_string_assignment.py.snap

@@ -299,6 +299,155 @@ aaaaa[aaaaaaaaaaa] = (
 )
 
 
+#############################################################
+# T-Strings
+#############################################################
+
+# Flatten and join the t-string
+aaaaaaaaaaa = t"test{
+expression}flat" t"cean beeeeeeee {joined} eeeeeeeeeeeeeeeee" # inline
+
+# Parenthesize the value and join it, inline the comment
+aaaaaaaaaaa = t"test{
+expression}flat" t"cean beeeeeeee {joined} eeeeeeeeeeeeeeeeeeeeeeeeeee" # inline
+
+# Parenthesize the t-string and keep it multiline because it doesn't fit on a single line including the comment
+aaaaaaaaaaa = t"test{
+expression
+}flat" t"cean beeeeeeee {
+joined
+} eeeeeeeeeeeeeeeeeeeeeeeeeeeee" # inline
+
+
+# The target splits because of a magic trailing comma
+# The string is joined and not parenthesized because it just fits into the line length (including comment).
+a[
+    aaaaaaa,
+    b,
+] = t"ccccc{
+expression}ccccccccccc" t"cccccccccccccccccccccccccccccccccccccccccc"  # comment
+
+
+# Same but starting with a joined string. They should both result in the same formatting.
+[
+    aaaaaaa,
+    b,
+] = t"ccccc{
+expression}ccccccccccccccccccccccccccccccccccccccccccccccccccccc"  # comment
+
+# The target splits because of the magic trailing comma
+# The string is **not** joined because it with the inlined comment exceeds the line length limit.
+a[
+    aaaaaaa,
+    b,
+] = t"ccccc{
+expression}cccccccccccccccccccc" t"cccccccccccccccccccccccccccccccccccccccccc"  # comment
+
+
+# The target should be flat
+# The string should be joined because it fits into the line length
+a[
+    aaaaaaa,
+    b
+] = (
+    t"ccccc{
+    expression}ccccccccccc" "cccccccccccccccccccccccc"  # comment
+)
+
+# Same but starting with a joined string. They should both result in the same formatting.
+a[
+    aaaaaaa,
+    b
+] = t"ccccc{
+expression}ccccccccccccccccccccccccccccccccccc"  # comment
+
+# The target should be flat
+# The string gets parenthesized because it, with the inlined comment, exceeds the line length limit.
+a[
+    aaaaaaa,
+    b
+] = t"ccccc{
+expression}ccccccccccc" "ccccccccccccccccccccccccccccccccccccccccccc"  # comment
+
+
+# Split an overlong target, but join the string if it fits
+a[
+    aaaaaaa,
+    b
+].bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb = (
+    t"ccccc{
+    expression}ccccccccccc" "cccccccccccccccccccccccccccccc"  # comment
+)
+
+# Split both if necessary and keep multiline
+a[
+    aaaaaaa,
+    b
+].bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb = (
+    t"ccccc{
+    expression}cccccccccccccccccccccccccccccccc" "ccccccccccccccccccccccccccccccc"  # comment
+)
+
+# Don't inline t-strings that contain expressions that are guaranteed to split, e.b. because of a magic trailing comma
+aaaaaaaaaaaaaaaaaa = t"testeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee{
+[a,]
+}" "moreeeeeeeeeeeeeeeeeeee" "test" # comment
+
+aaaaaaaaaaaaaaaaaa = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee{
+[a,]
+}" "moreeeeeeeeeeeeeeeeeeee" "test" # comment
+)
+
+aaaaa[aaaaaaaaaaa] = t"testeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee{
+[a,]
+}" "moreeeeeeeeeeeeeeeeeeee" "test" # comment
+
+aaaaa[aaaaaaaaaaa] = (t"testeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee{
+[a,]
+}" "moreeeeeeeeeeeeeeeeeeee" "test" # comment
+)
+
+# Don't inline t-strings that contain commented expressions
+aaaaaaaaaaaaaaaaaa = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{[
+        a  # comment
+    ]}" "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+aaaaa[aaaaaaaaaaa] = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{[
+        a  # comment
+    ]}" "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+# Don't inline t-strings with multiline debug expressions:
+aaaaaaaaaaaaaaaaaa = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{
+    a=}" "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+aaaaaaaaaaaaaaaaaa = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{a +
+    b=}" "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+aaaaaaaaaaaaaaaaaa = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{a
+    =}" "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+aaaaa[aaaaaaaaaaa] = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{
+    a=}" "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+aaaaa[aaaaaaaaaaa] = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{a
+    =}" "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+
 # Trailing last-part comments
 
 a = (
@@ -380,7 +529,8 @@ self._attr_unique_id = (
 return (
     f"Exception in {call_back_name} when handling msg on "
     f"'{msg.topic}': '{msg.payload}'"  # type: ignore[str-bytes-safe]
-)```
+)
+```
 
 ## Output
 ```python
@@ -704,6 +854,172 @@ aaaaa[aaaaaaaaaaa] = (
 )
 
 
+#############################################################
+# T-Strings
+#############################################################
+
+# Flatten and join the t-string
+aaaaaaaaaaa = t"test{expression}flatcean beeeeeeee {joined} eeeeeeeeeeeeeeeee"  # inline
+
+# Parenthesize the value and join it, inline the comment
+aaaaaaaaaaa = (
+    t"test{expression}flatcean beeeeeeee {joined} eeeeeeeeeeeeeeeeeeeeeeeeeee"  # inline
+)
+
+# Parenthesize the t-string and keep it multiline because it doesn't fit on a single line including the comment
+aaaaaaaaaaa = (
+    t"test{expression}flat"
+    t"cean beeeeeeee {joined} eeeeeeeeeeeeeeeeeeeeeeeeeeeee"
+)  # inline
+
+
+# The target splits because of a magic trailing comma
+# The string is joined and not parenthesized because it just fits into the line length (including comment).
+a[
+    aaaaaaa,
+    b,
+] = t"ccccc{expression}ccccccccccccccccccccccccccccccccccccccccccccccccccccc"  # comment
+
+
+# Same but starting with a joined string. They should both result in the same formatting.
+[
+    aaaaaaa,
+    b,
+] = t"ccccc{expression}ccccccccccccccccccccccccccccccccccccccccccccccccccccc"  # comment
+
+# The target splits because of the magic trailing comma
+# The string is **not** joined because it with the inlined comment exceeds the line length limit.
+a[
+    aaaaaaa,
+    b,
+] = (
+    t"ccccc{expression}cccccccccccccccccccc"
+    t"cccccccccccccccccccccccccccccccccccccccccc"
+)  # comment
+
+
+# The target should be flat
+# The string should be joined because it fits into the line length
+a[aaaaaaa, b] = t"ccccc{expression}ccccccccccccccccccccccccccccccccccc"  # comment
+
+# Same but starting with a joined string. They should both result in the same formatting.
+a[aaaaaaa, b] = t"ccccc{expression}ccccccccccccccccccccccccccccccccccc"  # comment
+
+# The target should be flat
+# The string gets parenthesized because it, with the inlined comment, exceeds the line length limit.
+a[aaaaaaa, b] = (
+    t"ccccc{expression}ccccccccccc"
+    "ccccccccccccccccccccccccccccccccccccccccccc"
+)  # comment
+
+
+# Split an overlong target, but join the string if it fits
+a[
+    aaaaaaa, b
+].bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb = (
+    t"ccccc{expression}ccccccccccccccccccccccccccccccccccccccccc"  # comment
+)
+
+# Split both if necessary and keep multiline
+a[
+    aaaaaaa, b
+].bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb = (
+    t"ccccc{expression}cccccccccccccccccccccccccccccccc"
+    "ccccccccccccccccccccccccccccccc"
+)  # comment
+
+# Don't inline t-strings that contain expressions that are guaranteed to split, e.b. because of a magic trailing comma
+aaaaaaaaaaaaaaaaaa = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee{
+        [
+            a,
+        ]
+    }"
+    "moreeeeeeeeeeeeeeeeeeee"
+    "test"
+)  # comment
+
+aaaaaaaaaaaaaaaaaa = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee{
+        [
+            a,
+        ]
+    }"
+    "moreeeeeeeeeeeeeeeeeeee"
+    "test"  # comment
+)
+
+aaaaa[aaaaaaaaaaa] = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee{
+        [
+            a,
+        ]
+    }"
+    "moreeeeeeeeeeeeeeeeeeee"
+    "test"
+)  # comment
+
+aaaaa[aaaaaaaaaaa] = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee{
+        [
+            a,
+        ]
+    }"
+    "moreeeeeeeeeeeeeeeeeeee"
+    "test"  # comment
+)
+
+# Don't inline t-strings that contain commented expressions
+aaaaaaaaaaaaaaaaaa = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{
+        [
+            a  # comment
+        ]
+    }"
+    "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+aaaaa[aaaaaaaaaaa] = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{
+        [
+            a  # comment
+        ]
+    }"
+    "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+# Don't inline t-strings with multiline debug expressions:
+aaaaaaaaaaaaaaaaaa = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{
+    a=}"
+    "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+aaaaaaaaaaaaaaaaaa = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{a +
+    b=}"
+    "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+aaaaaaaaaaaaaaaaaa = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{a
+    =}"
+    "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+aaaaa[aaaaaaaaaaa] = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{
+    a=}"
+    "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+aaaaa[aaaaaaaaaaa] = (
+    t"testeeeeeeeeeeeeeeeeeeeeeeeee{a
+    =}"
+    "moreeeeeeeeeeeeeeeeeetest"  # comment
+)
+
+
 # Trailing last-part comments
 
 a = (