Add an implicit concatenation flag to string and bytes constants (#6512)

## Summary

Per the discussion in
https://github.com/astral-sh/ruff/discussions/6183, this PR adds an
`implicit_concatenated` flag to the string and bytes constant variants.
It's not actually _used_ anywhere as of this PR, but it is covered by
the tests.

Specifically, we now use a struct for the string and bytes cases, along
with the `Expr::FString` node. That struct holds the value, plus the
flag:

```rust
#[derive(Clone, Debug, PartialEq, is_macro::Is)]
pub enum Constant {
    Str(StringConstant),
    Bytes(BytesConstant),
    ...
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct StringConstant {
    /// The string value as resolved by the parser (i.e., without quotes, or escape sequences, or
    /// implicit concatenations).
    pub value: String,
    /// Whether the string contains multiple string tokens that were implicitly concatenated.
    pub implicit_concatenated: bool,
}

impl Deref for StringConstant {
    type Target = str;
    fn deref(&self) -> &Self::Target {
        self.value.as_str()
    }
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub struct BytesConstant {
    /// The bytes value as resolved by the parser (i.e., without quotes, or escape sequences, or
    /// implicit concatenations).
    pub value: Vec<u8>,
    /// Whether the string contains multiple string tokens that were implicitly concatenated.
    pub implicit_concatenated: bool,
}

impl Deref for BytesConstant {
    type Target = [u8];
    fn deref(&self) -> &Self::Target {
        self.value.as_slice()
    }
}
```

## Test Plan

`cargo test`

crates/ruff_python_ast/src/all.rs

@@ -24,7 +24,7 @@ where
     fn add_to_names<'a>(elts: &'a [Expr], names: &mut Vec<&'a str>, flags: &mut DunderAllFlags) {
         for elt in elts {
             if let Expr::Constant(ast::ExprConstant {
-                value: Constant::Str(value),
+                value: Constant::Str(ast::StringConstant { value, .. }),
                 ..
             }) = elt
             {

crates/ruff_python_ast/src/comparable.rs

@@ -326,8 +326,18 @@ impl<'a> From<&'a ast::Constant> for ComparableConstant<'a> {
         match constant {
             ast::Constant::None => Self::None,
             ast::Constant::Bool(value) => Self::Bool(value),
-            ast::Constant::Str(value) => Self::Str(value),
-            ast::Constant::Bytes(value) => Self::Bytes(value),
+            ast::Constant::Str(ast::StringConstant {
+                value,
+                // Compare strings based on resolved value, not representation (i.e., ignore whether
+                // the string was implicitly concatenated).
+                implicit_concatenated: _,
+            }) => Self::Str(value),
+            ast::Constant::Bytes(ast::BytesConstant {
+                value,
+                // Compare bytes based on resolved value, not representation (i.e., ignore whether
+                // the bytes were implicitly concatenated).
+                implicit_concatenated: _,
+            }) => Self::Bytes(value),
             ast::Constant::Int(value) => Self::Int(value),
             ast::Constant::Float(value) => Self::Float(value.to_bits()),
             ast::Constant::Complex { real, imag } => Self::Complex {
@@ -865,11 +875,13 @@ impl<'a> From<&'a ast::Expr> for ComparableExpr<'a> {
                 debug_text: debug_text.as_ref(),
                 format_spec: format_spec.as_ref().map(Into::into),
             }),
-            ast::Expr::FString(ast::ExprFString { values, range: _ }) => {
-                Self::FString(ExprFString {
-                    values: values.iter().map(Into::into).collect(),
-                })
-            }
+            ast::Expr::FString(ast::ExprFString {
+                values,
+                implicit_concatenated: _,
+                range: _,
+            }) => Self::FString(ExprFString {
+                values: values.iter().map(Into::into).collect(),
+            }),
             ast::Expr::Constant(ast::ExprConstant {
                 value,
                 kind,

crates/ruff_python_ast/src/helpers.rs

@@ -123,10 +123,8 @@ where
         return true;
     }
     match expr {
-        Expr::BoolOp(ast::ExprBoolOp {
-            values, range: _, ..
-        })
-        | Expr::FString(ast::ExprFString { values, range: _ }) => {
+        Expr::BoolOp(ast::ExprBoolOp { values, .. })
+        | Expr::FString(ast::ExprFString { values, .. }) => {
             values.iter().any(|expr| any_over_expr(expr, func))
         }
         Expr::NamedExpr(ast::ExprNamedExpr {
@@ -1087,25 +1085,26 @@ impl Truthiness {
             Expr::Constant(ast::ExprConstant { value, .. }) => match value {
                 Constant::Bool(value) => Some(*value),
                 Constant::None => Some(false),
-                Constant::Str(string) => Some(!string.is_empty()),
+                Constant::Str(ast::StringConstant { value, .. }) => Some(!value.is_empty()),
                 Constant::Bytes(bytes) => Some(!bytes.is_empty()),
                 Constant::Int(int) => Some(!int.is_zero()),
                 Constant::Float(float) => Some(*float != 0.0),
                 Constant::Complex { real, imag } => Some(*real != 0.0 || *imag != 0.0),
                 Constant::Ellipsis => Some(true),
             },
-            Expr::FString(ast::ExprFString { values, range: _ }) => {
+            Expr::FString(ast::ExprFString { values, .. }) => {
                 if values.is_empty() {
                     Some(false)
                 } else if values.iter().any(|value| {
-                    let Expr::Constant(ast::ExprConstant {
-                        value: Constant::Str(string),
+                    if let Expr::Constant(ast::ExprConstant {
+                        value: Constant::Str(ast::StringConstant { value, .. }),
                         ..
                     }) = &value
-                    else {
-                        return false;
-                    };
-                    !string.is_empty()
+                    {
+                        !value.is_empty()
+                    } else {
+                        false
+                    }
                 }) {
                     Some(true)
                 } else {

crates/ruff_python_ast/src/node.rs

@@ -2676,7 +2676,11 @@ impl AstNode for ast::ExprFString {
     where
         V: PreorderVisitor<'a> + ?Sized,
     {
-        let ast::ExprFString { values, range: _ } = self;
+        let ast::ExprFString {
+            values,
+            implicit_concatenated: _,
+            range: _,
+        } = self;
 
         for expr in values {
             visitor.visit_expr(expr);

crates/ruff_python_ast/src/nodes.rs

@@ -881,6 +881,8 @@ pub struct DebugText {
 pub struct ExprFString {
     pub range: TextRange,
     pub values: Vec<Expr>,
+    /// Whether the f-string contains multiple string tokens that were implicitly concatenated.
+    pub implicit_concatenated: bool,
 }
 
 impl From<ExprFString> for Expr {
@@ -2463,8 +2465,8 @@ impl std::cmp::PartialEq<usize> for Int {
 pub enum Constant {
     None,
     Bool(bool),
-    Str(String),
-    Bytes(Vec<u8>),
+    Str(StringConstant),
+    Bytes(BytesConstant),
     Int(BigInt),
     Float(f64),
     Complex { real: f64, imag: f64 },
@@ -2472,38 +2474,68 @@ pub enum Constant {
 }
 
 impl Constant {
-    pub fn is_true(self) -> bool {
-        self.bool().is_some_and(|b| b)
-    }
-    pub fn is_false(self) -> bool {
-        self.bool().is_some_and(|b| !b)
-    }
-    pub fn complex(self) -> Option<(f64, f64)> {
+    /// Returns `true` if the constant is a string or bytes constant that contains multiple,
+    /// implicitly concatenated string tokens.
+    pub fn is_implicit_concatenated(&self) -> bool {
         match self {
-            Constant::Complex { real, imag } => Some((real, imag)),
-            _ => None,
+            Constant::Str(value) => value.implicit_concatenated,
+            Constant::Bytes(value) => value.implicit_concatenated,
+            _ => false,
         }
     }
 }
 
-impl From<String> for Constant {
-    fn from(s: String) -> Constant {
-        Self::Str(s)
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct StringConstant {
+    /// The string value as resolved by the parser (i.e., without quotes, or escape sequences, or
+    /// implicit concatenations).
+    pub value: String,
+    /// Whether the string contains multiple string tokens that were implicitly concatenated.
+    pub implicit_concatenated: bool,
+}
+
+impl Deref for StringConstant {
+    type Target = str;
+    fn deref(&self) -> &Self::Target {
+        self.value.as_str()
     }
 }
+
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct BytesConstant {
+    /// The bytes value as resolved by the parser (i.e., without quotes, or escape sequences, or
+    /// implicit concatenations).
+    pub value: Vec<u8>,
+    /// Whether the string contains multiple string tokens that were implicitly concatenated.
+    pub implicit_concatenated: bool,
+}
+
+impl Deref for BytesConstant {
+    type Target = [u8];
+    fn deref(&self) -> &Self::Target {
+        self.value.as_slice()
+    }
+}
+
 impl From<Vec<u8>> for Constant {
-    fn from(b: Vec<u8>) -> Constant {
-        Self::Bytes(b)
+    fn from(value: Vec<u8>) -> Constant {
+        Self::Bytes(BytesConstant {
+            value,
+            implicit_concatenated: false,
+        })
+    }
+}
+impl From<String> for Constant {
+    fn from(value: String) -> Constant {
+        Self::Str(StringConstant {
+            value,
+            implicit_concatenated: false,
+        })
     }
 }
 impl From<bool> for Constant {
-    fn from(b: bool) -> Constant {
-        Self::Bool(b)
-    }
-}
-impl From<BigInt> for Constant {
-    fn from(i: BigInt) -> Constant {
-        Self::Int(i)
+    fn from(value: bool) -> Constant {
+        Self::Bool(value)
     }
 }
 
@@ -3056,7 +3088,7 @@ mod size_assertions {
     assert_eq_size!(StmtClassDef, [u8; 104]);
     assert_eq_size!(StmtTry, [u8; 104]);
     assert_eq_size!(Expr, [u8; 80]);
-    assert_eq_size!(Constant, [u8; 32]);
+    assert_eq_size!(Constant, [u8; 40]);
     assert_eq_size!(Pattern, [u8; 96]);
     assert_eq_size!(Mod, [u8; 32]);
 }

crates/ruff_python_ast/src/relocate.rs

@@ -140,7 +140,7 @@ pub fn relocate_expr(expr: &mut Expr, location: TextRange) {
                 relocate_expr(expr, location);
             }
         }
-        Expr::FString(nodes::ExprFString { values, range }) => {
+        Expr::FString(nodes::ExprFString { values, range, .. }) => {
             *range = location;
             for expr in values {
                 relocate_expr(expr, location);

crates/ruff_python_ast/src/visitor.rs

@@ -476,7 +476,7 @@ pub fn walk_expr<'a, V: Visitor<'a> + ?Sized>(visitor: &mut V, expr: &'a Expr) {
                 visitor.visit_format_spec(expr);
             }
         }
-        Expr::FString(ast::ExprFString { values, range: _ }) => {
+        Expr::FString(ast::ExprFString { values, .. }) => {
             for expr in values {
                 visitor.visit_expr(expr);
             }