Fix FormattedValue location

parser/src/token.rs

@@ -1,17 +1,29 @@
 //! Different token definitions.
 //! Loosely based on token.h from CPython source:
 use num_bigint::BigInt;
-use std::fmt::{self, Write};
+use std::fmt;
 
 /// Python source code can be tokenized in a sequence of these tokens.
 #[derive(Clone, Debug, PartialEq)]
 pub enum Tok {
-    Name { name: String },
-    Int { value: BigInt },
-    Float { value: f64 },
-    Complex { real: f64, imag: f64 },
-    String { value: String, kind: StringKind },
-    Bytes { value: Vec<u8> },
+    Name {
+        name: String,
+    },
+    Int {
+        value: BigInt,
+    },
+    Float {
+        value: f64,
+    },
+    Complex {
+        real: f64,
+        imag: f64,
+    },
+    String {
+        value: String,
+        kind: StringKind,
+        triple_quoted: bool,
+    },
     Newline,
     Indent,
     Dedent,
@@ -107,13 +119,6 @@ pub enum Tok {
     Yield,
 }
 
-#[derive(PartialEq, Eq, Debug, Clone)]
-pub enum StringKind {
-    Normal,
-    F,
-    U,
-}
-
 impl fmt::Display for Tok {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         use Tok::*;
@@ -122,26 +127,13 @@ impl fmt::Display for Tok {
             Int { value } => write!(f, "'{value}'"),
             Float { value } => write!(f, "'{value}'"),
             Complex { real, imag } => write!(f, "{real}j{imag}"),
-            String { value, kind } => {
-                match kind {
-                    StringKind::F => f.write_str("f")?,
-                    StringKind::U => f.write_str("u")?,
-                    StringKind::Normal => {}
-                }
-                write!(f, "{value:?}")
-            }
-            Bytes { value } => {
-                write!(f, "b\"")?;
-                for i in value {
-                    match i {
-                        9 => f.write_str("\\t")?,
-                        10 => f.write_str("\\n")?,
-                        13 => f.write_str("\\r")?,
-                        32..=126 => f.write_char(*i as char)?,
-                        _ => write!(f, "\\x{i:02x}")?,
-                    }
-                }
-                f.write_str("\"")
+            String {
+                value,
+                kind,
+                triple_quoted,
+            } => {
+                let quotes = "\"".repeat(if *triple_quoted { 3 } else { 1 });
+                write!(f, "{kind}{quotes}{value}{quotes}")
             }
             Newline => f.write_str("Newline"),
             Indent => f.write_str("Indent"),
@@ -236,3 +228,50 @@ impl fmt::Display for Tok {
         }
     }
 }
+
+#[derive(PartialEq, Eq, Debug, Clone)]
+pub enum StringKind {
+    String,
+    FString,
+    Bytes,
+    RawString,
+    RawFString,
+    RawBytes,
+    Unicode,
+}
+
+impl fmt::Display for StringKind {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        use StringKind::*;
+        match self {
+            String => f.write_str(""),
+            FString => f.write_str("f"),
+            Bytes => f.write_str("b"),
+            RawString => f.write_str("r"),
+            RawFString => f.write_str("rf"),
+            RawBytes => f.write_str("rb"),
+            Unicode => f.write_str("u"),
+        }
+    }
+}
+
+impl StringKind {
+    pub fn is_raw(&self) -> bool {
+        use StringKind::{RawBytes, RawFString, RawString};
+        matches!(self, RawString | RawFString | RawBytes)
+    }
+
+    pub fn is_fstring(&self) -> bool {
+        use StringKind::{FString, RawFString};
+        matches!(self, FString | RawFString)
+    }
+
+    pub fn is_bytes(&self) -> bool {
+        use StringKind::{Bytes, RawBytes};
+        matches!(self, Bytes | RawBytes)
+    }
+
+    pub fn is_unicode(&self) -> bool {
+        matches!(self, StringKind::Unicode)
+    }
+}