check

crates/djls-template-ast/src/parser.rs

@@ -82,17 +82,17 @@ impl Parser {
         let start = token.start().unwrap_or(0);
 
         Ok(Node::Comment {
-            content: token.token_type().to_string(),
-            span: Span::new(start, token.token_type().len().unwrap_or(0) as u32),
+            content: token.content().to_string(),
+            span: Span::new(start, token.content().len() as u32),
         })
     }
 
     fn parse_django_block(&mut self, content: &str) -> Result<Node, ParserError> {
         let token = self.peek_previous()?;
         let start = token.start().unwrap_or(0);
-        let length = token.length().unwrap_or(0);
+        let length = token.content().len();
 
-        let span = Span::new(start, length);
+        let span = Span::new(start, length as u32);
 
         let bits: Vec<String> = content.split_whitespace().map(String::from).collect();
         let tag_name = bits.first().ok_or(ParserError::EmptyTag)?.clone();
@@ -227,7 +227,7 @@ impl Parser {
     fn parse_django_variable(&mut self) -> Result<Node, ParserError> {
         let token = self.peek_previous()?;
         let start = token.start().unwrap_or(0);
-        let content = token.token_type().lexeme();
+        let content = token.content();
 
         let parts: Vec<&str> = content.split('|').collect();
         let bits: Vec<String> = parts[0].split('.').map(|s| s.trim().to_string()).collect();
@@ -267,7 +267,7 @@ impl Parser {
             return self.next_node();
         }
 
-        let mut text = token.token_type().to_string();
+        let mut text = token.lexeme();
 
         while let Ok(token) = self.peek() {
             match token.token_type() {
@@ -277,7 +277,7 @@ impl Parser {
                 | TokenType::Newline
                 | TokenType::Eof => break,
                 _ => {
-                    let token_text = token.token_type().to_string();
+                    let token_text = token.lexeme();
                     text.push_str(&token_text);
                     self.consume()?;
                 }

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap

@@ -9,9 +9,9 @@ nodes:
         start: 0
         length: 21
   - Comment:
-      content: "{# Django comment #}"
+      content: Django comment
       span:
         start: 21
-        length: 20
+        length: 14
 line_offsets:
   - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap

@@ -29,10 +29,10 @@ nodes:
           assignment: ~
         nodes:
           - Comment:
-              content: "{# This if is unclosed which does matter #}"
+              content: This if is unclosed which does matter
               span:
                 start: 87
-                length: 43
+                length: 37
           - Text:
               content: "<p>Welcome"
               span:
@@ -57,10 +57,10 @@ nodes:
                 start: 178
                 length: 5
           - Comment:
-              content: "{# This div is unclosed which doesn't matter #}"
+              content: "This div is unclosed which doesn't matter"
               span:
                 start: 196
-                length: 47
+                length: 41
           - Block:
               Block:
                 tag:

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap

@@ -109,10 +109,10 @@ nodes:
           assignment: ~
         nodes:
           - Comment:
-              content: "{# Welcome message #}"
+              content: Welcome message
               span:
                 start: 510
-                length: 21
+                length: 15
           - Text:
               content: "<h1>Welcome,"
               span:

crates/djls-template-ast/src/tokens.rs

@@ -1,5 +1,4 @@
 use serde::Serialize;
-use std::fmt;
 use std::ops::{Deref, DerefMut};
 
 #[derive(Clone, Debug, Serialize, PartialEq)]
@@ -33,56 +32,10 @@ impl TokenType {
             | TokenType::StyleTagOpen(s)
             | TokenType::StyleTagClose(s)
             | TokenType::Text(s) => Some(s.len()),
-            TokenType::Comment(content, start, end) => match end {
-                Some(end) => Some(start.len() + 1 + content.len() + 1 + end.len()),
-                None => Some(start.len() + 1 + content.len()),
-            },
-            TokenType::Whitespace(len) => Some(*len),
+            TokenType::Comment(content, _, _) => Some(content.len()),
+            TokenType::Whitespace(n) => Some(*n),
             TokenType::Newline => Some(1),
-            TokenType::Eof => None,
-        }
-    }
-
-    pub fn lexeme(&self) -> &str {
-        match self {
-            TokenType::DjangoBlock(s)
-            | TokenType::DjangoVariable(s)
-            | TokenType::HtmlTagOpen(s)
-            | TokenType::HtmlTagClose(s)
-            | TokenType::HtmlTagVoid(s)
-            | TokenType::ScriptTagOpen(s)
-            | TokenType::ScriptTagClose(s)
-            | TokenType::StyleTagOpen(s)
-            | TokenType::StyleTagClose(s)
-            | TokenType::Text(s) => s,
-            TokenType::Comment(content, _, _) => content,
-            TokenType::Whitespace(_) => " ",
-            TokenType::Newline => "\n",
-            TokenType::Eof => "",
-        }
-    }
-}
-
-impl fmt::Display for TokenType {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        use TokenType::*;
-
-        match self {
-            Comment(content, start, end) => match end {
-                Some(end) => write!(f, "{} {} {}", start, content, end),
-                None => write!(f, "{} {}", start, content),
-            },
-            DjangoBlock(s) => write!(f, "{{% {} %}}", s),
-            DjangoVariable(s) => write!(f, "{{{{ {} }}}}", s),
-            Eof => Ok(()),
-            HtmlTagOpen(s) | ScriptTagOpen(s) | StyleTagOpen(s) => write!(f, "<{}>", s),
-            HtmlTagClose(s) => write!(f, "</{}>", s),
-            HtmlTagVoid(s) => write!(f, "<{}/>", s),
-            Newline => f.write_str("\n"),
-            ScriptTagClose(_) => f.write_str("</script>"),
-            StyleTagClose(_) => f.write_str("</style>"),
-            Text(s) => f.write_str(s),
-            Whitespace(len) => f.write_str(&" ".repeat(*len)),
+            TokenType::Eof => Some(0),
         }
     }
 }
@@ -103,15 +56,43 @@ impl Token {
         }
     }
 
-    pub fn lexeme_from_source<'a>(&self, source: &'a str) -> Option<&'a str> {
-        match (self.start, self.token_type.len()) {
-            (Some(start), Some(len)) => Some(&source[start..start + len]),
-            _ => None,
+    pub fn lexeme(&self) -> String {
+        match &self.token_type {
+            TokenType::Comment(_, start, end) => match end {
+                Some(end) => format!("{} {} {}", start, self.content(), end),
+                None => format!("{} {}", start, self.content()),
+            },
+            TokenType::DjangoBlock(_) => format!("{{% {} %}}", self.content()),
+            TokenType::DjangoVariable(_) => format!("{{{{ {} }}}}", self.content()),
+            TokenType::Eof => String::new(),
+            TokenType::HtmlTagOpen(_)
+            | TokenType::ScriptTagOpen(_)
+            | TokenType::StyleTagOpen(_) => format!("<{}>", self.content()),
+            TokenType::HtmlTagClose(_)
+            | TokenType::StyleTagClose(_)
+            | TokenType::ScriptTagClose(_) => format!("</{}>", self.content()),
+            TokenType::HtmlTagVoid(_) => format!("<{}/>", self.content()),
+            TokenType::Newline | TokenType::Text(_) | TokenType::Whitespace(_) => self.content(),
         }
     }
 
-    pub fn lexeme(&self) -> &str {
-        self.token_type.lexeme()
+    pub fn content(&self) -> String {
+        match &self.token_type {
+            TokenType::Comment(s, _, _)
+            | TokenType::DjangoBlock(s)
+            | TokenType::DjangoVariable(s)
+            | TokenType::Text(s)
+            | TokenType::HtmlTagOpen(s)
+            | TokenType::HtmlTagClose(s)
+            | TokenType::HtmlTagVoid(s)
+            | TokenType::ScriptTagOpen(s)
+            | TokenType::ScriptTagClose(s)
+            | TokenType::StyleTagOpen(s)
+            | TokenType::StyleTagClose(s) => s.to_string(),
+            TokenType::Whitespace(len) => " ".repeat(*len),
+            TokenType::Newline => "\n".to_string(),
+            TokenType::Eof => "".to_string(),
+        }
     }
 
     pub fn token_type(&self) -> &TokenType {