diff --git a/crates/djls-template-ast/src/parser.rs b/crates/djls-template-ast/src/parser.rs index 1236023..2d1889a 100644 --- a/crates/djls-template-ast/src/parser.rs +++ b/crates/djls-template-ast/src/parser.rs @@ -82,17 +82,17 @@ impl Parser { let start = token.start().unwrap_or(0); Ok(Node::Comment { - content: token.token_type().to_string(), - span: Span::new(start, token.token_type().len().unwrap_or(0) as u32), + content: token.content().to_string(), + span: Span::new(start, token.content().len() as u32), }) } fn parse_django_block(&mut self, content: &str) -> Result { let token = self.peek_previous()?; let start = token.start().unwrap_or(0); - let length = token.length().unwrap_or(0); + let length = token.content().len(); - let span = Span::new(start, length); + let span = Span::new(start, length as u32); let bits: Vec = content.split_whitespace().map(String::from).collect(); let tag_name = bits.first().ok_or(ParserError::EmptyTag)?.clone(); @@ -227,7 +227,7 @@ impl Parser { fn parse_django_variable(&mut self) -> Result { let token = self.peek_previous()?; let start = token.start().unwrap_or(0); - let content = token.token_type().lexeme(); + let content = token.content(); let parts: Vec<&str> = content.split('|').collect(); let bits: Vec = parts[0].split('.').map(|s| s.trim().to_string()).collect(); @@ -267,7 +267,7 @@ impl Parser { return self.next_node(); } - let mut text = token.token_type().to_string(); + let mut text = token.lexeme(); while let Ok(token) = self.peek() { match token.token_type() { @@ -277,7 +277,7 @@ impl Parser { | TokenType::Newline | TokenType::Eof => break, _ => { - let token_text = token.token_type().to_string(); + let token_text = token.lexeme(); text.push_str(&token_text); self.consume()?; } diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap index 507c57c..d3852fe 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap @@ -9,9 +9,9 @@ nodes: start: 0 length: 21 - Comment: - content: "{# Django comment #}" + content: Django comment span: start: 21 - length: 20 + length: 14 line_offsets: - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap index e628428..c314a91 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap @@ -29,10 +29,10 @@ nodes: assignment: ~ nodes: - Comment: - content: "{# This if is unclosed which does matter #}" + content: This if is unclosed which does matter span: start: 87 - length: 43 + length: 37 - Text: content: "

Welcome" span: @@ -57,10 +57,10 @@ nodes: start: 178 length: 5 - Comment: - content: "{# This div is unclosed which doesn't matter #}" + content: "This div is unclosed which doesn't matter" span: start: 196 - length: 47 + length: 41 - Block: Block: tag: diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap index ee077ca..fdd15b9 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap @@ -109,10 +109,10 @@ nodes: assignment: ~ nodes: - Comment: - content: "{# Welcome message #}" + content: Welcome message span: start: 510 - length: 21 + length: 15 - Text: content: "

Welcome," span: diff --git a/crates/djls-template-ast/src/tokens.rs b/crates/djls-template-ast/src/tokens.rs index 525a15d..88f8b3f 100644 --- a/crates/djls-template-ast/src/tokens.rs +++ b/crates/djls-template-ast/src/tokens.rs @@ -1,5 +1,4 @@ use serde::Serialize; -use std::fmt; use std::ops::{Deref, DerefMut}; #[derive(Clone, Debug, Serialize, PartialEq)] @@ -33,56 +32,10 @@ impl TokenType { | TokenType::StyleTagOpen(s) | TokenType::StyleTagClose(s) | TokenType::Text(s) => Some(s.len()), - TokenType::Comment(content, start, end) => match end { - Some(end) => Some(start.len() + 1 + content.len() + 1 + end.len()), - None => Some(start.len() + 1 + content.len()), - }, - TokenType::Whitespace(len) => Some(*len), + TokenType::Comment(content, _, _) => Some(content.len()), + TokenType::Whitespace(n) => Some(*n), TokenType::Newline => Some(1), - TokenType::Eof => None, - } - } - - pub fn lexeme(&self) -> &str { - match self { - TokenType::DjangoBlock(s) - | TokenType::DjangoVariable(s) - | TokenType::HtmlTagOpen(s) - | TokenType::HtmlTagClose(s) - | TokenType::HtmlTagVoid(s) - | TokenType::ScriptTagOpen(s) - | TokenType::ScriptTagClose(s) - | TokenType::StyleTagOpen(s) - | TokenType::StyleTagClose(s) - | TokenType::Text(s) => s, - TokenType::Comment(content, _, _) => content, - TokenType::Whitespace(_) => " ", - TokenType::Newline => "\n", - TokenType::Eof => "", - } - } -} - -impl fmt::Display for TokenType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use TokenType::*; - - match self { - Comment(content, start, end) => match end { - Some(end) => write!(f, "{} {} {}", start, content, end), - None => write!(f, "{} {}", start, content), - }, - DjangoBlock(s) => write!(f, "{{% {} %}}", s), - DjangoVariable(s) => write!(f, "{{{{ {} }}}}", s), - Eof => Ok(()), - HtmlTagOpen(s) | ScriptTagOpen(s) | StyleTagOpen(s) => write!(f, "<{}>", s), - HtmlTagClose(s) => write!(f, "", s), - HtmlTagVoid(s) => write!(f, "<{}/>", s), - Newline => f.write_str("\n"), - ScriptTagClose(_) => f.write_str(""), - StyleTagClose(_) => f.write_str(""), - Text(s) => f.write_str(s), - Whitespace(len) => f.write_str(&" ".repeat(*len)), + TokenType::Eof => Some(0), } } } @@ -103,15 +56,43 @@ impl Token { } } - pub fn lexeme_from_source<'a>(&self, source: &'a str) -> Option<&'a str> { - match (self.start, self.token_type.len()) { - (Some(start), Some(len)) => Some(&source[start..start + len]), - _ => None, + pub fn lexeme(&self) -> String { + match &self.token_type { + TokenType::Comment(_, start, end) => match end { + Some(end) => format!("{} {} {}", start, self.content(), end), + None => format!("{} {}", start, self.content()), + }, + TokenType::DjangoBlock(_) => format!("{{% {} %}}", self.content()), + TokenType::DjangoVariable(_) => format!("{{{{ {} }}}}", self.content()), + TokenType::Eof => String::new(), + TokenType::HtmlTagOpen(_) + | TokenType::ScriptTagOpen(_) + | TokenType::StyleTagOpen(_) => format!("<{}>", self.content()), + TokenType::HtmlTagClose(_) + | TokenType::StyleTagClose(_) + | TokenType::ScriptTagClose(_) => format!("", self.content()), + TokenType::HtmlTagVoid(_) => format!("<{}/>", self.content()), + TokenType::Newline | TokenType::Text(_) | TokenType::Whitespace(_) => self.content(), } } - pub fn lexeme(&self) -> &str { - self.token_type.lexeme() + pub fn content(&self) -> String { + match &self.token_type { + TokenType::Comment(s, _, _) + | TokenType::DjangoBlock(s) + | TokenType::DjangoVariable(s) + | TokenType::Text(s) + | TokenType::HtmlTagOpen(s) + | TokenType::HtmlTagClose(s) + | TokenType::HtmlTagVoid(s) + | TokenType::ScriptTagOpen(s) + | TokenType::ScriptTagClose(s) + | TokenType::StyleTagOpen(s) + | TokenType::StyleTagClose(s) => s.to_string(), + TokenType::Whitespace(len) => " ".repeat(*len), + TokenType::Newline => "\n".to_string(), + TokenType::Eof => "".to_string(), + } } pub fn token_type(&self) -> &TokenType {