josh/django-language-server
1
0
Fork 0
mirror of https://github.com/joshuadavidthomas/django-language-server.git synced 2025-09-10 12:26:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

simplify

Browse source
This commit is contained in:
Josh Thomas 2025-01-04 22:10:22 -06:00
parent 92ecfcb83c
commit baeed3ed66
20 changed files with 617 additions and 778 deletions
Download patch file Download diff file Expand all files Collapse all files

53
crates/djls-template-ast/src/ast.rs
View file

@ -36,9 +36,6 @@ impl Ast {
#[derive(Clone, Debug, Serialize)]
pub enum Node {
Django(DjangoNode),
Html(HtmlNode),
Script(ScriptNode),
Style(StyleNode),
Text(String),
}
@ -82,56 +79,6 @@ impl DjangoFilter {
}
}
#[derive(Clone, Debug, Serialize)]
pub enum HtmlNode {
Comment(String),
Doctype(String),
Element {
tag_name: String,
attributes: Attributes,
children: Vec<Node>,
},
Void {
tag_name: String,
attributes: Attributes,
},
}
#[derive(Clone, Debug, Serialize)]
pub enum ScriptNode {
Comment {
content: String,
kind: ScriptCommentKind,
},
Element {
attributes: Attributes,
children: Vec<Node>,
},
}
#[derive(Clone, Debug, Serialize)]
pub enum ScriptCommentKind {
SingleLine, // //
MultiLine, // /* */
}
#[derive(Clone, Debug, Serialize)]
pub enum StyleNode {
Comment(String),
Element {
attributes: Attributes,
children: Vec<Node>,
},
}
#[derive(Clone, Debug, Serialize)]
pub enum AttributeValue {
Value(String),
Boolean,
}
pub type Attributes = BTreeMap<String, AttributeValue>;
#[derive(Clone, Debug, Error, Serialize)]
pub enum AstError {
#[error("Empty AST")]

116
crates/djls-template-ast/src/lexer.rs
View file

@ -53,107 +53,47 @@ impl Lexer {
self.consume_n(2)?; // #}
TokenType::Comment(content, "{#".to_string(), Some("#}".to_string()))
}
_ => {
self.consume()?; // {
TokenType::Text(String::from("{"))
}
},
'<' => match self.peek_next()? {
'/' => {
self.consume_n(2)?; // </
let tag = self.consume_until(">")?;
self.consume()?; // >
TokenType::HtmlTagClose(tag)
}
'!' if self.matches("<!--")? => {
self.consume_n(4)?; // <!--
let content = self.consume_until("-->")?;
self.consume_n(3)?; // -->
TokenType::Comment(content, "<!--".to_string(), Some("-->".to_string()))
}
_ => {
self.consume()?; // consume <
let tag = self.consume_until(">")?;
self.consume()?; // consume >
if tag.starts_with("script") {
TokenType::ScriptTagOpen(tag)
} else if tag.starts_with("style") {
TokenType::StyleTagOpen(tag)
} else if tag.ends_with("/") {
TokenType::HtmlTagVoid(tag.trim_end_matches("/").to_string())
} else {
TokenType::HtmlTagOpen(tag)
}
}
},
'/' => match self.peek_next()? {
'/' => {
self.consume_n(2)?; // //
let content = self.consume_until("\n")?;
TokenType::Comment(content, "//".to_string(), None)
}
'*' => {
self.consume_n(2)?; // /*
let content = self.consume_until("*/")?;
self.consume_n(2)?; // */
TokenType::Comment(content, "/*".to_string(), Some("*/".to_string()))
}
_ => {
self.consume()?;
TokenType::Text("/".to_string())
TokenType::Text("{".to_string())
}
},
c if c.is_whitespace() => {
if c == '\n' || c == '\r' {
self.consume()?; // \r or \n
if c == '\r' && self.peek()? == '\n' {
self.consume()?; // \n of \r\n
}
'\n' => {
self.consume()?;
self.line += 1;
TokenType::Newline
} else {
self.consume()?; // Consume the first whitespace
while !self.is_at_end() && self.peek()?.is_whitespace() {
if self.peek()? == '\n' || self.peek()? == '\r' {
}
' ' | '\t' | '\r' => {
let mut count = 1;
self.consume()?;
while let Ok(c) = self.peek() {
if c != ' ' && c != '\t' && c != '\r' {
break;
}
self.consume()?;
count += 1;
}
let whitespace_count = self.current - self.start;
TokenType::Whitespace(whitespace_count)
TokenType::Whitespace(count)
}
}
_ => {
let mut text = String::new();
while !self.is_at_end() {
let c = self.peek()?;
if c == '{' || c == '<' || c == '\n' {
break;
match self.peek()? {
'{' => break,
'\n' | ' ' | '\t' | '\r' => break,
_ => {
text.push(self.consume()?);
}
text.push(c);
self.consume()?;
}
}
if text.is_empty() {
return Err(LexerError::EmptyToken(self.line));
}
TokenType::Text(text)
}
};
let token = Token::new(token_type, self.line, Some(self.start));
match self.peek_previous()? {
'\n' => self.line += 1,
'\r' => {
self.line += 1;
if self.peek()? == '\n' {
self.current += 1;
}
}
_ => {}
}
Ok(token)
Ok(Token::new(token_type, self.line, Some(self.start)))
}
fn peek(&self) -> Result<char, LexerError> {
@ -310,15 +250,7 @@ mod tests {
#[test]
fn test_tokenize_comments() {
let source = r#"<!-- HTML comment -->
{# Django comment #}
<script>
// JS single line comment
/* JS multi-line
comment */
</script>
<style>
/* CSS comment */
</style>"#;
{# Django comment #}"#;
let mut lexer = Lexer::new(source);
let tokens = lexer.tokenize().unwrap();
insta::assert_yaml_snapshot!(tokens);
@ -357,7 +289,7 @@ mod tests {
assert!(Lexer::new("{{ user.name").tokenize().is_err()); // No closing }}
assert!(Lexer::new("{% if").tokenize().is_err()); // No closing %}
assert!(Lexer::new("{#").tokenize().is_err()); // No closing #}
assert!(Lexer::new("<div").tokenize().is_err()); // No closing >
assert!(Lexer::new("<div").tokenize().is_ok()); // No closing >, but HTML is treated as text
// Invalid characters or syntax within tokens
assert!(Lexer::new("{{}}").tokenize().is_ok()); // Empty but valid

272
crates/djls-template-ast/src/parser.rs
View file

@ -1,10 +1,6 @@
use crate::ast::{
Ast, AstError, AttributeValue, DjangoFilter, DjangoNode, HtmlNode, Node, ScriptCommentKind,
ScriptNode, StyleNode, TagNode,
};
use crate::ast::{Ast, AstError, DjangoFilter, DjangoNode, Node, TagNode};
use crate::tagspecs::TagSpec;
use crate::tokens::{Token, TokenStream, TokenType};
use std::collections::BTreeMap;
use thiserror::Error;
pub struct Parser {
@ -19,18 +15,13 @@ impl Parser {
pub fn parse(&mut self) -> Result<Ast, ParserError> {
let mut ast = Ast::default();
let mut had_nodes = false;
while !self.is_at_end() {
match self.next_node() {
Ok(node) => {
ast.add_node(node);
had_nodes = true;
}
Err(ParserError::Ast(AstError::StreamError(kind))) if kind == "AtEnd" => {
if !had_nodes {
return Ok(ast.finalize()?);
}
break;
}
Err(ParserError::ErrorSignal(Signal::SpecialTag(_))) => {
@ -58,43 +49,31 @@ impl Parser {
return Err(ParserError::Ast(AstError::StreamError("AtEnd".to_string())));
}
let token = self.consume()?;
let token = self.peek()?;
let node = match token.token_type() {
TokenType::Comment(s, start, end) => self.parse_comment(s, start, end.as_deref()),
TokenType::DjangoBlock(s) => self.parse_django_block(s),
TokenType::DjangoVariable(s) => self.parse_django_variable(s),
TokenType::Eof => {
if self.is_at_end() {
Err(ParserError::Ast(AstError::StreamError("AtEnd".to_string())))
} else {
self.next_node()
TokenType::Comment(content, start, end) => {
self.consume()?;
self.parse_comment(content, start, end.as_deref())
}
TokenType::DjangoBlock(content) => {
self.consume()?;
self.parse_django_block(content)
}
TokenType::HtmlTagClose(tag) => {
self.backtrack(1)?;
Err(ParserError::ErrorSignal(Signal::ClosingTagFound(
tag.to_string(),
)))
TokenType::DjangoVariable(content) => {
self.consume()?;
self.parse_django_variable(content)
}
TokenType::HtmlTagOpen(s) => self.parse_tag_open(s),
TokenType::HtmlTagVoid(s) => self.parse_html_tag_void(s),
TokenType::Newline => self.next_node(),
TokenType::ScriptTagClose(_) => {
self.backtrack(1)?;
Err(ParserError::ErrorSignal(Signal::ClosingTagFound(
"script".to_string(),
)))
}
TokenType::ScriptTagOpen(s) => self.parse_tag_open(s),
TokenType::StyleTagClose(_) => {
self.backtrack(1)?;
Err(ParserError::ErrorSignal(Signal::ClosingTagFound(
"style".to_string(),
)))
}
TokenType::StyleTagOpen(s) => self.parse_tag_open(s),
TokenType::Text(s) => Ok(Node::Text(s.to_string())),
TokenType::Whitespace(_) => self.next_node(),
TokenType::Text(_)
| TokenType::Whitespace(_)
| TokenType::Newline
| TokenType::HtmlTagOpen(_)
| TokenType::HtmlTagClose(_)
| TokenType::HtmlTagVoid(_)
| TokenType::ScriptTagOpen(_)
| TokenType::ScriptTagClose(_)
| TokenType::StyleTagOpen(_)
| TokenType::StyleTagClose(_) => self.parse_text(),
TokenType::Eof => Err(ParserError::Ast(AstError::StreamError("AtEnd".to_string()))),
}?;
Ok(node)
}
@ -107,49 +86,12 @@ impl Parser {
) -> Result<Node, ParserError> {
match start {
"{#" => Ok(Node::Django(DjangoNode::Comment(content.to_string()))),
"<!--" => Ok(Node::Html(HtmlNode::Comment(content.to_string()))),
"//" => Ok(Node::Script(ScriptNode::Comment {
content: content.to_string(),
kind: ScriptCommentKind::SingleLine,
})),
"/*" => {
// Look back for script/style context
let token_type = self
.peek_back(self.current)?
.iter()
.find_map(|token| match token.token_type() {
TokenType::ScriptTagOpen(_) => {
Some(TokenType::ScriptTagOpen(String::new()))
}
TokenType::StyleTagOpen(_) => Some(TokenType::StyleTagOpen(String::new())),
TokenType::ScriptTagClose(_) | TokenType::StyleTagClose(_) => None,
_ => None,
})
.ok_or(ParserError::InvalidMultiLineComment)?;
match token_type {
TokenType::ScriptTagOpen(_) => Ok(Node::Script(ScriptNode::Comment {
content: content.to_string(),
kind: ScriptCommentKind::MultiLine,
})),
TokenType::StyleTagOpen(_) => {
Ok(Node::Style(StyleNode::Comment(content.to_string())))
}
_ => unreachable!(),
}
}
_ => Err(ParserError::token_error(
"valid token",
Token::new(
TokenType::Comment(
content.to_string(),
start.to_string(),
end.map(String::from),
),
0,
None,
),
)),
_ => Ok(Node::Text(format!(
"{}{}{}",
start,
content,
end.unwrap_or("")
))),
}
}
@ -274,100 +216,39 @@ impl Parser {
Ok(Node::Django(DjangoNode::Variable { bits, filters }))
}
fn parse_tag_open(&mut self, s: &str) -> Result<Node, ParserError> {
let mut parts = s.split_whitespace();
let token_type = self.peek_previous()?.token_type().clone();
let tag_name = match token_type {
TokenType::HtmlTagOpen(_) => {
let name = parts
.next()
.ok_or(ParserError::Ast(AstError::EmptyTag))?
.to_string();
if name.to_lowercase() == "!doctype" {
return Ok(Node::Html(HtmlNode::Doctype("!DOCTYPE html".to_string())));
}
name
}
TokenType::ScriptTagOpen(_) => {
parts.next(); // Skip the tag name
"script".to_string()
}
TokenType::StyleTagOpen(_) => {
parts.next(); // Skip the tag name
"style".to_string()
}
_ => return Err(ParserError::invalid_tag("Unknown tag type".to_string())),
};
let mut attributes = BTreeMap::new();
for attr in parts {
if let Some((key, value)) = parse_attribute(attr)? {
attributes.insert(key, value);
}
}
let mut children = Vec::new();
let mut found_closing_tag = false;
while !self.is_at_end() {
match self.next_node() {
Ok(node) => {
children.push(node);
}
Err(ParserError::ErrorSignal(Signal::ClosingTagFound(tag))) => {
if tag == tag_name {
found_closing_tag = true;
fn parse_text(&mut self) -> Result<Node, ParserError> {
let mut text = String::new();
while let Ok(token) = self.peek() {
match token.token_type() {
TokenType::DjangoBlock(_)
| TokenType::DjangoVariable(_)
| TokenType::Comment(_, _, _) => break,
TokenType::Text(s) => {
self.consume()?;
break;
text.push_str(s);
}
TokenType::HtmlTagOpen(s)
| TokenType::HtmlTagClose(s)
| TokenType::HtmlTagVoid(s)
| TokenType::ScriptTagOpen(s)
| TokenType::ScriptTagClose(s)
| TokenType::StyleTagOpen(s)
| TokenType::StyleTagClose(s) => {
self.consume()?;
text.push_str(s);
}
TokenType::Whitespace(len) => {
self.consume()?;
text.push_str(&" ".repeat(*len));
}
TokenType::Newline => {
self.consume()?;
text.push('\n');
}
TokenType::Eof => break,
}
}
Err(e) => return Err(e),
}
}
if !found_closing_tag {
return Err(ParserError::Ast(AstError::UnclosedTag(tag_name.clone())));
}
Ok(match token_type {
TokenType::HtmlTagOpen(_) => Node::Html(HtmlNode::Element {
tag_name,
attributes,
children,
}),
TokenType::ScriptTagOpen(_) => Node::Script(ScriptNode::Element {
attributes,
children,
}),
TokenType::StyleTagOpen(_) => Node::Style(StyleNode::Element {
attributes,
children,
}),
_ => return Err(ParserError::invalid_tag("Unknown tag type".to_string())),
})
}
fn parse_html_tag_void(&mut self, s: &str) -> Result<Node, ParserError> {
let mut parts = s.split_whitespace();
let tag_name = parts
.next()
.ok_or(ParserError::Ast(AstError::EmptyTag))?
.to_string();
let mut attributes = BTreeMap::new();
for attr in parts {
if let Some((key, value)) = parse_attribute(attr)? {
attributes.insert(key, value);
}
}
Ok(Node::Html(HtmlNode::Void {
tag_name,
attributes,
}))
Ok(Node::Text(text))
}
fn peek(&self) -> Result<Token, ParserError> {
@ -429,20 +310,17 @@ impl Parser {
}
fn synchronize(&mut self) -> Result<(), ParserError> {
const SYNC_TYPES: &[TokenType] = &[
let sync_types = [
TokenType::DjangoBlock(String::new()),
TokenType::HtmlTagOpen(String::new()),
TokenType::HtmlTagVoid(String::new()),
TokenType::ScriptTagOpen(String::new()),
TokenType::StyleTagOpen(String::new()),
TokenType::Newline,
TokenType::DjangoVariable(String::new()),
TokenType::Comment(String::new(), String::from("{#"), Some(String::from("#}"))),
TokenType::Eof,
];
while !self.is_at_end() {
let current = self.peek()?;
for sync_type in SYNC_TYPES {
for sync_type in &sync_types {
if current.token_type() == sync_type {
return Ok(());
}
@ -463,17 +341,6 @@ pub enum Signal {
ClosingTag,
}
fn parse_attribute(attr: &str) -> Result<Option<(String, AttributeValue)>, ParserError> {
if let Some((key, value)) = attr.split_once('=') {
Ok(Some((
key.to_string(),
AttributeValue::Value(value.trim_matches('"').to_string()),
)))
} else {
Ok(Some((attr.to_string(), AttributeValue::Boolean)))
}
}
#[derive(Error, Debug)]
pub enum ParserError {
#[error(transparent)]
@ -694,8 +561,7 @@ mod tests {
let mut parser = Parser::new(tokens);
let ast = parser.parse().unwrap();
insta::assert_yaml_snapshot!(ast);
assert_eq!(ast.errors().len(), 1);
assert!(matches!(&ast.errors()[0], AstError::UnclosedTag(tag) if tag == "div"));
assert_eq!(ast.errors().len(), 0);
}
#[test]
@ -727,8 +593,7 @@ mod tests {
let mut parser = Parser::new(tokens);
let ast = parser.parse().unwrap();
insta::assert_yaml_snapshot!(ast);
assert_eq!(ast.errors().len(), 1);
assert!(matches!(&ast.errors()[0], AstError::UnclosedTag(tag) if tag == "script"));
assert_eq!(ast.errors().len(), 0);
}
#[test]
@ -738,8 +603,7 @@ mod tests {
let mut parser = Parser::new(tokens);
let ast = parser.parse().unwrap();
insta::assert_yaml_snapshot!(ast);
assert_eq!(ast.errors().len(), 1);
assert!(matches!(&ast.errors()[0], AstError::UnclosedTag(tag) if tag == "style"));
assert_eq!(ast.errors().len(), 0);
}
#[test]
@ -747,13 +611,13 @@ mod tests {
let source = r#"<div class="container">
<h1>Header</h1>
{% if user.is_authenticated %}
{# This if is unclosed which does matter #}
<p>Welcome {{ user.name }}</p>
<div>
{# This div is unclosed #}
{# This div is unclosed which doesn't matter #}
{% for item in items %}
<span>{{ item }}</span>
{% endfor %}
{% endif %}
<footer>Page Footer</footer>
</div>"#;
let tokens = Lexer::new(source).tokenize().unwrap();
@ -790,7 +654,7 @@ mod tests {
<div class="header" id="main" data-value="123" disabled>
{% if user.is_authenticated %}
{# Welcome message #}
<h1>Welcome, {{ user.name|default:"Guest"|title }}!</h1>
<h1>Welcome, {{ user.name|title|default:'Guest' }}!</h1>
{% if user.is_staff %}
<span>Admin</span>
{% else %}

103
crates/djls-template-ast/src/snapshots/djls_template_ast__lexer__tests__tokenize_comments.snap
View file

@ -3,14 +3,35 @@ source: crates/djls-template-ast/src/lexer.rs
expression: tokens
---
- token_type:
Comment:
- HTML comment
- "<!--"
- "-->"
Text: "<!--"
line: 1
start: 0
- token_type: Newline
- token_type:
Whitespace: 1
line: 1
start: 4
- token_type:
Text: HTML
line: 1
start: 5
- token_type:
Whitespace: 1
line: 1
start: 9
- token_type:
Text: comment
line: 1
start: 10
- token_type:
Whitespace: 1
line: 1
start: 17
- token_type:
Text: "-->"
line: 1
start: 18
- token_type: Newline
line: 2
start: 21
- token_type:
Comment:
@ -19,76 +40,6 @@ expression: tokens
- "#}"
line: 2
start: 22
- token_type: Newline
line: 2
start: 42
- token_type:
ScriptTagOpen: script
line: 3
start: 43
- token_type: Newline
line: 3
start: 51
- token_type:
Whitespace: 4
line: 4
start: 52
- token_type:
Comment:
- JS single line comment
- //
- ~
line: 4
start: 56
- token_type: Newline
line: 4
start: 81
- token_type:
Whitespace: 4
line: 5
start: 82
- token_type:
Comment:
- "JS multi-line\n comment"
- /*
- "*/"
line: 5
start: 86
- token_type: Newline
line: 5
start: 120
- token_type:
HtmlTagClose: script
line: 6
start: 121
- token_type: Newline
line: 6
start: 130
- token_type:
StyleTagOpen: style
line: 7
start: 131
- token_type: Newline
line: 7
start: 138
- token_type:
Whitespace: 4
line: 8
start: 139
- token_type:
Comment:
- CSS comment
- /*
- "*/"
line: 8
start: 143
- token_type: Newline
line: 8
start: 160
- token_type:
HtmlTagClose: style
line: 9
start: 161
- token_type: Eof
line: 9
line: 2
start: ~

328
crates/djls-template-ast/src/snapshots/djls_template_ast__lexer__tests__tokenize_everything.snap
View file

@ -3,59 +3,100 @@ source: crates/djls-template-ast/src/lexer.rs
expression: tokens
---
- token_type:
HtmlTagOpen: "!DOCTYPE html"
Text: "<!DOCTYPE"
line: 1
start: 0
- token_type: Newline
- token_type:
Whitespace: 1
line: 1
start: 9
- token_type:
Text: html>
line: 1
start: 10
- token_type: Newline
line: 2
start: 15
- token_type:
HtmlTagOpen: html
Text: "<html>"
line: 2
start: 16
- token_type: Newline
line: 2
line: 3
start: 22
- token_type:
HtmlTagOpen: head
Text: "<head>"
line: 3
start: 23
- token_type: Newline
line: 3
line: 4
start: 29
- token_type:
Whitespace: 4
line: 4
start: 30
- token_type:
StyleTagOpen: "style type=\"text/css\""
Text: "<style"
line: 4
start: 34
- token_type: Newline
- token_type:
Whitespace: 1
line: 4
start: 40
- token_type:
Text: "type=\"text/css\">"
line: 4
start: 41
- token_type: Newline
line: 5
start: 57
- token_type:
Whitespace: 8
line: 5
start: 58
- token_type:
Comment:
- Style header
- /*
- "*/"
Text: /*
line: 5
start: 66
- token_type: Newline
- token_type:
Whitespace: 1
line: 5
start: 68
- token_type:
Text: Style
line: 5
start: 69
- token_type:
Whitespace: 1
line: 5
start: 74
- token_type:
Text: header
line: 5
start: 75
- token_type:
Whitespace: 1
line: 5
start: 81
- token_type:
Text: "*/"
line: 5
start: 82
- token_type: Newline
line: 6
start: 84
- token_type:
Whitespace: 8
line: 6
start: 85
- token_type:
Text: ".header "
Text: ".header"
line: 6
start: 93
- token_type:
Whitespace: 1
line: 6
start: 100
- token_type:
Text: "{"
line: 6
@ -65,87 +106,165 @@ expression: tokens
line: 6
start: 102
- token_type:
Text: "color: blue; }"
Text: "color:"
line: 6
start: 103
- token_type: Newline
- token_type:
Whitespace: 1
line: 6
start: 109
- token_type:
Text: blue;
line: 6
start: 110
- token_type:
Whitespace: 1
line: 6
start: 115
- token_type:
Text: "}"
line: 6
start: 116
- token_type: Newline
line: 7
start: 117
- token_type:
Whitespace: 4
line: 7
start: 118
- token_type:
HtmlTagClose: style
Text: "</style>"
line: 7
start: 122
- token_type: Newline
line: 7
line: 8
start: 130
- token_type:
Whitespace: 4
line: 8
start: 131
- token_type:
ScriptTagOpen: "script type=\"text/javascript\""
Text: "<script"
line: 8
start: 135
- token_type: Newline
- token_type:
Whitespace: 1
line: 8
start: 142
- token_type:
Text: "type=\"text/javascript\">"
line: 8
start: 143
- token_type: Newline
line: 9
start: 166
- token_type:
Whitespace: 8
line: 9
start: 167
- token_type:
Comment:
- Init app
- //
- ~
Text: //
line: 9
start: 175
- token_type: Newline
- token_type:
Whitespace: 1
line: 9
start: 177
- token_type:
Text: Init
line: 9
start: 178
- token_type:
Whitespace: 1
line: 9
start: 182
- token_type:
Text: app
line: 9
start: 183
- token_type: Newline
line: 10
start: 186
- token_type:
Whitespace: 8
line: 10
start: 187
- token_type:
Text: "const app = "
Text: const
line: 10
start: 195
- token_type:
Whitespace: 1
line: 10
start: 200
- token_type:
Text: app
line: 10
start: 201
- token_type:
Whitespace: 1
line: 10
start: 204
- token_type:
Text: "="
line: 10
start: 205
- token_type:
Whitespace: 1
line: 10
start: 206
- token_type:
Text: "{"
line: 10
start: 207
- token_type: Newline
line: 10
line: 11
start: 208
- token_type:
Whitespace: 12
line: 11
start: 209
- token_type:
Comment:
- Config
- /*
- "*/"
Text: /*
line: 11
start: 221
- token_type: Newline
- token_type:
Whitespace: 1
line: 11
start: 223
- token_type:
Text: Config
line: 11
start: 224
- token_type:
Whitespace: 1
line: 11
start: 230
- token_type:
Text: "*/"
line: 11
start: 231
- token_type: Newline
line: 12
start: 233
- token_type:
Whitespace: 12
line: 12
start: 234
- token_type:
Text: "debug: true"
Text: "debug:"
line: 12
start: 246
- token_type: Newline
- token_type:
Whitespace: 1
line: 12
start: 252
- token_type:
Text: "true"
line: 12
start: 253
- token_type: Newline
line: 13
start: 257
- token_type:
Whitespace: 8
@ -156,57 +275,110 @@ expression: tokens
line: 13
start: 266
- token_type: Newline
line: 13
line: 14
start: 268
- token_type:
Whitespace: 4
line: 14
start: 269
- token_type:
HtmlTagClose: script
Text: "</script>"
line: 14
start: 273
- token_type: Newline
line: 14
line: 15
start: 282
- token_type:
HtmlTagClose: head
Text: "</head>"
line: 15
start: 283
- token_type: Newline
line: 15
line: 16
start: 290
- token_type:
HtmlTagOpen: body
Text: "<body>"
line: 16
start: 291
- token_type: Newline
line: 16
line: 17
start: 297
- token_type:
Whitespace: 4
line: 17
start: 298
- token_type:
Comment:
- Header section
- "<!--"
- "-->"
Text: "<!--"
line: 17
start: 302
- token_type: Newline
- token_type:
Whitespace: 1
line: 17
start: 306
- token_type:
Text: Header
line: 17
start: 307
- token_type:
Whitespace: 1
line: 17
start: 313
- token_type:
Text: section
line: 17
start: 314
- token_type:
Whitespace: 1
line: 17
start: 321
- token_type:
Text: "-->"
line: 17
start: 322
- token_type: Newline
line: 18
start: 325
- token_type:
Whitespace: 4
line: 18
start: 326
- token_type:
HtmlTagOpen: "div class=\"header\" id=\"main\" data-value=\"123\" disabled"
Text: "<div"
line: 18
start: 330
- token_type: Newline
- token_type:
Whitespace: 1
line: 18
start: 334
- token_type:
Text: "class=\"header\""
line: 18
start: 335
- token_type:
Whitespace: 1
line: 18
start: 349
- token_type:
Text: "id=\"main\""
line: 18
start: 350
- token_type:
Whitespace: 1
line: 18
start: 359
- token_type:
Text: "data-value=\"123\""
line: 18
start: 360
- token_type:
Whitespace: 1
line: 18
start: 376
- token_type:
Text: disabled>
line: 18
start: 377
- token_type: Newline
line: 19
start: 386
- token_type:
Whitespace: 8
@ -217,7 +389,7 @@ expression: tokens
line: 19
start: 395
- token_type: Newline
line: 19
line: 20
start: 425
- token_type:
Whitespace: 12
@ -231,34 +403,30 @@ expression: tokens
line: 20
start: 438
- token_type: Newline
line: 20
line: 21
start: 459
- token_type:
Whitespace: 12
line: 21
start: 460
- token_type:
HtmlTagOpen: h1
Text: "<h1>Welcome,"
line: 21
start: 472
- token_type:
Text: "Welcome, "
Whitespace: 1
line: 21
start: 476
start: 484
- token_type:
DjangoVariable: "user.name|default:\"Guest\"|title"
line: 21
start: 485
- token_type:
Text: "!"
Text: "!</h1>"
line: 21
start: 522
- token_type:
HtmlTagClose: h1
line: 21
start: 523
- token_type: Newline
line: 21
line: 22
start: 528
- token_type:
Whitespace: 12
@ -269,26 +437,18 @@ expression: tokens
line: 22
start: 541
- token_type: Newline
line: 22
line: 23
start: 563
- token_type:
Whitespace: 16
line: 23
start: 564
- token_type:
HtmlTagOpen: span
Text: "<span>Admin</span>"
line: 23
start: 580
- token_type:
Text: Admin
line: 23
start: 586
- token_type:
HtmlTagClose: span
line: 23
start: 591
- token_type: Newline
line: 23
line: 24
start: 598
- token_type:
Whitespace: 12
@ -299,26 +459,18 @@ expression: tokens
line: 24
start: 611
- token_type: Newline
line: 24
line: 25
start: 621
- token_type:
Whitespace: 16
line: 25
start: 622
- token_type:
HtmlTagOpen: span
Text: "<span>User</span>"
line: 25
start: 638
- token_type:
Text: User
line: 25
start: 644
- token_type:
HtmlTagClose: span
line: 25
start: 648
- token_type: Newline
line: 25
line: 26
start: 655
- token_type:
Whitespace: 12
@ -329,7 +481,7 @@ expression: tokens
line: 26
start: 668
- token_type: Newline
line: 26
line: 27
start: 679
- token_type:
Whitespace: 8
@ -340,28 +492,28 @@ expression: tokens
line: 27
start: 688
- token_type: Newline
line: 27
line: 28
start: 699
- token_type:
Whitespace: 4
line: 28
start: 700
- token_type:
HtmlTagClose: div
Text: "</div>"
line: 28
start: 704
- token_type: Newline
line: 28
line: 29
start: 710
- token_type:
HtmlTagClose: body
Text: "</body>"
line: 29
start: 711
- token_type: Newline
line: 29
line: 30
start: 718
- token_type:
HtmlTagClose: html
Text: "</html>"
line: 30
start: 719
- token_type: Eof

26
crates/djls-template-ast/src/snapshots/djls_template_ast__lexer__tests__tokenize_html.snap
View file

@ -3,13 +3,33 @@ source: crates/djls-template-ast/src/lexer.rs
expression: tokens
---
- token_type:
HtmlTagOpen: "div class=\"container\" id=\"main\" disabled"
Text: "<div"
line: 1
start: 0
- token_type:
HtmlTagClose: div
Whitespace: 1
line: 1
start: 42
start: 4
- token_type:
Text: "class=\"container\""
line: 1
start: 5
- token_type:
Whitespace: 1
line: 1
start: 22
- token_type:
Text: "id=\"main\""
line: 1
start: 23
- token_type:
Whitespace: 1
line: 1
start: 32
- token_type:
Text: disabled></div>
line: 1
start: 33
- token_type: Eof
line: 1
start: ~

117
crates/djls-template-ast/src/snapshots/djls_template_ast__lexer__tests__tokenize_script.snap
View file

@ -3,66 +3,143 @@ source: crates/djls-template-ast/src/lexer.rs
expression: tokens
---
- token_type:
ScriptTagOpen: "script type=\"text/javascript\""
Text: "<script"
line: 1
start: 0
- token_type: Newline
- token_type:
Whitespace: 1
line: 1
start: 7
- token_type:
Text: "type=\"text/javascript\">"
line: 1
start: 8
- token_type: Newline
line: 2
start: 31
- token_type:
Whitespace: 4
line: 2
start: 32
- token_type:
Comment:
- Single line comment
- //
- ~
Text: //
line: 2
start: 36
- token_type: Newline
- token_type:
Whitespace: 1
line: 2
start: 38
- token_type:
Text: Single
line: 2
start: 39
- token_type:
Whitespace: 1
line: 2
start: 45
- token_type:
Text: line
line: 2
start: 46
- token_type:
Whitespace: 1
line: 2
start: 50
- token_type:
Text: comment
line: 2
start: 51
- token_type: Newline
line: 3
start: 58
- token_type:
Whitespace: 4
line: 3
start: 59
- token_type:
Text: const x = 1;
Text: const
line: 3
start: 63
- token_type: Newline
- token_type:
Whitespace: 1
line: 3
start: 68
- token_type:
Text: x
line: 3
start: 69
- token_type:
Whitespace: 1
line: 3
start: 70
- token_type:
Text: "="
line: 3
start: 71
- token_type:
Whitespace: 1
line: 3
start: 72
- token_type:
Text: 1;
line: 3
start: 73
- token_type: Newline
line: 4
start: 75
- token_type:
Whitespace: 4
line: 4
start: 76
- token_type:
Comment:
- "Multi-line\n comment"
- /*
- "*/"
Text: /*
line: 4
start: 80
- token_type: Newline
- token_type:
Whitespace: 1
line: 4
start: 82
- token_type:
Text: Multi-line
line: 4
start: 83
- token_type: Newline
line: 5
start: 93
- token_type:
Whitespace: 7
line: 5
start: 94
- token_type:
Text: comment
line: 5
start: 101
- token_type:
Whitespace: 1
line: 5
start: 108
- token_type:
Text: "*/"
line: 5
start: 109
- token_type: Newline
line: 6
start: 111
- token_type:
Whitespace: 4
line: 5
line: 6
start: 112
- token_type:
Text: console.log(x);
line: 5
line: 6
start: 116
- token_type: Newline
line: 5
line: 7
start: 131
- token_type:
HtmlTagClose: script
line: 6
Text: "</script>"
line: 7
start: 132
- token_type: Eof
line: 6
line: 7
start: ~

67
crates/djls-template-ast/src/snapshots/djls_template_ast__lexer__tests__tokenize_style.snap
View file

@ -3,51 +3,92 @@ source: crates/djls-template-ast/src/lexer.rs
expression: tokens
---
- token_type:
StyleTagOpen: "style type=\"text/css\""
Text: "<style"
line: 1
start: 0
- token_type: Newline
- token_type:
Whitespace: 1
line: 1
start: 6
- token_type:
Text: "type=\"text/css\">"
line: 1
start: 7
- token_type: Newline
line: 2
start: 23
- token_type:
Whitespace: 4
line: 2
start: 24
- token_type:
Comment:
- Header styles
- /*
- "*/"
Text: /*
line: 2
start: 28
- token_type: Newline
- token_type:
Whitespace: 1
line: 2
start: 30
- token_type:
Text: Header
line: 2
start: 31
- token_type:
Whitespace: 1
line: 2
start: 37
- token_type:
Text: styles
line: 2
start: 38
- token_type:
Whitespace: 1
line: 2
start: 44
- token_type:
Text: "*/"
line: 2
start: 45
- token_type: Newline
line: 3
start: 47
- token_type:
Whitespace: 4
line: 3
start: 48
- token_type:
Text: ".header "
Text: ".header"
line: 3
start: 52
- token_type:
Whitespace: 1
line: 3
start: 59
- token_type:
Text: "{"
line: 3
start: 60
- token_type: Newline
line: 3
line: 4
start: 61
- token_type:
Whitespace: 8
line: 4
start: 62
- token_type:
Text: "color: blue;"
Text: "color:"
line: 4
start: 70
- token_type: Newline
- token_type:
Whitespace: 1
line: 4
start: 76
- token_type:
Text: blue;
line: 4
start: 77
- token_type: Newline
line: 5
start: 82
- token_type:
Whitespace: 4
@ -58,10 +99,10 @@ expression: tokens
line: 5
start: 87
- token_type: Newline
line: 5
line: 6
start: 88
- token_type:
HtmlTagClose: style
Text: "</style>"
line: 6
start: 89
- token_type: Eof

3
crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap
View file

@ -3,8 +3,7 @@ source: crates/djls-template-ast/src/parser.rs
expression: ast
---
nodes:
- Html:
Comment: HTML comment
- Text: "<!-- HTML comment -->"
- Django:
Comment: Django comment
errors: []

12
crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap
View file

@ -12,6 +12,7 @@ nodes:
- if
- user.is_authenticated
children:
- Text: "\n "
- Django:
Variable:
bits:
@ -23,6 +24,7 @@ nodes:
- name: default
arguments:
- "'Guest'"
- Text: "\n "
- Django:
Tag:
Block:
@ -33,6 +35,7 @@ nodes:
- in
- user.groups
children:
- Text: "\n "
- Django:
Tag:
Block:
@ -47,12 +50,14 @@ nodes:
Closing:
name: endif
bits: []
- Text: "\n "
- Django:
Variable:
bits:
- group
- name
filters: []
- Text: "\n "
- Django:
Tag:
Block:
@ -68,6 +73,7 @@ nodes:
Closing:
name: endif
bits: []
- Text: "\n "
- Django:
Tag:
Block:
@ -82,25 +88,27 @@ nodes:
Closing:
name: endif
bits: []
- Text: "\n "
- Django:
Tag:
Branch:
name: empty
bits: []
children:
- Text: (no groups)
- Text: "\n (no groups)\n "
- Django:
Tag:
Closing:
name: endfor
bits: []
- Text: "\n"
- Django:
Tag:
Branch:
name: else
bits: []
children:
- Text: Guest
- Text: "\n Guest\n"
- Django:
Tag:
Closing:

48
crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap
View file

@ -3,48 +3,6 @@ source: crates/djls-template-ast/src/parser.rs
expression: ast
---
nodes:
- Html:
Element:
tag_name: div
attributes:
class:
Value: container
children:
- Html:
Element:
tag_name: h1
attributes: {}
children:
- Text: Header
- Django:
Tag:
Block:
name: if
bits:
- if
- user.is_authenticated
children:
- Html:
Element:
tag_name: p
attributes: {}
children:
- Text: "Welcome "
- Django:
Variable:
bits:
- user
- name
filters: []
- Django:
Tag:
Closing:
name: endif
bits: []
- Html:
Element:
tag_name: footer
attributes: {}
children:
- Text: Page Footer
errors: []
- Text: "<div class=\"container\">\n <h1>Header</h1>\n "
errors:
- UnclosedTag: if

6
crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_html_tag.snap
View file

@ -2,6 +2,6 @@
source: crates/djls-template-ast/src/parser.rs
expression: ast
---
nodes: []
errors:
- UnclosedTag: div
nodes:
- Text: "<div>"
errors: []

6
crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_script.snap
View file

@ -2,6 +2,6 @@
source: crates/djls-template-ast/src/parser.rs
expression: ast
---
nodes: []
errors:
- UnclosedTag: script
nodes:
- Text: "<script>console.log('test');"
errors: []

6
crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_style.snap
View file

@ -2,6 +2,6 @@
source: crates/djls-template-ast/src/parser.rs
expression: ast
---
nodes: []
errors:
- UnclosedTag: style
nodes:
- Text: "<style>body { color: blue; "
errors: []

93
crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap
View file

@ -3,66 +3,7 @@ source: crates/djls-template-ast/src/parser.rs
expression: ast
---
nodes:
- Html:
Doctype: "!DOCTYPE html"
- Html:
Element:
tag_name: html
attributes: {}
children:
- Html:
Element:
tag_name: head
attributes: {}
children:
- Style:
Element:
attributes:
type:
Value: text/css
children:
- Style:
Comment: Style header
- Text: ".header "
- Text: "{"
- Text: "color: blue; }"
- Script:
Element:
attributes:
type:
Value: text/javascript
children:
- Script:
Comment:
content: Init app
kind: SingleLine
- Text: "const app = "
- Text: "{"
- Script:
Comment:
content: Config
kind: MultiLine
- Text: "debug: true"
- Text: "};"
- Html:
Element:
tag_name: body
attributes: {}
children:
- Html:
Comment: Header section
- Html:
Element:
tag_name: div
attributes:
class:
Value: header
data-value:
Value: "123"
disabled: Boolean
id:
Value: main
children:
- Text: "<!DOCTYPE html>\n<html>\n <head>\n <style type=\"text/css\">\n /* Style header */\n .header { color: blue; }\n </style>\n <script type=\"text/javascript\">\n // Init app\n const app = {\n /* Config */\n debug: true\n };\n </script>\n </head>\n <body>\n <!-- Header section -->\n <div class=\"header\" id=\"main\" data-value=\"123\" disabled>\n "
- Django:
Tag:
Block:
@ -71,26 +12,22 @@ nodes:
- if
- user.is_authenticated
children:
- Text: "\n "
- Django:
Comment: Welcome message
- Html:
Element:
tag_name: h1
attributes: {}
children:
- Text: "Welcome, "
- Text: "\n <h1>Welcome, "
- Django:
Variable:
bits:
- user
- name
filters:
- name: default
arguments:
- Guest
- name: title
arguments: []
- Text: "!"
- name: default
arguments:
- "'Guest'"
- Text: "!</h1>\n "
- Django:
Tag:
Block:
@ -99,32 +36,24 @@ nodes:
- if
- user.is_staff
children:
- Html:
Element:
tag_name: span
attributes: {}
children:
- Text: Admin
- Text: "\n <span>Admin</span>\n "
- Django:
Tag:
Branch:
name: else
bits: []
children:
- Html:
Element:
tag_name: span
attributes: {}
children:
- Text: User
- Text: "\n <span>User</span>\n "
- Django:
Tag:
Closing:
name: endif
bits: []
- Text: "\n "
- Django:
Tag:
Closing:
name: endif
bits: []
- Text: "\n </div>\n </body>\n</html>"
errors: []

3
crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_doctype.snap
View file

@ -3,6 +3,5 @@ source: crates/djls-template-ast/src/parser.rs
expression: ast
---
nodes:
- Html:
Doctype: "!DOCTYPE html"
- Text: "<!DOCTYPE html>"
errors: []

9
crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_tag.snap
View file

@ -3,12 +3,5 @@ source: crates/djls-template-ast/src/parser.rs
expression: ast
---
nodes:
- Html:
Element:
tag_name: div
attributes:
class:
Value: container
children:
- Text: Hello
- Text: "<div class=\"container\">Hello</div>"
errors: []

7
crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_void.snap
View file

@ -3,10 +3,5 @@ source: crates/djls-template-ast/src/parser.rs
expression: ast
---
nodes:
- Html:
Void:
tag_name: input
attributes:
type:
Value: text
- Text: "<input type=\"text\" />"
errors: []

17
crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__script__parse_script.snap
View file

@ -3,20 +3,5 @@ source: crates/djls-template-ast/src/parser.rs
expression: ast
---
nodes:
- Script:
Element:
attributes:
type:
Value: text/javascript
children:
- Script:
Comment:
content: Single line comment
kind: SingleLine
- Text: const x = 1;
- Script:
Comment:
content: "Multi-line\n comment"
kind: MultiLine
- Text: console.log(x);
- Text: "<script type=\"text/javascript\">\n // Single line comment\n const x = 1;\n /* Multi-line\n comment */\n console.log(x);\n</script>"
errors: []

13
crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__style__parse_style.snap
View file

@ -3,16 +3,5 @@ source: crates/djls-template-ast/src/parser.rs
expression: ast
---
nodes:
- Style:
Element:
attributes:
type:
Value: text/css
children:
- Style:
Comment: Header styles
- Text: ".header "
- Text: "{"
- Text: "color: blue;"
- Text: "}"
- Text: "<style type=\"text/css\">\n /* Header styles */\n .header {\n color: blue;\n }\n</style>"
errors: []