diff --git a/crates/djls-template-ast/Cargo.toml b/crates/djls-template-ast/Cargo.toml index 9eeaa20..f71fea6 100644 --- a/crates/djls-template-ast/Cargo.toml +++ b/crates/djls-template-ast/Cargo.toml @@ -4,8 +4,11 @@ version = "0.0.0" edition = "2021" [dependencies] +anyhow = { workspace = true } serde = { workspace = true } thiserror = { workspace = true } +toml = "0.8" + [dev-dependencies] insta = { version = "1.41", features = ["yaml"] } diff --git a/crates/djls-template-ast/src/ast.rs b/crates/djls-template-ast/src/ast.rs index 3bb034c..672b075 100644 --- a/crates/djls-template-ast/src/ast.rs +++ b/crates/djls-template-ast/src/ast.rs @@ -1,6 +1,5 @@ use serde::Serialize; use std::collections::BTreeMap; -use std::str::FromStr; use thiserror::Error; #[derive(Clone, Debug, Default, Serialize)] @@ -37,11 +36,7 @@ pub enum Node { #[derive(Clone, Debug, Serialize)] pub enum DjangoNode { Comment(String), - Tag { - kind: DjangoTagKind, - bits: Vec, - children: Vec, - }, + Tag(TagNode), Variable { bits: Vec, filters: Vec, @@ -49,109 +44,18 @@ pub enum DjangoNode { } #[derive(Clone, Debug, Serialize)] -pub enum DjangoTagKind { - Autoescape, - Block, - Comment, - CsrfToken, - Cycle, - Debug, - Elif, - Else, - Empty, - Extends, - Filter, - FirstOf, - For, - If, - IfChanged, - Include, - Load, - Lorem, - Now, - Other(String), - Querystring, // 5.1 - Regroup, - ResetCycle, - Spaceless, - TemplateTag, - Url, - Verbatim, - WidthRatio, - With, -} - -impl DjangoTagKind { - const AUTOESCAPE: &'static str = "autoescape"; - const BLOCK: &'static str = "block"; - const COMMENT: &'static str = "comment"; - const CSRF_TOKEN: &'static str = "csrf_token"; - const CYCLE: &'static str = "cycle"; - const DEBUG: &'static str = "debug"; - const ELIF: &'static str = "elif"; - const ELSE: &'static str = "else"; - const EMPTY: &'static str = "empty"; - const EXTENDS: &'static str = "extends"; - const FILTER: &'static str = "filter"; - const FIRST_OF: &'static str = "firstof"; - const FOR: &'static str = "for"; - const IF: &'static str = "if"; - const IF_CHANGED: &'static str = "ifchanged"; - const INCLUDE: &'static str = "include"; - const LOAD: &'static str = "load"; - const LOREM: &'static str = "lorem"; - const NOW: &'static str = "now"; - const QUERYSTRING: &'static str = "querystring"; - const REGROUP: &'static str = "regroup"; - const RESET_CYCLE: &'static str = "resetcycle"; - const SPACELESS: &'static str = "spaceless"; - const TEMPLATE_TAG: &'static str = "templatetag"; - const URL: &'static str = "url"; - const VERBATIM: &'static str = "verbatim"; - const WIDTH_RATIO: &'static str = "widthratio"; - const WITH: &'static str = "with"; -} - -impl FromStr for DjangoTagKind { - type Err = AstError; - - fn from_str(s: &str) -> Result { - if s.is_empty() { - return Err(AstError::EmptyTag); - } - - match s { - Self::AUTOESCAPE => Ok(Self::Autoescape), - Self::BLOCK => Ok(Self::Block), - Self::COMMENT => Ok(Self::Comment), - Self::CSRF_TOKEN => Ok(Self::CsrfToken), - Self::CYCLE => Ok(Self::Cycle), - Self::DEBUG => Ok(Self::Debug), - Self::ELIF => Ok(Self::Elif), - Self::ELSE => Ok(Self::Else), - Self::EMPTY => Ok(Self::Empty), - Self::EXTENDS => Ok(Self::Extends), - Self::FILTER => Ok(Self::Filter), - Self::FIRST_OF => Ok(Self::FirstOf), - Self::FOR => Ok(Self::For), - Self::IF => Ok(Self::If), - Self::IF_CHANGED => Ok(Self::IfChanged), - Self::INCLUDE => Ok(Self::Include), - Self::LOAD => Ok(Self::Load), - Self::LOREM => Ok(Self::Lorem), - Self::NOW => Ok(Self::Now), - Self::QUERYSTRING => Ok(Self::Querystring), - Self::REGROUP => Ok(Self::Regroup), - Self::RESET_CYCLE => Ok(Self::ResetCycle), - Self::SPACELESS => Ok(Self::Spaceless), - Self::TEMPLATE_TAG => Ok(Self::TemplateTag), - Self::URL => Ok(Self::Url), - Self::VERBATIM => Ok(Self::Verbatim), - Self::WIDTH_RATIO => Ok(Self::WidthRatio), - Self::WITH => Ok(Self::With), - other => Ok(Self::Other(other.to_string())), - } - } +pub enum TagNode { + Block { + name: String, + bits: Vec, + children: Vec, + }, + Branching { + name: String, + bits: Vec, + children: Vec, + branches: Vec, + }, } #[derive(Clone, Debug, Serialize)] diff --git a/crates/djls-template-ast/src/lib.rs b/crates/djls-template-ast/src/lib.rs index 6490bf8..4c76429 100644 --- a/crates/djls-template-ast/src/lib.rs +++ b/crates/djls-template-ast/src/lib.rs @@ -1,6 +1,7 @@ mod ast; mod lexer; mod parser; +mod tagspecs; mod tokens; pub use ast::Ast; diff --git a/crates/djls-template-ast/src/parser.rs b/crates/djls-template-ast/src/parser.rs index ff50e2f..2c7013d 100644 --- a/crates/djls-template-ast/src/parser.rs +++ b/crates/djls-template-ast/src/parser.rs @@ -1,41 +1,65 @@ use crate::ast::{ - Ast, AstError, AttributeValue, DjangoFilter, DjangoNode, DjangoTagKind, HtmlNode, Node, - ScriptCommentKind, ScriptNode, StyleNode, + Ast, AstError, AttributeValue, DjangoFilter, DjangoNode, HtmlNode, Node, ScriptCommentKind, + ScriptNode, StyleNode, TagNode, }; +use crate::tagspecs::TagSpec; use crate::tokens::{Token, TokenStream, TokenType}; -use std::collections::BTreeMap; -use std::str::FromStr; +use std::collections::{BTreeMap, HashMap}; use thiserror::Error; pub struct Parser { tokens: TokenStream, current: usize, + specs: HashMap, } impl Parser { pub fn new(tokens: TokenStream) -> Self { - Parser { tokens, current: 0 } + Parser { + tokens, + current: 0, + specs: TagSpec::load_builtin_specs().unwrap_or_default(), + } } pub fn parse(&mut self) -> Result { let mut ast = Ast::default(); + let mut had_nodes = false; + while !self.is_at_end() { match self.next_node() { Ok(node) => { + eprintln!("Adding node: {:?}", node); ast.add_node(node); + had_nodes = true; } Err(ParserError::StreamError(Stream::AtEnd)) => { - if ast.nodes().is_empty() { + eprintln!("Stream at end, nodes: {:?}", ast.nodes()); + if !had_nodes { return Err(ParserError::StreamError(Stream::UnexpectedEof)); } break; } - Err(_) => { + Err(ParserError::ErrorSignal(Signal::SpecialTag(tag))) => { + eprintln!("Got special tag: {}", tag); + continue; + } + Err(ParserError::UnclosedTag(tag)) => { + eprintln!("Got unclosed tag: {}", tag); + return Err(ParserError::UnclosedTag(tag)); + } + Err(e) => { + eprintln!("Got error: {:?}", e); self.synchronize()?; continue; } } } + + eprintln!("Final nodes: {:?}", ast.nodes()); + if !had_nodes { + return Err(ParserError::StreamError(Stream::UnexpectedEof)); + } ast.finalize()?; Ok(ast) } @@ -79,6 +103,7 @@ impl Parser { TokenType::Text(s) => Ok(Node::Text(s.to_string())), TokenType::Whitespace(_) => self.next_node(), }?; + eprintln!("{:?}", node); Ok(node) } @@ -134,61 +159,96 @@ impl Parser { } fn parse_django_block(&mut self, s: &str) -> Result { + eprintln!("Parsing django block: {}", s); let bits: Vec = s.split_whitespace().map(String::from).collect(); - let kind = DjangoTagKind::from_str(&bits[0])?; + let tag_name = bits.first().ok_or(AstError::EmptyTag)?.clone(); + eprintln!("Tag name: {}", tag_name); - if bits[0].starts_with("end") { - return Err(ParserError::ErrorSignal(Signal::ClosingTagFound( - bits[0].clone(), - ))); + eprintln!("Loaded specs: {:?}", self.specs); + + // Check if this is a closing tag according to ANY spec + for (_, spec) in self.specs.iter() { + if Some(&tag_name) == spec.closing.as_ref() { + eprintln!("Found closing tag: {}", tag_name); + return Err(ParserError::ErrorSignal(Signal::SpecialTag(tag_name))); + } } - let mut all_children = Vec::new(); - let mut current_section = Vec::new(); - let end_tag = format!("end{}", bits[0]); + // Check if this is an intermediate tag according to ANY spec + for (_, spec) in self.specs.iter() { + if let Some(intermediates) = &spec.intermediates { + if intermediates.contains(&tag_name) { + eprintln!("Found intermediate tag: {}", tag_name); + return Err(ParserError::ErrorSignal(Signal::SpecialTag(tag_name))); + } + } + } + + // Get the tag spec for this tag + let tag_spec = self.specs.get(tag_name.as_str()).cloned(); + eprintln!("Tag spec: {:?}", tag_spec); + + let mut children = Vec::new(); + let mut branches = Vec::new(); while !self.is_at_end() { match self.next_node() { Ok(node) => { - current_section.push(node); + eprintln!("Adding node: {:?}", node); + children.push(node); } - Err(ParserError::ErrorSignal(Signal::ClosingTagFound(tag))) => { - match tag.as_str() { - tag if tag == end_tag.as_str() => { - // Found matching end tag, complete the block - all_children.extend(current_section); - return Ok(Node::Django(DjangoNode::Tag { - kind, - bits, - children: all_children, - })); + Err(ParserError::ErrorSignal(Signal::SpecialTag(tag))) => { + eprintln!("Got special tag: {}", tag); + if let Some(spec) = &tag_spec { + // Check if this is a closing tag + if Some(&tag) == spec.closing.as_ref() { + eprintln!("Found matching closing tag: {}", tag); + // Found our closing tag, create appropriate tag type + let tag_node = if !branches.is_empty() { + TagNode::Branching { + name: tag_name, + bits, + children, + branches, + } + } else { + TagNode::Block { + name: tag_name, + bits, + children, + } + }; + return Ok(Node::Django(DjangoNode::Tag(tag_node))); } - tag if !tag.starts_with("end") => { - // Found intermediate tag (like 'else', 'elif') - all_children.extend(current_section); - all_children.push(Node::Django(DjangoNode::Tag { - kind: DjangoTagKind::from_str(tag)?, - bits: vec![tag.to_string()], - children: Vec::new(), - })); - current_section = Vec::new(); - continue; // Continue parsing after intermediate tag - } - tag => { - // Found unexpected end tag - return Err(ParserError::ErrorSignal(Signal::ClosingTagFound( - tag.to_string(), - ))); + // Check if this is an intermediate tag + if let Some(intermediates) = &spec.intermediates { + if intermediates.contains(&tag) { + eprintln!("Found intermediate tag: {}", tag); + // Add current children as a branch and start fresh + branches.push(TagNode::Block { + name: tag.clone(), + bits: vec![tag.clone()], + children, + }); + children = Vec::new(); + continue; + } } } + // If we get here, it's an unexpected tag + eprintln!("Unexpected tag: {}", tag); + return Err(ParserError::UnexpectedTag(tag)); } Err(e) => { + eprintln!("Error: {:?}", e); return Err(e); } } } - Err(ParserError::StreamError(Stream::UnexpectedEof)) + // If we get here, we never found the closing tag + eprintln!("Never found closing tag: {}", tag_name); + Err(ParserError::UnclosedTag(tag_name)) } fn parse_django_variable(&mut self, s: &str) -> Result { @@ -359,6 +419,7 @@ impl Parser { } let mut children = Vec::new(); + let mut found_closing_tag = false; while !self.is_at_end() { match self.next_node() { @@ -368,6 +429,7 @@ impl Parser { Err(ParserError::ErrorSignal(Signal::ClosingTagFound(tag))) => { if tag == "style" { self.consume()?; + found_closing_tag = true; break; } // If it's not our closing tag, keep collecting children @@ -376,6 +438,10 @@ impl Parser { } } + if !found_closing_tag { + return Err(ParserError::UnclosedTag("style".to_string())); + } + Ok(Node::Style(StyleNode::Element { attributes, children, @@ -454,12 +520,13 @@ impl Parser { } fn consume_if(&mut self, token_type: TokenType) -> Result { - let token = self.peek()?; - if token.is_token_type(&token_type) { - return Err(ParserError::ExpectedTokenType(token_type)); + let token = self.consume()?; + if token.token_type() == &token_type { + Ok(token) + } else { + self.backtrack(1)?; + Err(ParserError::ExpectedTokenType(format!("{:?}", token_type))) } - self.consume()?; - Ok(token) } fn consume_until(&mut self, end_type: TokenType) -> Result, ParserError> { @@ -472,7 +539,6 @@ impl Parser { } fn synchronize(&mut self) -> Result<(), ParserError> { - println!("--- Starting synchronization ---"); const SYNC_TYPES: &[TokenType] = &[ TokenType::DjangoBlock(String::new()), TokenType::HtmlTagOpen(String::new()), @@ -485,39 +551,53 @@ impl Parser { while !self.is_at_end() { let current = self.peek()?; - println!("--- Sync checking token: {:?}", current); - // Debug print for token type comparison for sync_type in SYNC_TYPES { - println!("--- Comparing with sync type: {:?}", sync_type); if matches!(current.token_type(), sync_type) { - println!("--- Found sync point at: {:?}", current); return Ok(()); } } - - println!("--- Consuming token in sync: {:?}", current); self.consume()?; } - println!("--- Reached end during synchronization"); + Ok(()) } } #[derive(Error, Debug)] pub enum ParserError { - #[error("token stream {0}")] - StreamError(Stream), - #[error("parsing signal: {0:?}")] + #[error("unclosed tag: {0}")] + UnclosedTag(String), + #[error("unexpected tag: {0}")] + UnexpectedTag(String), + #[error("unsupported tag type")] + UnsupportedTagType, + #[error("empty tag")] + EmptyTag, + #[error("invalid tag type")] + InvalidTagType, + #[error("missing required args")] + MissingRequiredArgs, + #[error("invalid argument '{0:?}' '{1:?}")] + InvalidArgument(String, String), + #[error("unexpected closing tag {0}")] + UnexpectedClosingTag(String), + #[error("unexpected intermediate tag {0}")] + UnexpectedIntermediateTag(String), + #[error("unclosed block {0}")] + UnclosedBlock(String), + #[error(transparent)] + StreamError(#[from] Stream), + #[error("internal signal: {0:?}")] ErrorSignal(Signal), - #[error("unexpected token, expected type '{0:?}'")] - ExpectedTokenType(TokenType), + #[error("expected token: {0}")] + ExpectedTokenType(String), #[error("unexpected token '{0:?}'")] UnexpectedToken(Token), #[error("multi-line comment outside of script or style context")] InvalidMultLineComment, #[error(transparent)] - Ast(#[from] AstError), + AstError(#[from] AstError), } #[derive(Debug)] @@ -529,10 +609,7 @@ pub enum Stream { InvalidAccess, } -#[derive(Debug)] -pub enum Signal { - ClosingTagFound(String), -} +impl std::error::Error for Stream {} impl std::fmt::Display for Stream { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { @@ -546,149 +623,254 @@ impl std::fmt::Display for Stream { } } +#[derive(Debug)] +pub enum Signal { + ClosingTagFound(String), + IntermediateTagFound(String, Vec), + IntermediateTag(String), + SpecialTag(String), + ClosingTag, +} + #[cfg(test)] mod tests { + use super::Stream; use super::*; use crate::lexer::Lexer; - #[test] - fn test_parse_comments() { - let source = r#" -{# Django comment #} - -"#; - let tokens = Lexer::new(source).tokenize().unwrap(); - let mut parser = Parser::new(tokens); - let ast = parser.parse().unwrap(); - insta::assert_yaml_snapshot!(ast); + mod html { + use super::*; + + #[test] + fn test_parse_html_doctype() { + let source = ""; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } + + #[test] + fn test_parse_html_tag() { + let source = "
Hello
"; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } + + #[test] + fn test_parse_html_void() { + let source = ""; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } } - #[test] - fn test_parse_django_block() { - let source = r#"{% if user.is_staff %}Admin{% else %}User{% endif %}"#; - let tokens = Lexer::new(source).tokenize().unwrap(); - let mut parser = Parser::new(tokens); - let ast = parser.parse().unwrap(); - insta::assert_yaml_snapshot!(ast); + mod django { + use super::*; + + #[test] + fn test_parse_django_variable() { + let source = "{{ user.name|title }}"; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } + + #[test] + fn test_parse_filter_chains() { + let source = "{{ value|default:'nothing'|title|upper }}"; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } + + #[test] + fn test_parse_django_if_block() { + let source = "{% if user.is_authenticated %}Welcome{% endif %}"; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } + + #[test] + fn test_parse_django_for_block() { + let source = "{% for item in items %}{{ item }}{% empty %}No items{% endfor %}"; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } + + #[test] + fn test_parse_complex_if_elif() { + let source = "{% if x > 0 %}Positive{% elif x < 0 %}Negative{% else %}Zero{% endif %}"; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } + + #[test] + fn test_parse_nested_for_if() { + let source = + "{% for item in items %}{% if item.active %}{{ item.name }}{% endif %}{% endfor %}"; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } + + #[test] + fn test_parse_mixed_content() { + let source = "Welcome, {% if user.is_authenticated %} + {{ user.name|title|default:'Guest' }} + {% for group in user.groups %} + {% if forloop.first %}({% endif %} + {{ group.name }} + {% if not forloop.last %}, {% endif %} + {% if forloop.last %}){% endif %} + {% empty %} + (no groups) + {% endfor %} + {% else %} + Guest + {% endif %}!"; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } } - #[test] - fn test_parse_django_variable() { - let source = r#"{{ user.name|default:"Anonymous"|title }}"#; - let tokens = Lexer::new(source).tokenize().unwrap(); - let mut parser = Parser::new(tokens); - let ast = parser.parse().unwrap(); - insta::assert_yaml_snapshot!(ast); - } - #[test] - fn test_parse_html_tag() { - let source = r#"
"#; - let tokens = Lexer::new(source).tokenize().unwrap(); - let mut parser = Parser::new(tokens); - let ast = parser.parse().unwrap(); - insta::assert_yaml_snapshot!(ast); + mod script { + use super::*; + + #[test] + fn test_parse_script() { + let source = ""; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } } - #[test] - fn test_parse_html_void() { - let source = r#""#; - let tokens = Lexer::new(source).tokenize().unwrap(); - let mut parser = Parser::new(tokens); - let ast = parser.parse().unwrap(); - insta::assert_yaml_snapshot!(ast); + mod style { + use super::*; + + #[test] + fn test_parse_style() { + let source = ""; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } } - #[test] - fn test_parse_html_doctype() { - let source = r#""#; - let tokens = Lexer::new(source).tokenize().unwrap(); - let mut parser = Parser::new(tokens); - let ast = parser.parse().unwrap(); - insta::assert_yaml_snapshot!(ast); + mod comments { + use super::*; + + #[test] + fn test_parse_comments() { + let source = "{# Django comment #}"; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } } - #[test] - fn test_parse_script() { - let source = r#""#; - let tokens = Lexer::new(source).tokenize().unwrap(); - let mut parser = Parser::new(tokens); - let ast = parser.parse().unwrap(); - insta::assert_yaml_snapshot!(ast); + mod errors { + use super::*; + + #[test] + fn test_parse_unexpected_eof() { + let source = "
\n"; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse(); + assert!(matches!( + ast, + Err(ParserError::StreamError(Stream::UnexpectedEof)) + )); + } + + #[test] + fn test_parse_unclosed_django_if() { + let source = "{% if user.is_authenticated %}Welcome"; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let result = parser.parse(); + println!("Error: {:?}", result); + assert!(matches!(result, Err(ParserError::UnclosedTag(tag)) if tag == "if")); + } + + #[test] + fn test_parse_unclosed_django_for() { + let source = "{% for item in items %}{{ item.name }}"; + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let result = parser.parse(); + println!("Error: {:?}", result); + assert!(matches!(result, Err(ParserError::UnclosedTag(tag)) if tag == "for")); + } + + #[test] + fn test_parse_unclosed_style() { + let source = ""#; - let tokens = Lexer::new(source).tokenize().unwrap(); - let mut parser = Parser::new(tokens); - let ast = parser.parse().unwrap(); - insta::assert_yaml_snapshot!(ast); - } + mod full_templates { + use super::*; - #[test] - fn test_parse_full() { - let source = r#" + #[test] + fn test_parse_full() { + let source = r#" - - - - - - -
- {% if user.is_authenticated %} - {# Welcome message #} -

Welcome, {{ user.name|default:"Guest"|title }}!

- {% if user.is_staff %} - Admin - {% else %} - User - {% endif %} - {% endif %} -
- + + {% block title %}Default Title{% endblock %} + + + +

Welcome{% if user.is_authenticated %}, {{ user.name }}{% endif %}!

+ + "#; - let tokens = Lexer::new(source).tokenize().unwrap(); - let mut parser = Parser::new(tokens); - let ast = parser.parse().unwrap(); - insta::assert_yaml_snapshot!(ast); - } - - #[test] - fn test_parse_unexpected_eof() { - let source = "
\n"; - let tokens = Lexer::new(source).tokenize().unwrap(); - let mut parser = Parser::new(tokens); - let ast = parser.parse(); - assert!(matches!( - ast, - Err(ParserError::StreamError(Stream::UnexpectedEof)) - )); + let tokens = Lexer::new(source).tokenize().unwrap(); + let mut parser = Parser::new(tokens); + let ast = parser.parse().unwrap(); + insta::assert_yaml_snapshot!(ast); + } } } diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap new file mode 100644 index 0000000..0bd0dc2 --- /dev/null +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap @@ -0,0 +1,9 @@ +--- +source: crates/djls-template-ast/src/parser.rs +expression: ast +--- +nodes: + - Html: + Comment: HTML comment + - Django: + Comment: Django comment diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_complex_if_elif.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_complex_if_elif.snap new file mode 100644 index 0000000..02414f5 --- /dev/null +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_complex_if_elif.snap @@ -0,0 +1,29 @@ +--- +source: crates/djls-template-ast/src/parser.rs +expression: ast +--- +nodes: + - Django: + Tag: + Branching: + name: if + bits: + - if + - x + - ">" + - "0" + children: + - Text: Zero + branches: + - Block: + name: elif + bits: + - elif + children: + - Text: Positive + - Block: + name: else + bits: + - else + children: + - Text: Negative diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_for_block.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_for_block.snap new file mode 100644 index 0000000..ba68416 --- /dev/null +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_for_block.snap @@ -0,0 +1,27 @@ +--- +source: crates/djls-template-ast/src/parser.rs +expression: ast +--- +nodes: + - Django: + Tag: + Branching: + name: for + bits: + - for + - item + - in + - items + children: + - Text: No items + branches: + - Block: + name: empty + bits: + - empty + children: + - Django: + Variable: + bits: + - item + filters: [] diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_if_block.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_if_block.snap new file mode 100644 index 0000000..166f1f3 --- /dev/null +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_if_block.snap @@ -0,0 +1,14 @@ +--- +source: crates/djls-template-ast/src/parser.rs +expression: ast +--- +nodes: + - Django: + Tag: + Block: + name: if + bits: + - if + - user.is_authenticated + children: + - Text: Welcome diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_django_variable.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_variable.snap similarity index 74% rename from crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_django_variable.snap rename to crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_variable.snap index 754b31e..57dab6e 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_django_variable.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_variable.snap @@ -9,8 +9,5 @@ nodes: - user - name filters: - - name: default - arguments: - - Anonymous - name: title arguments: [] diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_filter_chains.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_filter_chains.snap new file mode 100644 index 0000000..4593b1f --- /dev/null +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_filter_chains.snap @@ -0,0 +1,17 @@ +--- +source: crates/djls-template-ast/src/parser.rs +expression: ast +--- +nodes: + - Django: + Variable: + bits: + - value + filters: + - name: default + arguments: + - "'nothing'" + - name: title + arguments: [] + - name: upper + arguments: [] diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap new file mode 100644 index 0000000..ac86f42 --- /dev/null +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap @@ -0,0 +1,84 @@ +--- +source: crates/djls-template-ast/src/parser.rs +expression: ast +--- +nodes: + - Text: "Welcome, " + - Django: + Tag: + Branching: + name: if + bits: + - if + - user.is_authenticated + children: + - Text: Guest + branches: + - Block: + name: else + bits: + - else + children: + - Django: + Variable: + bits: + - user + - name + filters: + - name: title + arguments: [] + - name: default + arguments: + - "'Guest'" + - Django: + Tag: + Branching: + name: for + bits: + - for + - group + - in + - user.groups + children: + - Text: (no groups) + branches: + - Block: + name: empty + bits: + - empty + children: + - Django: + Tag: + Block: + name: if + bits: + - if + - forloop.first + children: + - Text: ( + - Django: + Variable: + bits: + - group + - name + filters: [] + - Django: + Tag: + Block: + name: if + bits: + - if + - not + - forloop.last + children: + - Text: ", " + - Django: + Tag: + Block: + name: if + bits: + - if + - forloop.last + children: + - Text: ) + - Text: "!" diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_nested_for_if.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_nested_for_if.snap new file mode 100644 index 0000000..8e4ac15 --- /dev/null +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_nested_for_if.snap @@ -0,0 +1,29 @@ +--- +source: crates/djls-template-ast/src/parser.rs +expression: ast +--- +nodes: + - Django: + Tag: + Block: + name: for + bits: + - for + - item + - in + - items + children: + - Django: + Tag: + Block: + name: if + bits: + - if + - item.active + children: + - Django: + Variable: + bits: + - item + - name + filters: [] diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap new file mode 100644 index 0000000..2a12086 --- /dev/null +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap @@ -0,0 +1,68 @@ +--- +source: crates/djls-template-ast/src/parser.rs +expression: ast +--- +nodes: + - Html: + Doctype: "!DOCTYPE" + - Html: + Element: + tag_name: html + attributes: {} + children: + - Html: + Element: + tag_name: head + attributes: {} + children: + - Html: + Element: + tag_name: title + attributes: {} + children: [] + - Style: + Element: + attributes: {} + children: + - Style: + Comment: CSS styles + - Text: "body " + - Text: "{" + - Text: "font-family: sans-serif; }" + - Html: + Element: + tag_name: body + attributes: {} + children: + - Html: + Element: + tag_name: h1 + attributes: {} + children: + - Text: Welcome + - Django: + Tag: + Block: + name: if + bits: + - if + - user.is_authenticated + children: + - Text: ", " + - Django: + Variable: + bits: + - user + - name + filters: [] + - Text: "!" + - Script: + Element: + attributes: + script: Boolean + children: + - Script: + Comment: + content: JavaScript code + kind: SingleLine + - Text: "console.log('Hello!');" diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_html_doctype.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_doctype.snap similarity index 100% rename from crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_html_doctype.snap rename to crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_doctype.snap diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_html_tag.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_tag.snap similarity index 68% rename from crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_html_tag.snap rename to crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_tag.snap index 3f8d16e..72786c7 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_html_tag.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_tag.snap @@ -9,7 +9,5 @@ nodes: attributes: class: Value: container - disabled: Boolean - id: - Value: main - children: [] + children: + - Text: Hello diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_html_void.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_void.snap similarity index 63% rename from crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_html_void.snap rename to crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_void.snap index ccdd908..bebb96e 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_html_void.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_void.snap @@ -5,7 +5,7 @@ expression: ast nodes: - Html: Void: - tag_name: img + tag_name: input attributes: - src: - Value: example.png + type: + Value: text diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_comments.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_comments.snap deleted file mode 100644 index dea3376..0000000 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_comments.snap +++ /dev/null @@ -1,28 +0,0 @@ ---- -source: crates/djls-template-ast/src/parser.rs -expression: ast ---- -nodes: - - Html: - Comment: HTML comment - - Django: - Comment: Django comment - - Script: - Element: - attributes: - script: Boolean - children: - - Script: - Comment: - content: JS single line - kind: SingleLine - - Script: - Comment: - content: "JS multi\n line" - kind: MultiLine - - Style: - Element: - attributes: {} - children: - - Style: - Comment: CSS comment diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_django_block.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_django_block.snap deleted file mode 100644 index 8a1fe87..0000000 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_django_block.snap +++ /dev/null @@ -1,13 +0,0 @@ ---- -source: crates/djls-template-ast/src/parser.rs -expression: ast ---- -nodes: - - Django: - Tag: - kind: If - bits: - - if - - user.is_staff - children: - - Text: Admin diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_full.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_full.snap deleted file mode 100644 index 31c87b3..0000000 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_full.snap +++ /dev/null @@ -1,106 +0,0 @@ ---- -source: crates/djls-template-ast/src/parser.rs -expression: ast ---- -nodes: - - Html: - Doctype: "!DOCTYPE" - - Html: - Element: - tag_name: html - attributes: {} - children: - - Html: - Element: - tag_name: head - attributes: {} - children: - - Style: - Element: - attributes: - type: - Value: text/css - children: - - Style: - Comment: Style header - - Text: ".header " - - Text: "{" - - Text: "color: blue; }" - - Script: - Element: - attributes: - script: Boolean - type: - Value: text/javascript - children: - - Script: - Comment: - content: Init app - kind: SingleLine - - Text: "const app = " - - Text: "{" - - Script: - Comment: - content: Config - kind: MultiLine - - Text: "debug: true" - - Text: "};" - - Html: - Element: - tag_name: body - attributes: {} - children: - - Html: - Comment: Header section - - Html: - Element: - tag_name: div - attributes: - class: - Value: header - data-value: - Value: "123" - disabled: Boolean - id: - Value: main - children: - - Django: - Tag: - kind: If - bits: - - if - - user.is_authenticated - children: - - Django: - Comment: Welcome message - - Html: - Element: - tag_name: h1 - attributes: {} - children: - - Text: "Welcome, " - - Django: - Variable: - bits: - - user - - name - filters: - - name: default - arguments: - - Guest - - name: title - arguments: [] - - Text: "!" - - Django: - Tag: - kind: If - bits: - - if - - user.is_staff - children: - - Html: - Element: - tag_name: span - attributes: {} - children: - - Text: Admin diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_script.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__script__parse_script.snap similarity index 66% rename from crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_script.snap rename to crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__script__parse_script.snap index 31a8bc0..b5a76ca 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_script.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__script__parse_script.snap @@ -7,16 +7,14 @@ nodes: Element: attributes: script: Boolean - type: - Value: text/javascript children: + - Text: const x = 42; - Script: Comment: - content: Single line comment + content: JavaScript comment kind: SingleLine - - Text: const x = 1; - Script: Comment: - content: "Multi-line\n comment" + content: "Multi-line\n comment" kind: MultiLine - Text: console.log(x); diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_style.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__style__parse_style.snap similarity index 53% rename from crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_style.snap rename to crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__style__parse_style.snap index 0b40855..fdef425 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_style.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__style__parse_style.snap @@ -5,13 +5,11 @@ expression: ast nodes: - Style: Element: - attributes: - type: - Value: text/css + attributes: {} children: - Style: - Comment: Header styles - - Text: ".header " + Comment: CSS comment + - Text: "body " - Text: "{" - - Text: "color: blue;" + - Text: "font-family: sans-serif;" - Text: "}" diff --git a/crates/djls-template-ast/src/tagspecs.rs b/crates/djls-template-ast/src/tagspecs.rs new file mode 100644 index 0000000..e7c744f --- /dev/null +++ b/crates/djls-template-ast/src/tagspecs.rs @@ -0,0 +1,116 @@ +use anyhow::{Context, Result}; +use serde::Deserialize; +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use toml::Value; + +#[derive(Clone, Debug, Deserialize)] +pub struct TagSpec { + #[serde(rename = "type")] + pub tag_type: TagType, + pub closing: Option, + pub intermediates: Option>, + pub args: Option>, +} + +impl TagSpec { + pub fn load_builtin_specs() -> Result> { + let specs_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tagspecs"); + + let mut all_specs = HashMap::new(); + + for entry in fs::read_dir(&specs_dir)? { + let entry = entry?; + let path = entry.path(); + + if path.extension().and_then(|ext| ext.to_str()) == Some("toml") { + let content = fs::read_to_string(&path) + .with_context(|| format!("Failed to read {:?}", path))?; + + let value: Value = toml::from_str(&content) + .with_context(|| format!("Failed to parse {:?}", path))?; + + Self::extract_specs(&value, "", &mut all_specs)?; + } + } + + Ok(all_specs) + } + + fn extract_specs( + value: &Value, + prefix: &str, + specs: &mut HashMap, + ) -> Result<()> { + if let Value::Table(table) = value { + // If this table has a 'type' field, try to parse it as a TagSpec + if table.contains_key("type") { + if let Ok(tag_spec) = TagSpec::deserialize(value.clone()) { + let name = prefix.split('.').last().unwrap_or(prefix); + specs.insert(name.to_string(), tag_spec); + return Ok(()); + } + } + + // Otherwise, recursively process each field + for (key, value) in table { + let new_prefix = if prefix.is_empty() { + key.clone() + } else { + format!("{}.{}", prefix, key) + }; + Self::extract_specs(value, &new_prefix, specs)?; + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum TagType { + Block, + Tag, + Assignment, + Variable, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ArgSpec { + pub name: String, + pub required: bool, +} + +impl ArgSpec { + pub fn is_placeholder(arg: &str) -> bool { + arg.starts_with('{') && arg.ends_with('}') + } + + pub fn get_placeholder_name(arg: &str) -> Option<&str> { + if Self::is_placeholder(arg) { + Some(&arg[1..arg.len() - 1]) + } else { + None + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_specs_are_valid() -> Result<()> { + let specs = TagSpec::load_builtin_specs()?; + + assert!(!specs.is_empty(), "Should have loaded at least one spec"); + + println!("Loaded {} tag specs:", specs.len()); + for (name, spec) in &specs { + println!(" {} ({:?})", name, spec.tag_type); + } + + Ok(()) + } +} diff --git a/crates/djls-template-ast/tagspecs/README.md b/crates/djls-template-ast/tagspecs/README.md new file mode 100644 index 0000000..ed9767f --- /dev/null +++ b/crates/djls-template-ast/tagspecs/README.md @@ -0,0 +1,83 @@ +# djls-template-ast Tag Specifications + +Configuration files defining template tag behavior for the Django Language Server Protocol. + +## Schema + +```toml +[package.module.path.tag_name] # Path where tag is registered, e.g., django.template.defaulttags +type = "block" | "tag" | "assignment" | "variable" # Required: Type of template tag +closing = "endtag" # Optional: Name of closing tag for block tags +intermediates = ["else", "elif"] # Optional: Allowed intermediate tags + +[[package.module.path.tag_name.args]] # Optional: Arguments specification +name = "arg_name" # Name of the argument +required = true | false # Whether the argument is required +``` + +## Tag Types + +- `block`: Tags that wrap content and require a closing tag + + ```django + {% if condition %}content{% endif %} + {% for item in items %}content{% endfor %} + ``` + +- `tag`: Single tags that don't wrap content + + ```django + {% csrf_token %} + {% include "template.html" %} + ``` + +- `assignment`: Tags that assign their output to a variable + + ```django + {% url 'view-name' as url_var %} + {% with total=business.employees.count %} + ``` + +- `variable`: Tags that output a value directly + + ```django + {% cycle 'odd' 'even' %} + {% firstof var1 var2 var3 %} + ``` + +## Argument Specification + +Arguments can be either: + +- Literal values that must match exactly (e.g., "in") +- Placeholders for variables (wrapped in curly braces, e.g., "{item}") + +## Examples + +```toml +[django.template.defaulttags.if] +type = "block" +closing = "endif" +intermediates = ["else", "elif"] + +[[django.template.defaulttags.if.args]] +name = "condition" +required = true + +[django.template.defaulttags.for] +type = "block" +closing = "endfor" +intermediates = ["empty"] + +[[django.template.defaulttags.for.args]] +name = "{item}" +required = true + +[[django.template.defaulttags.for.args]] +name = "in" +required = true + +[[django.template.defaulttags.for.args]] +name = "{iterable}" +required = true +``` diff --git a/crates/djls-template-ast/tagspecs/django.toml b/crates/djls-template-ast/tagspecs/django.toml new file mode 100644 index 0000000..7eb1b35 --- /dev/null +++ b/crates/djls-template-ast/tagspecs/django.toml @@ -0,0 +1,25 @@ +[django.template.defaulttags.if] +type = "block" +closing = "endif" +intermediates = ["else", "elif"] + +[[django.template.defaulttags.if.args]] +name = "condition" +required = true + +[django.template.defaulttags.for] +type = "block" +closing = "endfor" +intermediates = ["empty"] + +[[django.template.defaulttags.for.args]] +name = "{item}" +required = true + +[[django.template.defaulttags.for.args]] +name = "in" +required = true + +[[django.template.defaulttags.for.args]] +name = "{iterable}" +required = true