add TagSpecs and alter Parser (#68)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

crates/djls-project/src/lib.rs

@@ -95,6 +95,7 @@ impl fmt::Display for DjangoProject {
 
 #[derive(Debug)]
 struct PythonEnvironment {
+    #[allow(dead_code)]
     python_path: PathBuf,
     sys_path: Vec<PathBuf>,
     sys_prefix: PathBuf,

crates/djls-template-ast/Cargo.toml

@@ -4,8 +4,12 @@ version = "0.0.0"
 edition = "2021"
 
 [dependencies]
+anyhow = { workspace = true }
+lsp-types = { workspace = true }
 serde = { workspace = true }
 thiserror = { workspace = true }
+toml = "0.8"
 
 [dev-dependencies]
-insta = { version = "1.41", features = ["yaml"] }
+insta = { version = "1.42", features = ["yaml"] }
+tempfile = "3.19"

crates/djls-template-ast/src/ast.rs

@@ -1,225 +1,207 @@
+use crate::tokens::Token;
 use serde::Serialize;
-use std::collections::BTreeMap;
-use std::str::FromStr;
 use thiserror::Error;
 
-#[derive(Clone, Debug, Default, Serialize)]
-pub struct Ast {
+#[derive(Clone, Default, Debug, Serialize)]
+pub struct NodeList {
     nodes: Vec<Node>,
+    line_offsets: LineOffsets,
 }
 
-impl Ast {
+impl NodeList {
     pub fn nodes(&self) -> &Vec<Node> {
         &self.nodes
     }
 
+    pub fn line_offsets(&self) -> &LineOffsets {
+        &self.line_offsets
+    }
+
     pub fn add_node(&mut self, node: Node) {
         self.nodes.push(node);
     }
 
-    pub fn finalize(&mut self) -> Result<Ast, AstError> {
-        if self.nodes.is_empty() {
-            return Err(AstError::EmptyAst);
-        }
-        Ok(self.clone())
+    pub fn set_line_offsets(&mut self, line_offsets: LineOffsets) {
+        self.line_offsets = line_offsets
+    }
+
+    pub fn finalize(&mut self) -> NodeList {
+        self.clone()
+    }
+}
+
+#[derive(Clone, Default, Debug, Serialize)]
+pub struct LineOffsets(pub Vec<u32>);
+
+impl LineOffsets {
+    pub fn new() -> Self {
+        Self(vec![0])
+    }
+
+    pub fn add_line(&mut self, offset: u32) {
+        self.0.push(offset);
+    }
+
+    pub fn position_to_line_col(&self, position: usize) -> (usize, usize) {
+        let position = position as u32;
+        let line = match self.0.binary_search(&position) {
+            Ok(exact_line) => exact_line,    // Position is at start of this line
+            Err(0) => 0,                     // Before first line start
+            Err(next_line) => next_line - 1, // We're on the previous line
+        };
+
+        // Calculate column as offset from line start
+        let col = if line == 0 {
+            position as usize
+        } else {
+            (position - self.0[line]) as usize
+        };
+
+        // Convert to 1-based line number
+        (line + 1, col)
+    }
+
+    pub fn line_col_to_position(&self, line: u32, col: u32) -> u32 {
+        // line is 1-based, so subtract 1 to get the index
+        self.0[(line - 1) as usize] + col
     }
 }
 
 #[derive(Clone, Debug, Serialize)]
 pub enum Node {
-    Django(DjangoNode),
-    Html(HtmlNode),
-    Script(ScriptNode),
-    Style(StyleNode),
-    Text(String),
-}
-
-#[derive(Clone, Debug, Serialize)]
-pub enum DjangoNode {
-    Comment(String),
     Tag {
-        kind: DjangoTagKind,
+        name: String,
         bits: Vec<String>,
-        children: Vec<Node>,
+        span: Span,
     },
-    Variable {
-        bits: Vec<String>,
-        filters: Vec<DjangoFilter>,
-    },
-}
-
-#[derive(Clone, Debug, Serialize)]
-pub enum DjangoTagKind {
-    Autoescape,
-    Block,
-    Comment,
-    CsrfToken,
-    Cycle,
-    Debug,
-    Elif,
-    Else,
-    Empty,
-    Extends,
-    Filter,
-    FirstOf,
-    For,
-    If,
-    IfChanged,
-    Include,
-    Load,
-    Lorem,
-    Now,
-    Other(String),
-    Querystring, // 5.1
-    Regroup,
-    ResetCycle,
-    Spaceless,
-    TemplateTag,
-    Url,
-    Verbatim,
-    WidthRatio,
-    With,
-}
-
-impl DjangoTagKind {
-    const AUTOESCAPE: &'static str = "autoescape";
-    const BLOCK: &'static str = "block";
-    const COMMENT: &'static str = "comment";
-    const CSRF_TOKEN: &'static str = "csrf_token";
-    const CYCLE: &'static str = "cycle";
-    const DEBUG: &'static str = "debug";
-    const ELIF: &'static str = "elif";
-    const ELSE: &'static str = "else";
-    const EMPTY: &'static str = "empty";
-    const EXTENDS: &'static str = "extends";
-    const FILTER: &'static str = "filter";
-    const FIRST_OF: &'static str = "firstof";
-    const FOR: &'static str = "for";
-    const IF: &'static str = "if";
-    const IF_CHANGED: &'static str = "ifchanged";
-    const INCLUDE: &'static str = "include";
-    const LOAD: &'static str = "load";
-    const LOREM: &'static str = "lorem";
-    const NOW: &'static str = "now";
-    const QUERYSTRING: &'static str = "querystring";
-    const REGROUP: &'static str = "regroup";
-    const RESET_CYCLE: &'static str = "resetcycle";
-    const SPACELESS: &'static str = "spaceless";
-    const TEMPLATE_TAG: &'static str = "templatetag";
-    const URL: &'static str = "url";
-    const VERBATIM: &'static str = "verbatim";
-    const WIDTH_RATIO: &'static str = "widthratio";
-    const WITH: &'static str = "with";
-}
-
-impl FromStr for DjangoTagKind {
-    type Err = AstError;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        if s.is_empty() {
-            return Err(AstError::EmptyTag);
-        }
-
-        match s {
-            Self::AUTOESCAPE => Ok(Self::Autoescape),
-            Self::BLOCK => Ok(Self::Block),
-            Self::COMMENT => Ok(Self::Comment),
-            Self::CSRF_TOKEN => Ok(Self::CsrfToken),
-            Self::CYCLE => Ok(Self::Cycle),
-            Self::DEBUG => Ok(Self::Debug),
-            Self::ELIF => Ok(Self::Elif),
-            Self::ELSE => Ok(Self::Else),
-            Self::EMPTY => Ok(Self::Empty),
-            Self::EXTENDS => Ok(Self::Extends),
-            Self::FILTER => Ok(Self::Filter),
-            Self::FIRST_OF => Ok(Self::FirstOf),
-            Self::FOR => Ok(Self::For),
-            Self::IF => Ok(Self::If),
-            Self::IF_CHANGED => Ok(Self::IfChanged),
-            Self::INCLUDE => Ok(Self::Include),
-            Self::LOAD => Ok(Self::Load),
-            Self::LOREM => Ok(Self::Lorem),
-            Self::NOW => Ok(Self::Now),
-            Self::QUERYSTRING => Ok(Self::Querystring),
-            Self::REGROUP => Ok(Self::Regroup),
-            Self::RESET_CYCLE => Ok(Self::ResetCycle),
-            Self::SPACELESS => Ok(Self::Spaceless),
-            Self::TEMPLATE_TAG => Ok(Self::TemplateTag),
-            Self::URL => Ok(Self::Url),
-            Self::VERBATIM => Ok(Self::Verbatim),
-            Self::WIDTH_RATIO => Ok(Self::WidthRatio),
-            Self::WITH => Ok(Self::With),
-            other => Ok(Self::Other(other.to_string())),
-        }
-    }
-}
-
-#[derive(Clone, Debug, Serialize)]
-pub struct DjangoFilter {
-    name: String,
-    arguments: Vec<String>,
-}
-
-impl DjangoFilter {
-    pub fn new(name: String, arguments: Vec<String>) -> Self {
-        Self { name, arguments }
-    }
-}
-
-#[derive(Clone, Debug, Serialize)]
-pub enum HtmlNode {
-    Comment(String),
-    Doctype(String),
-    Element {
-        tag_name: String,
-        attributes: Attributes,
-        children: Vec<Node>,
-    },
-    Void {
-        tag_name: String,
-        attributes: Attributes,
-    },
-}
-
-#[derive(Clone, Debug, Serialize)]
-pub enum ScriptNode {
     Comment {
         content: String,
-        kind: ScriptCommentKind,
+        span: Span,
     },
-    Element {
-        attributes: Attributes,
-        children: Vec<Node>,
+    Text {
+        content: String,
+        span: Span,
+    },
+    Variable {
+        var: String,
+        filters: Vec<String>,
+        span: Span,
     },
 }
 
-#[derive(Clone, Debug, Serialize)]
-pub enum ScriptCommentKind {
-    SingleLine, // //
-    MultiLine,  // /* */
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)]
+pub struct Span {
+    start: u32,
+    length: u32,
 }
 
-#[derive(Clone, Debug, Serialize)]
-pub enum StyleNode {
-    Comment(String),
-    Element {
-        attributes: Attributes,
-        children: Vec<Node>,
-    },
+impl Span {
+    pub fn new(start: u32, length: u32) -> Self {
+        Self { start, length }
+    }
+
+    pub fn start(&self) -> &u32 {
+        &self.start
+    }
+
+    pub fn length(&self) -> &u32 {
+        &self.length
+    }
 }
 
-#[derive(Clone, Debug, Serialize)]
-pub enum AttributeValue {
-    Value(String),
-    Boolean,
+impl From<Token> for Span {
+    fn from(token: Token) -> Self {
+        let start = token.start().unwrap_or(0);
+        let length = token.content().len() as u32;
+        Span::new(start, length)
+    }
 }
 
-pub type Attributes = BTreeMap<String, AttributeValue>;
-
-#[derive(Error, Debug)]
+#[derive(Clone, Debug, Error, Serialize)]
 pub enum AstError {
-    #[error("error parsing django tag, recieved empty tag name")]
-    EmptyTag,
-    #[error("empty ast")]
+    #[error("Empty AST")]
     EmptyAst,
+    #[error("Invalid tag '{tag}' structure: {reason}")]
+    InvalidTagStructure {
+        tag: String,
+        reason: String,
+        span: Span,
+    },
+    #[error("Unbalanced structure: '{opening_tag}' at {opening_span:?} missing closing '{expected_closing}'")]
+    UnbalancedStructure {
+        opening_tag: String,
+        expected_closing: String,
+        opening_span: Span,
+        closing_span: Option<Span>,
+    },
+    #[error("Invalid {node_type} node: {reason}")]
+    InvalidNode {
+        node_type: String,
+        reason: String,
+        span: Span,
+    },
+    #[error("Unclosed tag: {0}")]
+    UnclosedTag(String),
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::lexer::Lexer;
+    use crate::parser::Parser;
+
+    mod line_offsets {
+        use super::*;
+
+        #[test]
+        fn test_new_starts_at_zero() {
+            let offsets = LineOffsets::new();
+            assert_eq!(offsets.position_to_line_col(0), (1, 0)); // Line 1, column 0
+        }
+
+        #[test]
+        fn test_start_of_lines() {
+            let mut offsets = LineOffsets::new();
+            offsets.add_line(10); // Line 2 starts at offset 10
+            offsets.add_line(25); // Line 3 starts at offset 25
+
+            assert_eq!(offsets.position_to_line_col(0), (1, 0)); // Line 1, start
+            assert_eq!(offsets.position_to_line_col(10), (2, 0)); // Line 2, start
+            assert_eq!(offsets.position_to_line_col(25), (3, 0)); // Line 3, start
+        }
+    }
+
+    mod spans_and_positions {
+        use super::*;
+
+        #[test]
+        fn test_variable_spans() {
+            let template = "Hello\n{{ user.name }}\nWorld";
+            let tokens = Lexer::new(template).tokenize().unwrap();
+            let mut parser = Parser::new(tokens);
+            let (nodelist, errors) = parser.parse().unwrap();
+            assert!(errors.is_empty());
+
+            // Find the variable node
+            let nodes = nodelist.nodes();
+            let var_node = nodes
+                .iter()
+                .find(|n| matches!(n, Node::Variable { .. }))
+                .unwrap();
+
+            if let Node::Variable { span, .. } = var_node {
+                // Variable starts after newline + "{{"
+                let (line, col) = nodelist
+                    .line_offsets()
+                    .position_to_line_col(*span.start() as usize);
+                assert_eq!(
+                    (line, col),
+                    (2, 0),
+                    "Variable should start at line 2, col 3"
+                );
+
+                assert_eq!(*span.length(), 9, "Variable span should cover 'user.name'");
+            }
+        }
+    }
 }

crates/djls-template-ast/src/error.rs

@@ -0,0 +1,99 @@
+use crate::ast::{AstError, Span};
+use crate::lexer::LexerError;
+use crate::parser::ParserError;
+use lsp_types;
+use serde::Serialize;
+use thiserror::Error;
+
+#[derive(Debug, Error, Serialize)]
+pub enum TemplateError {
+    #[error("Lexer error: {0}")]
+    Lexer(String),
+
+    #[error("Parser error: {0}")]
+    Parser(String),
+
+    #[error("Validation error: {0}")]
+    Validation(#[from] AstError),
+
+    #[error("IO error: {0}")]
+    Io(String),
+
+    #[error("Configuration error: {0}")]
+    Config(String),
+}
+
+impl From<LexerError> for TemplateError {
+    fn from(err: LexerError) -> Self {
+        Self::Lexer(err.to_string())
+    }
+}
+
+impl From<ParserError> for TemplateError {
+    fn from(err: ParserError) -> Self {
+        Self::Parser(err.to_string())
+    }
+}
+
+impl From<std::io::Error> for TemplateError {
+    fn from(err: std::io::Error) -> Self {
+        Self::Io(err.to_string())
+    }
+}
+
+impl TemplateError {
+    pub fn span(&self) -> Option<Span> {
+        match self {
+            TemplateError::Validation(AstError::InvalidTagStructure { span, .. }) => Some(*span),
+            _ => None,
+        }
+    }
+
+    pub fn severity(&self) -> lsp_types::DiagnosticSeverity {
+        match self {
+            TemplateError::Lexer(_) | TemplateError::Parser(_) => {
+                lsp_types::DiagnosticSeverity::ERROR
+            }
+            TemplateError::Validation(_) => lsp_types::DiagnosticSeverity::WARNING,
+            _ => lsp_types::DiagnosticSeverity::INFORMATION,
+        }
+    }
+
+    pub fn code(&self) -> &'static str {
+        match self {
+            TemplateError::Lexer(_) => "LEX",
+            TemplateError::Parser(_) => "PAR",
+            TemplateError::Validation(_) => "VAL",
+            TemplateError::Io(_) => "IO",
+            TemplateError::Config(_) => "CFG",
+        }
+    }
+}
+
+pub fn to_lsp_diagnostic(error: &TemplateError, _source: &str) -> lsp_types::Diagnostic {
+    let range = error.span().map_or_else(
+        || lsp_types::Range::default(),
+        |span| {
+            let start = lsp_types::Position::new(0, *span.start());
+            let end = lsp_types::Position::new(0, span.start() + span.length());
+            lsp_types::Range::new(start, end)
+        },
+    );
+
+    lsp_types::Diagnostic {
+        range,
+        severity: Some(error.severity()),
+        code: Some(lsp_types::NumberOrString::String(error.code().to_string())),
+        code_description: None,
+        source: Some("djls-template".to_string()),
+        message: error.to_string(),
+        related_information: None,
+        tags: None,
+        data: None,
+    }
+}
+
+pub struct QuickFix {
+    pub title: String,
+    pub edit: String,
+}

crates/djls-template-ast/src/lexer.rs

@@ -22,140 +22,136 @@ impl Lexer {
 
     pub fn tokenize(&mut self) -> Result<TokenStream, LexerError> {
         let mut tokens = TokenStream::default();
+
         while !self.is_at_end() {
-            let token = self.next_token()?;
+            self.start = self.current;
+
+            let token_type = match self.peek()? {
+                '{' => match self.peek_next()? {
+                    '%' => {
+                        self.consume_n(2)?; // {%
+                        let content = self.consume_until("%}")?;
+                        self.consume_n(2)?; // %}
+                        TokenType::DjangoBlock(content)
+                    }
+                    '{' => {
+                        self.consume_n(2)?; // {{
+                        let content = self.consume_until("}}")?;
+                        self.consume_n(2)?; // }}
+                        TokenType::DjangoVariable(content)
+                    }
+                    '#' => {
+                        self.consume_n(2)?; // {#
+                        let content = self.consume_until("#}")?;
+                        self.consume_n(2)?; // #}
+                        TokenType::Comment(content, "{#".to_string(), Some("#}".to_string()))
+                    }
+                    _ => {
+                        self.consume()?; // {
+                        TokenType::Text(String::from("{"))
+                    }
+                },
+
+                '<' => match self.peek_next()? {
+                    '/' => {
+                        self.consume_n(2)?; // </
+                        let tag = self.consume_until(">")?;
+                        self.consume()?; // >
+                        TokenType::HtmlTagClose(tag)
+                    }
+                    '!' if self.matches("<!--")? => {
+                        self.consume_n(4)?; // <!--
+                        let content = self.consume_until("-->")?;
+                        self.consume_n(3)?; // -->
+                        TokenType::Comment(content, "<!--".to_string(), Some("-->".to_string()))
+                    }
+                    _ => {
+                        self.consume()?; // consume <
+                        let tag = self.consume_until(">")?;
+                        self.consume()?; // consume >
+                        if tag.starts_with("script") {
+                            TokenType::ScriptTagOpen(tag)
+                        } else if tag.starts_with("style") {
+                            TokenType::StyleTagOpen(tag)
+                        } else if tag.ends_with("/") {
+                            TokenType::HtmlTagVoid(tag.trim_end_matches("/").to_string())
+                        } else {
+                            TokenType::HtmlTagOpen(tag)
+                        }
+                    }
+                },
+
+                '/' => match self.peek_next()? {
+                    '/' => {
+                        self.consume_n(2)?; // //
+                        let content = self.consume_until("\n")?;
+                        TokenType::Comment(content, "//".to_string(), None)
+                    }
+                    '*' => {
+                        self.consume_n(2)?; // /*
+                        let content = self.consume_until("*/")?;
+                        self.consume_n(2)?; // */
+                        TokenType::Comment(content, "/*".to_string(), Some("*/".to_string()))
+                    }
+                    _ => {
+                        self.consume()?;
+                        TokenType::Text("/".to_string())
+                    }
+                },
+
+                c if c.is_whitespace() => {
+                    if c == '\n' || c == '\r' {
+                        self.consume()?; // \r or \n
+                        if c == '\r' && self.peek()? == '\n' {
+                            self.consume()?; // \n of \r\n
+                        }
+                        TokenType::Newline
+                    } else {
+                        self.consume()?; // Consume the first whitespace
+                        while !self.is_at_end() && self.peek()?.is_whitespace() {
+                            if self.peek()? == '\n' || self.peek()? == '\r' {
+                                break;
+                            }
+                            self.consume()?;
+                        }
+                        let whitespace_count = self.current - self.start;
+                        TokenType::Whitespace(whitespace_count)
+                    }
+                }
+
+                _ => {
+                    let mut text = String::new();
+                    while !self.is_at_end() {
+                        let c = self.peek()?;
+                        if c == '{' || c == '<' || c == '\n' {
+                            break;
+                        }
+                        text.push(c);
+                        self.consume()?;
+                    }
+                    TokenType::Text(text)
+                }
+            };
+
+            let token = Token::new(token_type, self.line, Some(self.start));
+
+            match self.peek_previous()? {
+                '\n' => self.line += 1,
+                '\r' => {
+                    self.line += 1;
+                    if self.peek()? == '\n' {
+                        self.current += 1;
+                    }
+                }
+                _ => {}
+            }
+
             tokens.add_token(token);
         }
         tokens.finalize(self.line);
         Ok(tokens)
     }
 
-    fn next_token(&mut self) -> Result<Token, LexerError> {
-        self.start = self.current;
-
-        let token_type = match self.peek()? {
-            '{' => match self.peek_next()? {
-                '%' => {
-                    self.consume_n(2)?; // {%
-                    let content = self.consume_until("%}")?;
-                    self.consume_n(2)?; // %}
-                    TokenType::DjangoBlock(content)
-                }
-                '{' => {
-                    self.consume_n(2)?; // {{
-                    let content = self.consume_until("}}")?;
-                    self.consume_n(2)?; // }}
-                    TokenType::DjangoVariable(content)
-                }
-                '#' => {
-                    self.consume_n(2)?; // {#
-                    let content = self.consume_until("#}")?;
-                    self.consume_n(2)?; // #}
-                    TokenType::Comment(content, "{#".to_string(), Some("#}".to_string()))
-                }
-                _ => {
-                    self.consume()?; // {
-                    TokenType::Text(String::from("{"))
-                }
-            },
-
-            '<' => match self.peek_next()? {
-                '/' => {
-                    self.consume_n(2)?; // </
-                    let tag = self.consume_until(">")?;
-                    self.consume()?; // >
-                    TokenType::HtmlTagClose(tag)
-                }
-                '!' if self.matches("<!--")? => {
-                    self.consume_n(4)?; // <!--
-                    let content = self.consume_until("-->")?;
-                    self.consume_n(3)?; // -->
-                    TokenType::Comment(content, "<!--".to_string(), Some("-->".to_string()))
-                }
-                _ => {
-                    self.consume()?; // consume <
-                    let tag = self.consume_until(">")?;
-                    self.consume()?; // consume >
-                    if tag.starts_with("script") {
-                        TokenType::ScriptTagOpen(tag)
-                    } else if tag.starts_with("style") {
-                        TokenType::StyleTagOpen(tag)
-                    } else if tag.ends_with("/") {
-                        TokenType::HtmlTagVoid(tag.trim_end_matches("/").to_string())
-                    } else {
-                        TokenType::HtmlTagOpen(tag)
-                    }
-                }
-            },
-
-            '/' => match self.peek_next()? {
-                '/' => {
-                    self.consume_n(2)?; // //
-                    let content = self.consume_until("\n")?;
-                    TokenType::Comment(content, "//".to_string(), None)
-                }
-                '*' => {
-                    self.consume_n(2)?; // /*
-                    let content = self.consume_until("*/")?;
-                    self.consume_n(2)?; // */
-                    TokenType::Comment(content, "/*".to_string(), Some("*/".to_string()))
-                }
-                _ => {
-                    self.consume()?;
-                    TokenType::Text("/".to_string())
-                }
-            },
-
-            c if c.is_whitespace() => {
-                if c == '\n' || c == '\r' {
-                    self.consume()?; // \r or \n
-                    if c == '\r' && self.peek()? == '\n' {
-                        self.consume()?; // \n of \r\n
-                    }
-                    TokenType::Newline
-                } else {
-                    self.consume()?; // Consume the first whitespace
-                    while !self.is_at_end() && self.peek()?.is_whitespace() {
-                        if self.peek()? == '\n' || self.peek()? == '\r' {
-                            break;
-                        }
-                        self.consume()?;
-                    }
-                    let whitespace_count = self.current - self.start;
-                    TokenType::Whitespace(whitespace_count)
-                }
-            }
-
-            _ => {
-                let mut text = String::new();
-                while !self.is_at_end() {
-                    let c = self.peek()?;
-                    if c == '{' || c == '<' || c == '\n' {
-                        break;
-                    }
-                    text.push(c);
-                    self.consume()?;
-                }
-                TokenType::Text(text)
-            }
-        };
-
-        let token = Token::new(token_type, self.line, Some(self.start));
-
-        match self.peek_previous()? {
-            '\n' => self.line += 1,
-            '\r' => {
-                self.line += 1;
-                if self.peek()? == '\n' {
-                    self.current += 1;
-                }
-            }
-            _ => {}
-        }
-
-        Ok(token)
-    }
-
     fn peek(&self) -> Result<char, LexerError> {
         self.peek_at(0)
     }

crates/djls-template-ast/src/lib.rs

@@ -1,7 +1,44 @@
 mod ast;
+mod error;
 mod lexer;
 mod parser;
+mod tagspecs;
 mod tokens;
 
-pub use ast::Ast;
-pub use parser::Parser;
+use ast::NodeList;
+pub use error::{to_lsp_diagnostic, QuickFix, TemplateError};
+
+use lexer::Lexer;
+pub use parser::{Parser, ParserError};
+
+/// Parses a Django template and returns the AST and any parsing errors.
+///
+/// - `source`: The template source code as a `&str`.
+/// - `tag_specs`: Optional `TagSpecs` to use for parsing (e.g., custom tags).
+///
+/// Returns a `Result` containing a tuple of `(Ast, Vec<ParserError>)` on success,
+/// or a `ParserError` on failure.
+pub fn parse_template(source: &str) -> Result<(NodeList, Vec<TemplateError>), TemplateError> {
+    let tokens = Lexer::new(source)
+        .tokenize()
+        .map_err(|e| TemplateError::Lexer(e.to_string()))?;
+
+    // let tag_specs = match tag_specs {
+    //     Some(specs) => specs.clone(),
+    //     None => TagSpecs::load_builtin_specs()
+    //         .map_err(|e| TemplateError::Config(format!("Failed to load builtin specs: {}", e)))?,
+    // };
+
+    let mut parser = Parser::new(tokens);
+    let (nodelist, parser_errors) = parser
+        .parse()
+        .map_err(|e| TemplateError::Parser(e.to_string()))?;
+
+    // Convert parser errors to TemplateError
+    let all_errors = parser_errors
+        .into_iter()
+        .map(|e| TemplateError::Parser(e.to_string()))
+        .collect::<Vec<_>>();
+
+    Ok((nodelist, all_errors))
+}

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap

@@ -0,0 +1,17 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: "<!-- HTML comment -->"
+      span:
+        start: 0
+        length: 21
+  - Comment:
+      content: Django comment
+      span:
+        start: 21
+        length: 14
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_complex_if_elif.snap

@@ -0,0 +1,52 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Tag:
+      name: if
+      bits:
+        - x
+        - ">"
+        - "0"
+      span:
+        start: 0
+        length: 8
+  - Text:
+      content: Positive
+      span:
+        start: 14
+        length: 8
+  - Tag:
+      name: elif
+      bits:
+        - x
+        - "<"
+        - "0"
+      span:
+        start: 22
+        length: 10
+  - Text:
+      content: Negative
+      span:
+        start: 38
+        length: 8
+  - Tag:
+      name: else
+      bits: []
+      span:
+        start: 46
+        length: 4
+  - Text:
+      content: Zero
+      span:
+        start: 56
+        length: 4
+  - Tag:
+      name: endif
+      bits: []
+      span:
+        start: 60
+        length: 5
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_for_block.snap

@@ -0,0 +1,39 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Tag:
+      name: for
+      bits:
+        - item
+        - in
+        - items
+      span:
+        start: 0
+        length: 17
+  - Variable:
+      var: item
+      filters: []
+      span:
+        start: 23
+        length: 4
+  - Tag:
+      name: empty
+      bits: []
+      span:
+        start: 33
+        length: 5
+  - Text:
+      content: No items
+      span:
+        start: 44
+        length: 8
+  - Tag:
+      name: endfor
+      bits: []
+      span:
+        start: 52
+        length: 6
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_if_block.snap

@@ -0,0 +1,25 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Tag:
+      name: if
+      bits:
+        - user.is_authenticated
+      span:
+        start: 0
+        length: 24
+  - Text:
+      content: Welcome
+      span:
+        start: 30
+        length: 7
+  - Tag:
+      name: endif
+      bits: []
+      span:
+        start: 37
+        length: 5
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_tag_assignment.snap

@@ -0,0 +1,16 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Tag:
+      name: url
+      bits:
+        - "'view-name'"
+        - as
+        - view
+      span:
+        start: 0
+        length: 23
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_variable.snap

@@ -0,0 +1,13 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Variable:
+      var: user.name
+      filters: []
+      span:
+        start: 0
+        length: 9
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_variable_with_filter.snap

@@ -0,0 +1,14 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Variable:
+      var: user.name
+      filters:
+        - title
+      span:
+        start: 0
+        length: 15
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_filter_chains.snap

@@ -0,0 +1,16 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Variable:
+      var: value
+      filters:
+        - "default:'nothing'"
+        - title
+        - upper
+      span:
+        start: 0
+        length: 35
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap

@@ -0,0 +1,148 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: "Welcome,"
+      span:
+        start: 0
+        length: 8
+  - Tag:
+      name: if
+      bits:
+        - user.is_authenticated
+      span:
+        start: 9
+        length: 24
+  - Variable:
+      var: user.name
+      filters:
+        - title
+        - "default:'Guest'"
+      span:
+        start: 44
+        length: 31
+  - Tag:
+      name: for
+      bits:
+        - group
+        - in
+        - user.groups
+      span:
+        start: 86
+        length: 24
+  - Tag:
+      name: if
+      bits:
+        - forloop.first
+      span:
+        start: 125
+        length: 16
+  - Text:
+      content: (
+      span:
+        start: 147
+        length: 1
+  - Tag:
+      name: endif
+      bits: []
+      span:
+        start: 148
+        length: 5
+  - Variable:
+      var: group.name
+      filters: []
+      span:
+        start: 168
+        length: 10
+  - Tag:
+      name: if
+      bits:
+        - not
+        - forloop.last
+      span:
+        start: 193
+        length: 19
+  - Text:
+      content: ","
+      span:
+        start: 218
+        length: 1
+  - Tag:
+      name: endif
+      bits: []
+      span:
+        start: 220
+        length: 5
+  - Tag:
+      name: if
+      bits:
+        - forloop.last
+      span:
+        start: 240
+        length: 15
+  - Text:
+      content: )
+      span:
+        start: 261
+        length: 1
+  - Tag:
+      name: endif
+      bits: []
+      span:
+        start: 262
+        length: 5
+  - Tag:
+      name: empty
+      bits: []
+      span:
+        start: 278
+        length: 5
+  - Text:
+      content: (no groups)
+      span:
+        start: 298
+        length: 11
+  - Tag:
+      name: endfor
+      bits: []
+      span:
+        start: 314
+        length: 6
+  - Tag:
+      name: else
+      bits: []
+      span:
+        start: 327
+        length: 4
+  - Text:
+      content: Guest
+      span:
+        start: 342
+        length: 5
+  - Tag:
+      name: endif
+      bits: []
+      span:
+        start: 348
+        length: 5
+  - Text:
+      content: "!"
+      span:
+        start: 359
+        length: 1
+line_offsets:
+  - 0
+  - 40
+  - 82
+  - 117
+  - 160
+  - 185
+  - 232
+  - 274
+  - 290
+  - 310
+  - 327
+  - 338
+  - 348

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_nested_for_if.snap

@@ -0,0 +1,41 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Tag:
+      name: for
+      bits:
+        - item
+        - in
+        - items
+      span:
+        start: 0
+        length: 17
+  - Tag:
+      name: if
+      bits:
+        - item.active
+      span:
+        start: 23
+        length: 14
+  - Variable:
+      var: item.name
+      filters: []
+      span:
+        start: 43
+        length: 9
+  - Tag:
+      name: endif
+      bits: []
+      span:
+        start: 58
+        length: 5
+  - Tag:
+      name: endfor
+      bits: []
+      span:
+        start: 69
+        length: 6
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap

@@ -0,0 +1,28 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: "<div class=\"container\">"
+      span:
+        start: 0
+        length: 23
+  - Text:
+      content: "<h1>Header</h1>"
+      span:
+        start: 28
+        length: 15
+line_offsets:
+  - 0
+  - 24
+  - 44
+  - 54
+  - 106
+  - 145
+  - 159
+  - 219
+  - 251
+  - 287
+  - 308
+  - 341

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_for.snap

@@ -0,0 +1,22 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Tag:
+      name: for
+      bits:
+        - item
+        - in
+        - items
+      span:
+        start: 0
+        length: 17
+  - Variable:
+      var: item.name
+      filters: []
+      span:
+        start: 23
+        length: 9
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_if.snap

@@ -0,0 +1,19 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Tag:
+      name: if
+      bits:
+        - user.is_authenticated
+      span:
+        start: 0
+        length: 24
+  - Text:
+      content: Welcome
+      span:
+        start: 30
+        length: 7
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_html_tag.snap

@@ -0,0 +1,12 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: "<div>"
+      span:
+        start: 0
+        length: 5
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_script.snap

@@ -0,0 +1,12 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: "<script>console.log('test');"
+      span:
+        start: 0
+        length: 28
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_style.snap

@@ -0,0 +1,12 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: "<style>body { color: blue;"
+      span:
+        start: 0
+        length: 26
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap

@@ -0,0 +1,206 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: "<!DOCTYPE html>"
+      span:
+        start: 0
+        length: 15
+  - Text:
+      content: "<html>"
+      span:
+        start: 16
+        length: 6
+  - Text:
+      content: "<head>"
+      span:
+        start: 27
+        length: 6
+  - Text:
+      content: "<style type=\"text/css\">"
+      span:
+        start: 42
+        length: 23
+  - Text:
+      content: /* Style header */
+      span:
+        start: 78
+        length: 18
+  - Text:
+      content: ".header { color: blue; }"
+      span:
+        start: 109
+        length: 24
+  - Text:
+      content: "</style>"
+      span:
+        start: 142
+        length: 8
+  - Text:
+      content: "<script type=\"text/javascript\">"
+      span:
+        start: 159
+        length: 31
+  - Text:
+      content: // Init app
+      span:
+        start: 203
+        length: 11
+  - Text:
+      content: "const app = {"
+      span:
+        start: 227
+        length: 13
+  - Text:
+      content: /* Config */
+      span:
+        start: 257
+        length: 12
+  - Text:
+      content: "debug: true"
+      span:
+        start: 286
+        length: 11
+  - Text:
+      content: "};"
+      span:
+        start: 310
+        length: 2
+  - Text:
+      content: "</script>"
+      span:
+        start: 321
+        length: 9
+  - Text:
+      content: "</head>"
+      span:
+        start: 335
+        length: 7
+  - Text:
+      content: "<body>"
+      span:
+        start: 347
+        length: 6
+  - Text:
+      content: "<!-- Header section -->"
+      span:
+        start: 362
+        length: 23
+  - Text:
+      content: "<div class=\"header\" id=\"main\" data-value=\"123\" disabled>"
+      span:
+        start: 394
+        length: 56
+  - Tag:
+      name: if
+      bits:
+        - user.is_authenticated
+      span:
+        start: 463
+        length: 24
+  - Comment:
+      content: Welcome message
+      span:
+        start: 510
+        length: 15
+  - Text:
+      content: "<h1>Welcome,"
+      span:
+        start: 548
+        length: 12
+  - Variable:
+      var: user.name
+      filters:
+        - title
+        - "default:'Guest'"
+      span:
+        start: 561
+        length: 31
+  - Text:
+      content: "!</h1>"
+      span:
+        start: 598
+        length: 6
+  - Tag:
+      name: if
+      bits:
+        - user.is_staff
+      span:
+        start: 621
+        length: 16
+  - Text:
+      content: "<span>Admin</span>"
+      span:
+        start: 664
+        length: 18
+  - Tag:
+      name: else
+      bits: []
+      span:
+        start: 699
+        length: 4
+  - Text:
+      content: "<span>User</span>"
+      span:
+        start: 730
+        length: 17
+  - Tag:
+      name: endif
+      bits: []
+      span:
+        start: 764
+        length: 5
+  - Tag:
+      name: endif
+      bits: []
+      span:
+        start: 788
+        length: 5
+  - Text:
+      content: "</div>"
+      span:
+        start: 808
+        length: 6
+  - Text:
+      content: "</body>"
+      span:
+        start: 819
+        length: 7
+  - Text:
+      content: "</html>"
+      span:
+        start: 827
+        length: 7
+line_offsets:
+  - 0
+  - 16
+  - 23
+  - 34
+  - 66
+  - 97
+  - 134
+  - 151
+  - 191
+  - 215
+  - 241
+  - 270
+  - 298
+  - 313
+  - 331
+  - 343
+  - 354
+  - 386
+  - 451
+  - 494
+  - 532
+  - 605
+  - 644
+  - 683
+  - 710
+  - 748
+  - 776
+  - 800
+  - 815
+  - 827

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_doctype.snap

@@ -0,0 +1,12 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: "<!DOCTYPE html>"
+      span:
+        start: 0
+        length: 15
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_tag.snap

@@ -0,0 +1,12 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: "<div class=\"container\">Hello</div>"
+      span:
+        start: 0
+        length: 34
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_void.snap

@@ -0,0 +1,12 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: "<input type=\"text\" />"
+      span:
+        start: 0
+        length: 21
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_comments.snap

@@ -1,28 +0,0 @@
----
-source: crates/djls-template-ast/src/parser.rs
-expression: ast
----
-nodes:
-  - Html:
-      Comment: HTML comment
-  - Django:
-      Comment: Django comment
-  - Script:
-      Element:
-        attributes:
-          script: Boolean
-        children:
-          - Script:
-              Comment:
-                content: JS single line
-                kind: SingleLine
-          - Script:
-              Comment:
-                content: "JS multi\n        line"
-                kind: MultiLine
-  - Style:
-      Element:
-        attributes: {}
-        children:
-          - Style:
-              Comment: CSS comment

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_django_block.snap

@@ -1,13 +0,0 @@
----
-source: crates/djls-template-ast/src/parser.rs
-expression: ast
----
-nodes:
-  - Django:
-      Tag:
-        kind: If
-        bits:
-          - if
-          - user.is_staff
-        children:
-          - Text: Admin

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_django_variable.snap

@@ -1,16 +0,0 @@
----
-source: crates/djls-template-ast/src/parser.rs
-expression: ast
----
-nodes:
-  - Django:
-      Variable:
-        bits:
-          - user
-          - name
-        filters:
-          - name: default
-            arguments:
-              - Anonymous
-          - name: title
-            arguments: []

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_full.snap

@@ -1,106 +0,0 @@
----
-source: crates/djls-template-ast/src/parser.rs
-expression: ast
----
-nodes:
-  - Html:
-      Doctype: "!DOCTYPE"
-  - Html:
-      Element:
-        tag_name: html
-        attributes: {}
-        children:
-          - Html:
-              Element:
-                tag_name: head
-                attributes: {}
-                children:
-                  - Style:
-                      Element:
-                        attributes:
-                          type:
-                            Value: text/css
-                        children:
-                          - Style:
-                              Comment: Style header
-                          - Text: ".header "
-                          - Text: "{"
-                          - Text: "color: blue; }"
-                  - Script:
-                      Element:
-                        attributes:
-                          script: Boolean
-                          type:
-                            Value: text/javascript
-                        children:
-                          - Script:
-                              Comment:
-                                content: Init app
-                                kind: SingleLine
-                          - Text: "const app = "
-                          - Text: "{"
-                          - Script:
-                              Comment:
-                                content: Config
-                                kind: MultiLine
-                          - Text: "debug: true"
-                          - Text: "};"
-          - Html:
-              Element:
-                tag_name: body
-                attributes: {}
-                children:
-                  - Html:
-                      Comment: Header section
-                  - Html:
-                      Element:
-                        tag_name: div
-                        attributes:
-                          class:
-                            Value: header
-                          data-value:
-                            Value: "123"
-                          disabled: Boolean
-                          id:
-                            Value: main
-                        children:
-                          - Django:
-                              Tag:
-                                kind: If
-                                bits:
-                                  - if
-                                  - user.is_authenticated
-                                children:
-                                  - Django:
-                                      Comment: Welcome message
-                                  - Html:
-                                      Element:
-                                        tag_name: h1
-                                        attributes: {}
-                                        children:
-                                          - Text: "Welcome, "
-                                          - Django:
-                                              Variable:
-                                                bits:
-                                                  - user
-                                                  - name
-                                                filters:
-                                                  - name: default
-                                                    arguments:
-                                                      - Guest
-                                                  - name: title
-                                                    arguments: []
-                                          - Text: "!"
-                                  - Django:
-                                      Tag:
-                                        kind: If
-                                        bits:
-                                          - if
-                                          - user.is_staff
-                                        children:
-                                          - Html:
-                                              Element:
-                                                tag_name: span
-                                                attributes: {}
-                                                children:
-                                                  - Text: Admin

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_html_doctype.snap

@@ -1,7 +0,0 @@
----
-source: crates/djls-template-ast/src/parser.rs
-expression: ast
----
-nodes:
-  - Html:
-      Doctype: "!DOCTYPE"

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_html_tag.snap

@@ -1,15 +0,0 @@
----
-source: crates/djls-template-ast/src/parser.rs
-expression: ast
----
-nodes:
-  - Html:
-      Element:
-        tag_name: div
-        attributes:
-          class:
-            Value: container
-          disabled: Boolean
-          id:
-            Value: main
-        children: []

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_html_void.snap

@@ -1,11 +0,0 @@
----
-source: crates/djls-template-ast/src/parser.rs
-expression: ast
----
-nodes:
-  - Html:
-      Void:
-        tag_name: img
-        attributes:
-          src:
-            Value: example.png

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_script.snap

@@ -1,22 +0,0 @@
----
-source: crates/djls-template-ast/src/parser.rs
-expression: ast
----
-nodes:
-  - Script:
-      Element:
-        attributes:
-          script: Boolean
-          type:
-            Value: text/javascript
-        children:
-          - Script:
-              Comment:
-                content: Single line comment
-                kind: SingleLine
-          - Text: const x = 1;
-          - Script:
-              Comment:
-                content: "Multi-line\n        comment"
-                kind: MultiLine
-          - Text: console.log(x);

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__parse_style.snap

@@ -1,17 +0,0 @@
----
-source: crates/djls-template-ast/src/parser.rs
-expression: ast
----
-nodes:
-  - Style:
-      Element:
-        attributes:
-          type:
-            Value: text/css
-        children:
-          - Style:
-              Comment: Header styles
-          - Text: ".header "
-          - Text: "{"
-          - Text: "color: blue;"
-          - Text: "}"

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__script__parse_script.snap

@@ -0,0 +1,42 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: "<script type=\"text/javascript\">"
+      span:
+        start: 0
+        length: 31
+  - Text:
+      content: // Single line comment
+      span:
+        start: 36
+        length: 22
+  - Text:
+      content: const x = 1;
+      span:
+        start: 63
+        length: 12
+  - Text:
+      content: "/* Multi-line\n        comment */"
+      span:
+        start: 80
+        length: 32
+  - Text:
+      content: console.log(x);
+      span:
+        start: 117
+        length: 15
+  - Text:
+      content: "</script>"
+      span:
+        start: 133
+        length: 9
+line_offsets:
+  - 0
+  - 32
+  - 59
+  - 76
+  - 113
+  - 133

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__style__parse_style.snap

@@ -0,0 +1,42 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: "<style type=\"text/css\">"
+      span:
+        start: 0
+        length: 23
+  - Text:
+      content: /* Header styles */
+      span:
+        start: 28
+        length: 19
+  - Text:
+      content: ".header {"
+      span:
+        start: 52
+        length: 9
+  - Text:
+      content: "color: blue;"
+      span:
+        start: 70
+        length: 12
+  - Text:
+      content: "}"
+      span:
+        start: 87
+        length: 1
+  - Text:
+      content: "</style>"
+      span:
+        start: 89
+        length: 8
+line_offsets:
+  - 0
+  - 24
+  - 48
+  - 62
+  - 83
+  - 89

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace.snap

@@ -0,0 +1,12 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: hello
+      span:
+        start: 5
+        length: 5
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace_newline.snap

@@ -0,0 +1,13 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: hello
+      span:
+        start: 6
+        length: 5
+line_offsets:
+  - 0
+  - 1

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace.snap

@@ -0,0 +1,12 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: hello
+      span:
+        start: 0
+        length: 5
+line_offsets:
+  - 0

crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace_newline.snap

@@ -0,0 +1,13 @@
+---
+source: crates/djls-template-ast/src/parser.rs
+expression: nodelist
+---
+nodes:
+  - Text:
+      content: hello
+      span:
+        start: 0
+        length: 5
+line_offsets:
+  - 0
+  - 11

crates/djls-template-ast/src/tagspecs.rs

@@ -0,0 +1,311 @@
+use anyhow::Result;
+use serde::Deserialize;
+use std::collections::HashMap;
+use std::fs;
+use std::path::Path;
+use thiserror::Error;
+use toml::Value;
+
+#[derive(Debug, Error)]
+pub enum TagSpecError {
+    #[error("Failed to read file: {0}")]
+    Io(#[from] std::io::Error),
+    #[error("Failed to parse TOML: {0}")]
+    Toml(#[from] toml::de::Error),
+    #[error("Failed to extract specs: {0}")]
+    Extract(String),
+    #[error(transparent)]
+    Other(#[from] anyhow::Error),
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct TagSpecs(HashMap<String, TagSpec>);
+
+impl TagSpecs {
+    pub fn get(&self, key: &str) -> Option<&TagSpec> {
+        self.0.get(key)
+    }
+
+    /// Load specs from a TOML file, looking under the specified table path
+    fn load_from_toml(path: &Path, table_path: &[&str]) -> Result<Self, anyhow::Error> {
+        let content = fs::read_to_string(path)?;
+        let value: Value = toml::from_str(&content)?;
+
+        // Navigate to the specified table
+        let table = table_path
+            .iter()
+            .try_fold(&value, |current, &key| {
+                current
+                    .get(key)
+                    .ok_or_else(|| anyhow::anyhow!("Missing table: {}", key))
+            })
+            .unwrap_or(&value);
+
+        let mut specs = HashMap::new();
+        TagSpec::extract_specs(table, None, &mut specs)
+            .map_err(|e| TagSpecError::Extract(e.to_string()))?;
+        Ok(TagSpecs(specs))
+    }
+
+    /// Load specs from a user's project directory
+    pub fn load_user_specs(project_root: &Path) -> Result<Self, anyhow::Error> {
+        // List of config files to try, in priority order
+        let config_files = ["djls.toml", ".djls.toml", "pyproject.toml"];
+
+        for &file in &config_files {
+            let path = project_root.join(file);
+            if path.exists() {
+                return match file {
+                    "pyproject.toml" => Self::load_from_toml(&path, &["tool", "djls", "tagspecs"]),
+                    _ => Self::load_from_toml(&path, &["tagspecs"]), // Root level for other files
+                };
+            }
+        }
+        Ok(Self::default())
+    }
+
+    /// Load builtin specs from the crate's tagspecs directory
+    pub fn load_builtin_specs() -> Result<Self, anyhow::Error> {
+        let specs_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tagspecs");
+        let mut specs = HashMap::new();
+
+        for entry in fs::read_dir(&specs_dir)? {
+            let entry = entry?;
+            let path = entry.path();
+            if path.extension().and_then(|ext| ext.to_str()) == Some("toml") {
+                let file_specs = Self::load_from_toml(&path, &[])?;
+                specs.extend(file_specs.0);
+            }
+        }
+
+        Ok(TagSpecs(specs))
+    }
+
+    /// Merge another TagSpecs into this one, with the other taking precedence
+    pub fn merge(&mut self, other: TagSpecs) -> &mut Self {
+        self.0.extend(other.0);
+        self
+    }
+
+    /// Load both builtin and user specs, with user specs taking precedence
+    pub fn load_all(project_root: &Path) -> Result<Self, anyhow::Error> {
+        let mut specs = Self::load_builtin_specs()?;
+        let user_specs = Self::load_user_specs(project_root)?;
+        Ok(specs.merge(user_specs).clone())
+    }
+}
+
+#[derive(Debug, Clone, Deserialize)]
+pub struct TagSpec {
+    #[serde(rename = "type")]
+    pub tag_type: TagType,
+    pub closing: Option<String>,
+    #[serde(default)]
+    pub branches: Option<Vec<String>>,
+    pub args: Option<Vec<ArgSpec>>,
+}
+
+impl TagSpec {
+    fn extract_specs(
+        value: &Value,
+        prefix: Option<&str>,
+        specs: &mut HashMap<String, TagSpec>,
+    ) -> Result<(), String> {
+        // Try to deserialize as a tag spec first
+        match TagSpec::deserialize(value.clone()) {
+            Ok(tag_spec) => {
+                let name = prefix.map_or_else(String::new, |p| {
+                    p.split('.').last().unwrap_or(p).to_string()
+                });
+                specs.insert(name, tag_spec);
+            }
+            Err(_) => {
+                // Not a tag spec, try recursing into any table values
+                for (key, value) in value.as_table().iter().flat_map(|t| t.iter()) {
+                    let new_prefix = match prefix {
+                        None => key.clone(),
+                        Some(p) => format!("{}.{}", p, key),
+                    };
+                    Self::extract_specs(value, Some(&new_prefix), specs)?;
+                }
+            }
+        }
+        Ok(())
+    }
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum TagType {
+    Container,
+    Inclusion,
+    Single,
+}
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct ArgSpec {
+    pub name: String,
+    pub required: bool,
+    #[serde(default)]
+    pub allowed_values: Option<Vec<String>>,
+    #[serde(default)]
+    pub is_kwarg: bool,
+}
+
+impl ArgSpec {
+    pub fn is_placeholder(arg: &str) -> bool {
+        arg.starts_with('{') && arg.ends_with('}')
+    }
+
+    pub fn get_placeholder_name(arg: &str) -> Option<&str> {
+        if Self::is_placeholder(arg) {
+            Some(&arg[1..arg.len() - 1])
+        } else {
+            None
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_can_load_builtins() -> Result<(), anyhow::Error> {
+        let specs = TagSpecs::load_builtin_specs()?;
+
+        assert!(!specs.0.is_empty(), "Should have loaded at least one spec");
+
+        for name in specs.0.keys() {
+            assert!(!name.is_empty(), "Tag name should not be empty");
+        }
+        Ok(())
+    }
+
+    #[test]
+    fn test_builtin_django_tags() -> Result<(), anyhow::Error> {
+        let specs = TagSpecs::load_builtin_specs()?;
+
+        let expected_tags = [
+            "autoescape",
+            "block",
+            "comment",
+            "cycle",
+            "debug",
+            "extends",
+            "filter",
+            "for",
+            "firstof",
+            "if",
+            "include",
+            "load",
+            "now",
+            "spaceless",
+            "templatetag",
+            "url",
+            "verbatim",
+            "with",
+        ];
+        let missing_tags = [
+            "csrf_token",
+            "ifchanged",
+            "lorem",
+            "querystring", // 5.1
+            "regroup",
+            "resetcycle",
+            "widthratio",
+        ];
+
+        for tag in expected_tags {
+            assert!(specs.get(tag).is_some(), "{} tag should be present", tag);
+        }
+
+        for tag in missing_tags {
+            assert!(
+                specs.get(tag).is_none(),
+                "{} tag should not be present yet",
+                tag
+            );
+        }
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_user_defined_tags() -> Result<(), anyhow::Error> {
+        let dir = tempfile::tempdir()?;
+        let root = dir.path();
+
+        let pyproject_content = r#"
+[tool.djls.template.tags.mytag]
+type = "container"
+closing = "endmytag"
+branches = ["mybranch"]
+args = [{ name = "myarg", required = true }]
+"#;
+        fs::write(root.join("pyproject.toml"), pyproject_content)?;
+
+        let specs = TagSpecs::load_all(root)?;
+
+        let if_tag = specs.get("if").expect("if tag should be present");
+        assert_eq!(if_tag.tag_type, TagType::Container);
+
+        let my_tag = specs.get("mytag").expect("mytag should be present");
+        assert_eq!(my_tag.tag_type, TagType::Container);
+        assert_eq!(my_tag.closing, Some("endmytag".to_string()));
+
+        let branches = my_tag
+            .branches
+            .as_ref()
+            .expect("mytag should have branches");
+        assert!(branches.iter().any(|b| b == "mybranch"));
+
+        let args = my_tag.args.as_ref().expect("mytag should have args");
+        let arg = &args[0];
+        assert_eq!(arg.name, "myarg");
+        assert!(arg.required);
+
+        dir.close()?;
+        Ok(())
+    }
+
+    #[test]
+    fn test_config_file_priority() -> Result<(), anyhow::Error> {
+        let dir = tempfile::tempdir()?;
+        let root = dir.path();
+
+        let djls_content = r#"
+[mytag1]
+type = "container"
+closing = "endmytag1"
+"#;
+        fs::write(root.join("djls.toml"), djls_content)?;
+
+        let pyproject_content = r#"
+[tool.djls.template.tags]
+mytag2.type = "container"
+mytag2.closing = "endmytag2"
+"#;
+        fs::write(root.join("pyproject.toml"), pyproject_content)?;
+
+        let specs = TagSpecs::load_user_specs(root)?;
+
+        assert!(specs.get("mytag1").is_some(), "mytag1 should be present");
+        assert!(
+            specs.get("mytag2").is_none(),
+            "mytag2 should not be present"
+        );
+
+        fs::remove_file(root.join("djls.toml"))?;
+        let specs = TagSpecs::load_user_specs(root)?;
+
+        assert!(
+            specs.get("mytag1").is_none(),
+            "mytag1 should not be present"
+        );
+        assert!(specs.get("mytag2").is_some(), "mytag2 should be present");
+
+        dir.close()?;
+        Ok(())
+    }
+}

crates/djls-template-ast/src/tokens.rs

@@ -1,5 +1,4 @@
 use serde::Serialize;
-use std::fmt;
 use std::ops::{Deref, DerefMut};
 
 #[derive(Clone, Debug, Serialize, PartialEq)]
@@ -33,57 +32,10 @@ impl TokenType {
             | TokenType::StyleTagOpen(s)
             | TokenType::StyleTagClose(s)
             | TokenType::Text(s) => Some(s.len()),
-            TokenType::Comment(content, start, end) => {
-                Some(content.len() + start.len() + end.as_ref().map_or(0, |e| e.len()))
-            }
-            TokenType::Whitespace(len) => Some(len.clone()),
+            TokenType::Comment(content, _, _) => Some(content.len()),
+            TokenType::Whitespace(n) => Some(*n),
             TokenType::Newline => Some(1),
-            TokenType::Eof => None,
-        }
-    }
-
-    pub fn lexeme(&self) -> &str {
-        match self {
-            TokenType::DjangoBlock(s)
-            | TokenType::DjangoVariable(s)
-            | TokenType::HtmlTagOpen(s)
-            | TokenType::HtmlTagClose(s)
-            | TokenType::HtmlTagVoid(s)
-            | TokenType::ScriptTagOpen(s)
-            | TokenType::ScriptTagClose(s)
-            | TokenType::StyleTagOpen(s)
-            | TokenType::StyleTagClose(s)
-            | TokenType::Text(s) => s,
-            TokenType::Comment(content, _, _) => content, // Just return the content
-            TokenType::Whitespace(_) => " ",
-            TokenType::Newline => "\n",
-            TokenType::Eof => "",
-        }
-    }
-}
-
-impl fmt::Display for TokenType {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        use TokenType::*;
-
-        match self {
-            Comment(content, start, end) => match end {
-                Some(end) => write!(f, "{}{}{}", start, content, end),
-                None => write!(f, "{}{}", start, content),
-            },
-            DjangoBlock(s) => write!(f, "{{% {} %}}", s),
-            DjangoVariable(s) => write!(f, "{{{{ {} }}}}", s),
-            Eof => Ok(()),
-            HtmlTagOpen(s) => write!(f, "<{}>", s),
-            HtmlTagClose(s) => write!(f, "</{}>", s),
-            HtmlTagVoid(s) => write!(f, "<{}/>", s),
-            Newline => f.write_str("\n"),
-            ScriptTagOpen(s) => write!(f, "<script{}>", s),
-            ScriptTagClose(_) => f.write_str("</script>"),
-            StyleTagOpen(s) => write!(f, "<style{}>", s),
-            StyleTagClose(_) => f.write_str("</style>"),
-            Text(s) => f.write_str(s),
-            Whitespace(len) => f.write_str(&" ".repeat(*len)),
+            TokenType::Eof => Some(0),
         }
     }
 }
@@ -104,21 +56,61 @@ impl Token {
         }
     }
 
-    pub fn lexeme_from_source<'a>(&self, source: &'a str) -> Option<&'a str> {
-        match (self.start, self.token_type.len()) {
-            (Some(start), Some(len)) => Some(&source[start..start + len]),
-            _ => None,
+    pub fn lexeme(&self) -> String {
+        match &self.token_type {
+            TokenType::Comment(_, start, end) => match end {
+                Some(end) => format!("{} {} {}", start, self.content(), end),
+                None => format!("{} {}", start, self.content()),
+            },
+            TokenType::DjangoBlock(_) => format!("{{% {} %}}", self.content()),
+            TokenType::DjangoVariable(_) => format!("{{{{ {} }}}}", self.content()),
+            TokenType::Eof => String::new(),
+            TokenType::HtmlTagOpen(_)
+            | TokenType::ScriptTagOpen(_)
+            | TokenType::StyleTagOpen(_) => format!("<{}>", self.content()),
+            TokenType::HtmlTagClose(_)
+            | TokenType::StyleTagClose(_)
+            | TokenType::ScriptTagClose(_) => format!("</{}>", self.content()),
+            TokenType::HtmlTagVoid(_) => format!("<{}/>", self.content()),
+            TokenType::Newline | TokenType::Text(_) | TokenType::Whitespace(_) => self.content(),
         }
     }
 
-    pub fn lexeme(&self) -> &str {
-        self.token_type.lexeme()
+    pub fn content(&self) -> String {
+        match &self.token_type {
+            TokenType::Comment(s, _, _)
+            | TokenType::DjangoBlock(s)
+            | TokenType::DjangoVariable(s)
+            | TokenType::Text(s)
+            | TokenType::HtmlTagOpen(s)
+            | TokenType::HtmlTagClose(s)
+            | TokenType::HtmlTagVoid(s)
+            | TokenType::ScriptTagOpen(s)
+            | TokenType::ScriptTagClose(s)
+            | TokenType::StyleTagOpen(s)
+            | TokenType::StyleTagClose(s) => s.to_string(),
+            TokenType::Whitespace(len) => " ".repeat(*len),
+            TokenType::Newline => "\n".to_string(),
+            TokenType::Eof => "".to_string(),
+        }
     }
 
     pub fn token_type(&self) -> &TokenType {
         &self.token_type
     }
 
+    pub fn line(&self) -> &usize {
+        &self.line
+    }
+
+    pub fn start(&self) -> Option<u32> {
+        self.start.map(|s| s as u32)
+    }
+
+    pub fn length(&self) -> Option<u32> {
+        self.token_type.len().map(|l| l as u32)
+    }
+
     pub fn is_token_type(&self, token_type: &TokenType) -> bool {
         &self.token_type == token_type
     }

crates/djls-template-ast/tagspecs/README.md

@@ -0,0 +1,89 @@
+# TagSpecs
+
+## Schema
+
+Tag Specifications (TagSpecs) define how tags are parsed and understood. They allow the parser to handle custom tags without hard-coding them.
+
+```toml
+[package.module.path.tag_name]  # Path where tag is registered, e.g., django.template.defaulttags
+type = "container" | "inclusion" | "single"
+closing = "closing_tag_name"        # For block tags that require a closing tag
+branches = ["branch_tag_name", ...] # For block tags that support branches
+
+# Arguments can be positional (matched by order) or keyword (matched by name)
+args = [
+    # Positional argument (position inferred from array index)
+    { name = "setting", required = true, allowed_values = ["on", "off"] },
+    # Keyword argument
+    { name = "key", required = false, is_kwarg = true }
+]
+```
+
+The `name` field in args should match the internal name used in Django's node implementation. For example, the `autoescape` tag's argument is stored as `setting` in Django's `AutoEscapeControlNode`.
+
+## Tag Types
+
+- `container`: Tags that wrap content and require a closing tag
+
+  ```django
+  {% if condition %}content{% endif %}
+  {% for item in items %}content{% endfor %}
+  ```
+
+- `inclusion`: Tags that include or extend templates.
+
+  ```django
+  {% extends "base.html" %}
+  {% include "partial.html" %}
+  ```
+
+- `single`: Single tags that don't wrap content
+
+  ```django
+  {% csrf_token %}
+  ```
+
+## Configuration
+
+- **Built-in TagSpecs**: The parser includes TagSpecs for Django's built-in tags and popular third-party tags.
+- **User-defined TagSpecs**: Users can expand or override TagSpecs via `pyproject.toml` or `djls.toml` files in their project, allowing custom tags and configurations to be seamlessly integrated.
+
+## Examples
+
+### If Tag
+
+```toml
+[django.template.defaulttags.if]
+type = "container"
+closing = "endif"
+branches = ["elif", "else"]
+args = [{ name = "condition", required = true }]
+```
+
+### Include Tag
+
+```toml
+[django.template.defaulttags.includes]
+type = "inclusion"
+args = [{ name = "template_name", required = true }]
+```
+
+### Autoescape Tag
+
+```toml
+[django.template.defaulttags.autoescape]
+type = "container"
+closing = "endautoescape"
+args = [{ name = "setting", required = true, allowed_values = ["on", "off"] }]
+```
+
+### Custom Tag with Kwargs
+
+```toml
+[my_module.templatetags.my_tags.my_custom_tag]
+type = "single"
+args = [
+    { name = "arg1", required = true },
+    { name = "kwarg1", required = false, is_kwarg = true }
+]
+```

crates/djls-template-ast/tagspecs/django.toml

@@ -0,0 +1,104 @@
+[django.template.defaulttags.autoescape]
+args = [{ name = "setting", required = true, allowed_values = ["on", "off"] }]
+closing = "endautoescape"
+type = "container"
+
+[django.template.defaulttags.block]
+closing = "endblock"
+type = "container"
+
+[django.template.defaulttags.comment]
+closing = "endcomment"
+type = "container"
+
+[django.template.defaulttags.cycle]
+args = [
+  { name = "cyclevars", required = true },
+  { name = "variable_name", required = false, is_kwarg = true },
+]
+type = "single"
+
+[django.template.defaulttags.debug]
+type = "single"
+
+[django.template.defaulttags.extends]
+args = [{ name = "parent_name", required = true }]
+type = "inclusion"
+
+[django.template.defaulttags.for]
+args = [
+  { name = "{item}", required = true },
+  { name = "in", required = true },
+  { name = "{iterable}", required = true },
+]
+branches = ["empty"]
+closing = "endfor"
+type = "container"
+
+[django.template.defaulttags.filter]
+args = [{ name = "filter_expr", required = true }]
+closing = "endfilter"
+type = "container"
+
+[django.template.defaulttags.firstof]
+args = [{ name = "variables", required = true }]
+type = "single"
+
+[django.template.defaulttags.if]
+args = [{ name = "condition", required = true }]
+branches = ["elif", "else"]
+closing = "endif"
+type = "container"
+
+[django.template.defaulttags.include]
+args = [
+{ name = "template", required = true },
+{ name = "with", required = false, is_kwarg = true },
+{ name = "only", required = false, is_kwarg = true },
+]
+type = "inclusion"
+
+[django.template.defaulttags.load]
+args = [{ name = "library", required = true }]
+type = "single"
+
+[django.template.defaulttags.now]
+args = [{ name = "format_string", required = true }]
+type = "single"
+
+[django.template.defaulttags.spaceless]
+closing = "endspaceless"
+type = "container"
+
+[django.template.defaulttags.templatetag]
+type = "single"
+
+[[django.template.defaulttags.templatetag.args]]
+allowed_values = [
+  "openblock",
+  "closeblock",
+  "openvariable",
+  "closevariable",
+  "openbrace",
+  "closebrace",
+  "opencomment",
+  "closecomment",
+]
+name = "tagtype"
+required = true
+
+[django.template.defaulttags.url]
+args = [
+{ name = "view_name", required = true },
+{ name = "asvar", required = false, is_kwarg = true },
+]
+type = "single"
+
+[django.template.defaulttags.verbatim]
+closing = "endverbatim"
+type = "container"
+
+[django.template.defaulttags.with]
+args = [{ name = "extra_context", required = true }]
+closing = "endwith"
+type = "container"

crates/djls-worker/src/lib.rs

@@ -68,6 +68,10 @@ impl Worker {
         }
     }
 
+    /// Attempts to execute a task immediately without waiting.
+    /// Returns an error if the worker's channel is full.
+    ///
+    /// Best for non-critical tasks where backpressure is desired.
     pub fn execute<T>(&self, task: T) -> Result<()>
     where
         T: Task + 'static,
@@ -78,6 +82,10 @@ impl Worker {
             .map_err(|e| anyhow::anyhow!("Failed to execute task: {}", e))
     }
 
+    /// Submits a task asynchronously, waiting if the channel is full.
+    ///
+    /// Good for tasks that must be processed but where you don't need
+    /// the result immediately.
     pub async fn submit<T>(&self, task: T) -> Result<()>
     where
         T: Task + 'static,
@@ -89,6 +97,10 @@ impl Worker {
             .map_err(|e| anyhow::anyhow!("Failed to submit task: {}", e))
     }
 
+    /// Submits a task and waits for its result.
+    ///
+    /// Best when you need the output of the task. This method will
+    /// wait both for space to submit the task and for its completion.
     pub async fn wait_for<T>(&self, task: T) -> Result<T::Output>
     where
         T: Task + 'static,