From 6d26d780998e2a81750e90a0fcfdddde413e0593 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 6 Jan 2025 10:35:05 -0600 Subject: [PATCH] checkpoint --- crates/djls-template-ast/src/parser.rs | 92 +++---- ...ests__django__parse_mixed_content.snap.new | 259 ++++++++++++++++++ ...sts__errors__parse_error_recovery.snap.new | 122 ++++++++- ...__errors__parse_unclosed_html_tag.snap.new | 9 - ...ts__errors__parse_unclosed_script.snap.new | 9 - ...sts__errors__parse_unclosed_style.snap.new | 9 - ...tests__full_templates__parse_full.snap.new | 225 ++++++++++++++- ...__tests__html__parse_html_doctype.snap.new | 9 - ...rser__tests__html__parse_html_tag.snap.new | 9 - ...ser__tests__html__parse_html_void.snap.new | 9 - ...rser__tests__script__parse_script.snap.new | 34 ++- ...parser__tests__style__parse_style.snap.new | 34 ++- crates/djls-template-ast/src/tokens.rs | 4 +- 13 files changed, 710 insertions(+), 114 deletions(-) create mode 100644 crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap.new delete mode 100644 crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_html_tag.snap.new delete mode 100644 crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_script.snap.new delete mode 100644 crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_style.snap.new delete mode 100644 crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_doctype.snap.new delete mode 100644 crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_tag.snap.new delete mode 100644 crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_void.snap.new diff --git a/crates/djls-template-ast/src/parser.rs b/crates/djls-template-ast/src/parser.rs index 731ac83..8ec6b39 100644 --- a/crates/djls-template-ast/src/parser.rs +++ b/crates/djls-template-ast/src/parser.rs @@ -52,9 +52,6 @@ impl Parser { } fn next_node(&mut self) -> Result { - if self.is_at_end() { - return Err(ParserError::Ast(AstError::StreamError("AtEnd".to_string()))); - } let token = self.peek()?; match token.token_type() { TokenType::DjangoBlock(content) => { @@ -65,6 +62,10 @@ impl Parser { self.consume()?; self.parse_django_variable(content) } + TokenType::Comment(content, start, end) => { + self.consume()?; + self.parse_comment(content, start, end.as_deref()) + } TokenType::Text(_) | TokenType::Whitespace(_) | TokenType::Newline @@ -74,10 +75,9 @@ impl Parser { | TokenType::ScriptTagOpen(_) | TokenType::ScriptTagClose(_) | TokenType::StyleTagOpen(_) - | TokenType::StyleTagClose(_) => self.parse_text(), - TokenType::Comment(content, start, end) => { + | TokenType::StyleTagClose(_) => { self.consume()?; - self.parse_comment(content, start, end.as_deref()) + self.parse_text() } TokenType::Eof => Err(ParserError::Ast(AstError::StreamError("AtEnd".to_string()))), } @@ -136,13 +136,16 @@ impl Parser { tag: branch_tag.clone(), nodes: branch_nodes.clone(), })); - closing = Some(Box::new(Block::Closing { tag: next_tag.clone() })); + closing = Some(Box::new(Block::Closing { + tag: next_tag.clone(), + })); found_closing = true; break; } } // Check if this is another branch tag - if branches.iter().any(|b| b.name == next_tag.name) { + if branches.iter().any(|b| b.name == next_tag.name) + { // Push the current branch and start a new one nodes.push(Node::Block(Block::Branch { tag: branch_tag.clone(), @@ -164,7 +167,9 @@ impl Parser { nodes: branch_nodes.clone(), })); // Add error for unclosed tag - self.errors.push(ParserError::Ast(AstError::UnclosedTag(tag_name.clone()))); + self.errors.push(ParserError::Ast(AstError::UnclosedTag( + tag_name.clone(), + ))); } if found_closing { break; @@ -198,9 +203,16 @@ impl Parser { }; // Add error if we didn't find a closing tag for a block - if let Block::Block { closing: None, tag: tag_ref, .. } = &block { + if let Block::Block { + closing: None, + tag: tag_ref, + .. + } = &block + { if let Some(expected_closing) = &spec.closing { - self.errors.push(ParserError::Ast(AstError::UnclosedTag(tag_ref.name.clone()))); + self.errors.push(ParserError::Ast(AstError::UnclosedTag( + tag_ref.name.clone(), + ))); } } @@ -246,58 +258,40 @@ impl Parser { } fn parse_text(&mut self) -> Result { - let start_token = self.peek()?; + let start_token = self.peek_previous()?; let start_pos = start_token.start().unwrap_or(0); - let mut total_length = start_token.length().unwrap_or(0); - // Handle newlines by returning next node + // If we start with a newline, skip it if matches!(start_token.token_type(), TokenType::Newline) { - self.consume()?; - let node = self.next_node()?; - return Ok(node); + return self.next_node(); } - let mut content = match start_token.token_type() { - TokenType::Text(text) => text.to_string(), - TokenType::Whitespace(count) => " ".repeat(*count), - _ => { - return Err(ParserError::Ast(AstError::InvalidTag( - "Expected text or whitespace token".to_string(), - ))) - } - }; - self.consume()?; + // Use TokenType's Display implementation for formatting + let mut text = start_token.token_type().to_string(); + let mut total_length: u32 = u32::try_from(text.len()).unwrap(); - // Look ahead for more tokens until newline - while let Ok(next_token) = self.peek() { - match next_token.token_type() { - TokenType::Text(text) => { - content.push_str(text); - total_length += next_token.length().unwrap_or(0); + while let Ok(token) = self.peek() { + match token.token_type() { + TokenType::DjangoBlock(_) + | TokenType::DjangoVariable(_) + | TokenType::Comment(_, _, _) + | TokenType::Newline + | TokenType::Eof => break, + _ => { + let token_text = token.token_type().to_string(); + text.push_str(&token_text); + total_length += u32::try_from(token_text.len()).unwrap(); self.consume()?; } - TokenType::Whitespace(count) => { - content.push_str(&" ".repeat(*count)); - total_length += next_token.length().unwrap_or(0); - self.consume()?; - } - TokenType::Newline => { - // Include newline in span but not content - total_length += next_token.length().unwrap_or(0); - self.consume()?; - break; - } - _ => break, } } // Skip empty text nodes - if content.trim().is_empty() { - let node = self.next_node()?; - Ok(node) + if text.trim().is_empty() { + self.next_node() } else { Ok(Node::Text { - content, + content: text, span: Span::new(start_pos, total_length), }) } diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap.new new file mode 100644 index 0000000..0d2e194 --- /dev/null +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap.new @@ -0,0 +1,259 @@ +--- +source: crates/djls-template-ast/src/parser.rs +assertion_line: 516 +expression: ast +snapshot_kind: text +--- +nodes: + - Text: + content: "Welcome, " + span: + start: 0 + length: 9 + - Block: + Block: + tag: + name: if + bits: + - if + - user.is_authenticated + span: + start: 9 + length: 24 + tag_span: + start: 9 + length: 24 + assignment: ~ + nodes: + - Variable: + bits: + - user + - name + filters: + - name: title + args: [] + span: + start: 48 + length: 31 + - name: default + args: + - "'Guest'" + span: + start: 48 + length: 31 + span: + start: 47 + length: 31 + - Block: + Block: + tag: + name: for + bits: + - for + - group + - in + - user.groups + span: + start: 86 + length: 24 + tag_span: + start: 86 + length: 24 + assignment: ~ + nodes: + - Block: + Block: + tag: + name: if + bits: + - if + - forloop.first + span: + start: 125 + length: 16 + tag_span: + start: 125 + length: 16 + assignment: ~ + nodes: + - Text: + content: ( + span: + start: 147 + length: 1 + closing: + Closing: + tag: + name: endif + bits: + - endif + span: + start: 148 + length: 5 + tag_span: + start: 148 + length: 5 + assignment: ~ + assignments: ~ + - Variable: + bits: + - group + - name + filters: [] + span: + start: 171 + length: 10 + - Block: + Block: + tag: + name: if + bits: + - if + - not + - forloop.last + span: + start: 193 + length: 19 + tag_span: + start: 193 + length: 19 + assignment: ~ + nodes: + - Text: + content: ", " + span: + start: 218 + length: 2 + closing: + Closing: + tag: + name: endif + bits: + - endif + span: + start: 220 + length: 5 + tag_span: + start: 220 + length: 5 + assignment: ~ + assignments: ~ + - Block: + Block: + tag: + name: if + bits: + - if + - forloop.last + span: + start: 240 + length: 15 + tag_span: + start: 240 + length: 15 + assignment: ~ + nodes: + - Text: + content: ) + span: + start: 261 + length: 1 + closing: + Closing: + tag: + name: endif + bits: + - endif + span: + start: 262 + length: 5 + tag_span: + start: 262 + length: 5 + assignment: ~ + assignments: ~ + - Block: + Branch: + tag: + name: empty + bits: + - empty + span: + start: 278 + length: 5 + tag_span: + start: 278 + length: 5 + assignment: ~ + nodes: + - Text: + content: " (no groups)" + span: + start: 290 + length: 19 + closing: + Closing: + tag: + name: endfor + bits: + - endfor + span: + start: 314 + length: 6 + tag_span: + start: 314 + length: 6 + assignment: ~ + assignments: ~ + - Block: + Branch: + tag: + name: else + bits: + - else + span: + start: 327 + length: 4 + tag_span: + start: 327 + length: 4 + assignment: ~ + nodes: + - Text: + content: " Guest" + span: + start: 338 + length: 9 + closing: + Closing: + tag: + name: endif + bits: + - endif + span: + start: 348 + length: 5 + tag_span: + start: 348 + length: 5 + assignment: ~ + assignments: ~ + - Text: + content: "!" + span: + start: 359 + length: 1 +line_offsets: + - 0 + - 40 + - 82 + - 117 + - 160 + - 185 + - 232 + - 274 + - 290 + - 310 + - 327 + - 338 + - 348 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap.new index 8d931e2..cb59410 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap.new +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap.new @@ -1,10 +1,128 @@ --- source: crates/djls-template-ast/src/parser.rs -assertion_line: 643 +assertion_line: 637 expression: ast snapshot_kind: text --- -nodes: [] +nodes: + - Text: + content: "
" + span: + start: 0 + length: 23 + - Text: + content: "

Header

" + span: + start: 24 + length: 19 + - Block: + Block: + tag: + name: if + bits: + - if + - user.is_authenticated + span: + start: 48 + length: 24 + tag_span: + start: 48 + length: 24 + assignment: ~ + nodes: + - Comment: + content: This if is unclosed which does matter + span: + start: 87 + length: 41 + - Text: + content: "

Welcome " + span: + start: 131 + length: 19 + - Variable: + bits: + - user + - name + filters: [] + span: + start: 153 + length: 9 + - Text: + content: "

" + span: + start: 165 + length: 4 + - Text: + content: "
" + span: + start: 170 + length: 13 + - Comment: + content: "This div is unclosed which doesn't matter" + span: + start: 196 + length: 45 + - Block: + Block: + tag: + name: for + bits: + - for + - item + - in + - items + span: + start: 252 + length: 17 + tag_span: + start: 252 + length: 17 + assignment: ~ + nodes: + - Text: + content: " " + span: + start: 276 + length: 18 + - Variable: + bits: + - item + filters: [] + span: + start: 297 + length: 4 + - Text: + content: "" + span: + start: 304 + length: 7 + closing: + Closing: + tag: + name: endfor + bits: + - endfor + span: + start: 320 + length: 6 + tag_span: + start: 320 + length: 6 + assignment: ~ + assignments: ~ + - Text: + content: "
Page Footer
" + span: + start: 333 + length: 32 + - Text: + content: "
" + span: + start: 366 + length: 6 + closing: ~ + assignments: ~ line_offsets: - 0 - 24 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_html_tag.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_html_tag.snap.new deleted file mode 100644 index 98e09ed..0000000 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_html_tag.snap.new +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: crates/djls-template-ast/src/parser.rs -assertion_line: 581 -expression: ast -snapshot_kind: text ---- -nodes: [] -line_offsets: - - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_script.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_script.snap.new deleted file mode 100644 index ba720fd..0000000 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_script.snap.new +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: crates/djls-template-ast/src/parser.rs -assertion_line: 614 -expression: ast -snapshot_kind: text ---- -nodes: [] -line_offsets: - - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_style.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_style.snap.new deleted file mode 100644 index 8331cf8..0000000 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_style.snap.new +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: crates/djls-template-ast/src/parser.rs -assertion_line: 623 -expression: ast -snapshot_kind: text ---- -nodes: [] -line_offsets: - - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap.new index b341e56..7c27ba1 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap.new +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap.new @@ -1,10 +1,231 @@ --- source: crates/djls-template-ast/src/parser.rs -assertion_line: 688 +assertion_line: 682 expression: ast snapshot_kind: text --- -nodes: [] +nodes: + - Text: + content: "" + span: + start: 0 + length: 15 + - Text: + content: "" + span: + start: 16 + length: 6 + - Text: + content: " " + span: + start: 23 + length: 10 + - Text: + content: " " + span: + start: 134 + length: 16 + - Text: + content: " " + span: + start: 313 + length: 17 + - Text: + content: " " + span: + start: 331 + length: 11 + - Text: + content: " " + span: + start: 343 + length: 10 + - Comment: + content: Header section + span: + start: 362 + length: 21 + - Text: + content: "
" + span: + start: 386 + length: 64 + - Block: + Block: + tag: + name: if + bits: + - if + - user.is_authenticated + span: + start: 463 + length: 24 + tag_span: + start: 463 + length: 24 + assignment: ~ + nodes: + - Comment: + content: Welcome message + span: + start: 510 + length: 19 + - Text: + content: "

Welcome, " + span: + start: 532 + length: 29 + - Variable: + bits: + - user + - name + filters: + - name: title + args: [] + span: + start: 565 + length: 31 + - name: default + args: + - "'Guest'" + span: + start: 565 + length: 31 + span: + start: 564 + length: 31 + - Text: + content: "!

" + span: + start: 598 + length: 6 + - Block: + Block: + tag: + name: if + bits: + - if + - user.is_staff + span: + start: 621 + length: 16 + tag_span: + start: 621 + length: 16 + assignment: ~ + nodes: + - Text: + content: " Admin" + span: + start: 644 + length: 38 + - Block: + Branch: + tag: + name: else + bits: + - else + span: + start: 699 + length: 4 + tag_span: + start: 699 + length: 4 + assignment: ~ + nodes: + - Text: + content: " User" + span: + start: 710 + length: 37 + closing: + Closing: + tag: + name: endif + bits: + - endif + span: + start: 764 + length: 5 + tag_span: + start: 764 + length: 5 + assignment: ~ + assignments: ~ + closing: + Closing: + tag: + name: endif + bits: + - endif + span: + start: 788 + length: 5 + tag_span: + start: 788 + length: 5 + assignment: ~ + assignments: ~ + - Text: + content: "
" + span: + start: 800 + length: 14 + - Text: + content: " " + span: + start: 815 + length: 11 + - Text: + content: "" + span: + start: 827 + length: 7 line_offsets: - 0 - 16 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_doctype.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_doctype.snap.new deleted file mode 100644 index f2a3d82..0000000 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_doctype.snap.new +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: crates/djls-template-ast/src/parser.rs -assertion_line: 425 -expression: ast -snapshot_kind: text ---- -nodes: [] -line_offsets: - - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_tag.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_tag.snap.new deleted file mode 100644 index 7df0605..0000000 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_tag.snap.new +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: crates/djls-template-ast/src/parser.rs -assertion_line: 434 -expression: ast -snapshot_kind: text ---- -nodes: [] -line_offsets: - - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_void.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_void.snap.new deleted file mode 100644 index 18a9865..0000000 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_html_void.snap.new +++ /dev/null @@ -1,9 +0,0 @@ ---- -source: crates/djls-template-ast/src/parser.rs -assertion_line: 443 -expression: ast -snapshot_kind: text ---- -nodes: [] -line_offsets: - - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__script__parse_script.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__script__parse_script.snap.new index 4051c25..04000cd 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__script__parse_script.snap.new +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__script__parse_script.snap.new @@ -1,10 +1,40 @@ --- source: crates/djls-template-ast/src/parser.rs -assertion_line: 540 +assertion_line: 534 expression: ast snapshot_kind: text --- -nodes: [] +nodes: + - Text: + content: "" + span: + start: 133 + length: 9 line_offsets: - 0 - 32 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__style__parse_style.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__style__parse_style.snap.new index 45fe79b..4fdd26b 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__style__parse_style.snap.new +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__style__parse_style.snap.new @@ -1,10 +1,40 @@ --- source: crates/djls-template-ast/src/parser.rs -assertion_line: 557 +assertion_line: 551 expression: ast snapshot_kind: text --- -nodes: [] +nodes: + - Text: + content: "" + span: + start: 89 + length: 8 line_offsets: - 0 - 24 diff --git a/crates/djls-template-ast/src/tokens.rs b/crates/djls-template-ast/src/tokens.rs index d18f722..33fc6ce 100644 --- a/crates/djls-template-ast/src/tokens.rs +++ b/crates/djls-template-ast/src/tokens.rs @@ -74,13 +74,11 @@ impl fmt::Display for TokenType { DjangoBlock(s) => write!(f, "{{% {} %}}", s), DjangoVariable(s) => write!(f, "{{{{ {} }}}}", s), Eof => Ok(()), - HtmlTagOpen(s) => write!(f, "<{}>", s), + HtmlTagOpen(s) | ScriptTagOpen(s) | StyleTagOpen(s) => write!(f, "<{}>", s), HtmlTagClose(s) => write!(f, "", s), HtmlTagVoid(s) => write!(f, "<{}/>", s), Newline => f.write_str("\n"), - ScriptTagOpen(s) => write!(f, "", s), ScriptTagClose(_) => f.write_str(""), - StyleTagOpen(s) => write!(f, "", s), StyleTagClose(_) => f.write_str(""), Text(s) => f.write_str(s), Whitespace(len) => f.write_str(&" ".repeat(*len)),