diff --git a/crates/djls-template-ast/src/parser.rs b/crates/djls-template-ast/src/parser.rs index 9f10e8c..acaf828 100644 --- a/crates/djls-template-ast/src/parser.rs +++ b/crates/djls-template-ast/src/parser.rs @@ -281,20 +281,11 @@ impl Parser { let start_token = self.peek_previous()?; let start_pos = start_token.start().unwrap_or(0); - match start_token.token_type() { - TokenType::Newline => return self.next_node(), - TokenType::Whitespace(_) - if self - .peek_at(-2) - .map_or(false, |t| matches!(t.token_type(), TokenType::Newline)) => - { - return self.next_node() - } - _ => {} + if start_token.token_type() == &TokenType::Newline { + return self.next_node(); } let mut text = start_token.token_type().to_string(); - let mut total_length: u32 = u32::try_from(text.len()).unwrap(); while let Ok(token) = self.peek() { match token.token_type() { @@ -303,39 +294,25 @@ impl Parser { | TokenType::Comment(_, _, _) | TokenType::Newline | TokenType::Eof => break, - TokenType::Whitespace(_) => { - // Check if next token is a newline - if let Ok(next) = self.peek_at(1) { - if matches!(next.token_type(), TokenType::Newline) { - self.consume()?; - break; - } - } - // Not before newline, treat as normal text - let token_text = token.token_type().to_string(); - text.push_str(&token_text); - total_length += u32::try_from(token_text.len()).unwrap(); - self.consume()?; - } _ => { let token_text = token.token_type().to_string(); text.push_str(&token_text); - total_length += u32::try_from(token_text.len()).unwrap(); self.consume()?; } } } - if text.trim().is_empty() { - self.next_node() - } else { - let trimmed = text.trim(); - Ok(Node::Text { - content: trimmed.to_string(), - span: Span::new(start_pos + u32::try_from(text.find(trimmed).unwrap_or(0)).unwrap(), - u32::try_from(trimmed.len()).unwrap()), - }) - } + let content = match text.trim() { + "" => return self.next_node(), + trimmed => trimmed.to_string(), + }; + let offset = u32::try_from(text.find(content.as_str()).unwrap_or(0)).unwrap(); + let length = u32::try_from(content.len()).unwrap(); + + Ok(Node::Text { + content, + span: Span::new(start_pos + offset, length), + }) } fn peek(&self) -> Result { diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap index e91cd00..e74051d 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap @@ -4,10 +4,10 @@ expression: ast --- nodes: - Text: - content: "Welcome, " + content: "Welcome," span: start: 0 - length: 9 + length: 8 - Block: Block: tag: @@ -118,10 +118,10 @@ nodes: assignment: ~ nodes: - Text: - content: ", " + content: "," span: start: 218 - length: 2 + length: 1 closing: Closing: tag: diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap.new deleted file mode 100644 index 5874bbb..0000000 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap.new +++ /dev/null @@ -1,259 +0,0 @@ ---- -source: crates/djls-template-ast/src/parser.rs -assertion_line: 550 -expression: ast -snapshot_kind: text ---- -nodes: - - Text: - content: "Welcome," - span: - start: 0 - length: 8 - - Block: - Block: - tag: - name: if - bits: - - if - - user.is_authenticated - span: - start: 9 - length: 24 - tag_span: - start: 9 - length: 24 - assignment: ~ - nodes: - - Variable: - bits: - - user - - name - filters: - - name: title - args: [] - span: - start: 48 - length: 31 - - name: default - args: - - "'Guest'" - span: - start: 48 - length: 31 - span: - start: 47 - length: 31 - - Block: - Block: - tag: - name: for - bits: - - for - - group - - in - - user.groups - span: - start: 86 - length: 24 - tag_span: - start: 86 - length: 24 - assignment: ~ - nodes: - - Block: - Block: - tag: - name: if - bits: - - if - - forloop.first - span: - start: 125 - length: 16 - tag_span: - start: 125 - length: 16 - assignment: ~ - nodes: - - Text: - content: ( - span: - start: 147 - length: 1 - closing: - Closing: - tag: - name: endif - bits: - - endif - span: - start: 148 - length: 5 - tag_span: - start: 148 - length: 5 - assignment: ~ - assignments: ~ - - Variable: - bits: - - group - - name - filters: [] - span: - start: 171 - length: 10 - - Block: - Block: - tag: - name: if - bits: - - if - - not - - forloop.last - span: - start: 193 - length: 19 - tag_span: - start: 193 - length: 19 - assignment: ~ - nodes: - - Text: - content: "," - span: - start: 218 - length: 1 - closing: - Closing: - tag: - name: endif - bits: - - endif - span: - start: 220 - length: 5 - tag_span: - start: 220 - length: 5 - assignment: ~ - assignments: ~ - - Block: - Block: - tag: - name: if - bits: - - if - - forloop.last - span: - start: 240 - length: 15 - tag_span: - start: 240 - length: 15 - assignment: ~ - nodes: - - Text: - content: ) - span: - start: 261 - length: 1 - closing: - Closing: - tag: - name: endif - bits: - - endif - span: - start: 262 - length: 5 - tag_span: - start: 262 - length: 5 - assignment: ~ - assignments: ~ - - Block: - Branch: - tag: - name: empty - bits: - - empty - span: - start: 278 - length: 5 - tag_span: - start: 278 - length: 5 - assignment: ~ - nodes: - - Text: - content: (no groups) - span: - start: 298 - length: 11 - closing: - Closing: - tag: - name: endfor - bits: - - endfor - span: - start: 314 - length: 6 - tag_span: - start: 314 - length: 6 - assignment: ~ - assignments: ~ - - Block: - Branch: - tag: - name: else - bits: - - else - span: - start: 327 - length: 4 - tag_span: - start: 327 - length: 4 - assignment: ~ - nodes: - - Text: - content: Guest - span: - start: 342 - length: 5 - closing: - Closing: - tag: - name: endif - bits: - - endif - span: - start: 348 - length: 5 - tag_span: - start: 348 - length: 5 - assignment: ~ - assignments: ~ - - Text: - content: "!" - span: - start: 359 - length: 1 -line_offsets: - - 0 - - 40 - - 82 - - 117 - - 160 - - 185 - - 232 - - 274 - - 290 - - 310 - - 327 - - 338 - - 348 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap index 5fc20f6..c72cd2a 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap @@ -34,10 +34,10 @@ nodes: start: 87 length: 43 - Text: - content: "

Welcome " + content: "

Welcome" span: start: 139 - length: 11 + length: 10 - Variable: bits: - user diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap.new deleted file mode 100644 index 90d3d53..0000000 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap.new +++ /dev/null @@ -1,138 +0,0 @@ ---- -source: crates/djls-template-ast/src/parser.rs -assertion_line: 728 -expression: ast -snapshot_kind: text ---- -nodes: - - Text: - content: "

" - span: - start: 0 - length: 23 - - Text: - content: "

Header

" - span: - start: 28 - length: 15 - - Block: - Block: - tag: - name: if - bits: - - if - - user.is_authenticated - span: - start: 48 - length: 24 - tag_span: - start: 48 - length: 24 - assignment: ~ - nodes: - - Comment: - content: This if is unclosed which does matter - span: - start: 87 - length: 43 - - Text: - content: "

Welcome" - span: - start: 139 - length: 10 - - Variable: - bits: - - user - - name - filters: [] - span: - start: 153 - length: 9 - - Text: - content: "

" - span: - start: 165 - length: 4 - - Text: - content: "
" - span: - start: 178 - length: 5 - - Comment: - content: "This div is unclosed which doesn't matter" - span: - start: 196 - length: 47 - - Block: - Block: - tag: - name: for - bits: - - for - - item - - in - - items - span: - start: 252 - length: 17 - tag_span: - start: 252 - length: 17 - assignment: ~ - nodes: - - Text: - content: "" - span: - start: 288 - length: 6 - - Variable: - bits: - - item - filters: [] - span: - start: 297 - length: 4 - - Text: - content: "" - span: - start: 304 - length: 7 - closing: - Closing: - tag: - name: endfor - bits: - - endfor - span: - start: 320 - length: 6 - tag_span: - start: 320 - length: 6 - assignment: ~ - assignments: ~ - - Text: - content: "
Page Footer
" - span: - start: 337 - length: 28 - - Text: - content: "
" - span: - start: 366 - length: 6 - closing: ~ - assignments: ~ -line_offsets: - - 0 - - 24 - - 44 - - 79 - - 131 - - 170 - - 184 - - 244 - - 276 - - 312 - - 333 - - 366 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_style.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_style.snap index f69fc15..be36f2e 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_style.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_style.snap @@ -4,9 +4,9 @@ expression: ast --- nodes: - Text: - content: "" - span: - start: 142 - length: 8 - - Text: - content: "" - span: - start: 321 - length: 9 - - Text: - content: "" - span: - start: 335 - length: 7 - - Text: - content: "" - span: - start: 347 - length: 6 - - Text: - content: "" - span: - start: 362 - length: 23 - - Text: - content: "
" - span: - start: 394 - length: 56 - - Block: - Block: - tag: - name: if - bits: - - if - - user.is_authenticated - span: - start: 463 - length: 24 - tag_span: - start: 463 - length: 24 - assignment: ~ - nodes: - - Comment: - content: Welcome message - span: - start: 510 - length: 21 - - Text: - content: "

Welcome," - span: - start: 548 - length: 12 - - Variable: - bits: - - user - - name - filters: - - name: title - args: [] - span: - start: 565 - length: 31 - - name: default - args: - - "'Guest'" - span: - start: 565 - length: 31 - span: - start: 564 - length: 31 - - Text: - content: "!

" - span: - start: 598 - length: 6 - - Block: - Block: - tag: - name: if - bits: - - if - - user.is_staff - span: - start: 621 - length: 16 - tag_span: - start: 621 - length: 16 - assignment: ~ - nodes: - - Text: - content: "Admin" - span: - start: 664 - length: 18 - - Block: - Branch: - tag: - name: else - bits: - - else - span: - start: 699 - length: 4 - tag_span: - start: 699 - length: 4 - assignment: ~ - nodes: - - Text: - content: "User" - span: - start: 730 - length: 17 - closing: - Closing: - tag: - name: endif - bits: - - endif - span: - start: 764 - length: 5 - tag_span: - start: 764 - length: 5 - assignment: ~ - assignments: ~ - closing: - Closing: - tag: - name: endif - bits: - - endif - span: - start: 788 - length: 5 - tag_span: - start: 788 - length: 5 - assignment: ~ - assignments: ~ - - Text: - content: "
" - span: - start: 808 - length: 6 - - Text: - content: "" - span: - start: 819 - length: 7 - - Text: - content: "" - span: - start: 827 - length: 7 -line_offsets: - - 0 - - 16 - - 23 - - 34 - - 66 - - 97 - - 134 - - 151 - - 191 - - 215 - - 241 - - 270 - - 298 - - 313 - - 331 - - 343 - - 354 - - 386 - - 451 - - 494 - - 532 - - 605 - - 644 - - 683 - - 710 - - 748 - - 776 - - 800 - - 815 - - 827 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_with_leading_whitespace.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_with_leading_whitespace.snap similarity index 84% rename from crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_with_leading_whitespace.snap.new rename to crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_with_leading_whitespace.snap index c9679b9..fa16a8b 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_with_leading_whitespace.snap.new +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__html__parse_with_leading_whitespace.snap @@ -1,8 +1,6 @@ --- source: crates/djls-template-ast/src/parser.rs -assertion_line: 472 expression: ast -snapshot_kind: text --- nodes: - Text: diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace.snap similarity index 81% rename from crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace.snap.new rename to crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace.snap index 7339bbf..05bf784 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace.snap.new +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace.snap @@ -1,8 +1,6 @@ --- source: crates/djls-template-ast/src/parser.rs -assertion_line: 617 expression: ast -snapshot_kind: text --- nodes: - Text: diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace_newline.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace_newline.snap similarity index 81% rename from crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace_newline.snap.new rename to crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace_newline.snap index f15e986..d3c1149 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace_newline.snap.new +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_leading_whitespace_newline.snap @@ -1,8 +1,6 @@ --- source: crates/djls-template-ast/src/parser.rs -assertion_line: 627 expression: ast -snapshot_kind: text --- nodes: - Text: diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace.snap similarity index 81% rename from crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace.snap.new rename to crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace.snap index 850fea7..5f5a86f 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace.snap.new +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace.snap @@ -1,8 +1,6 @@ --- source: crates/djls-template-ast/src/parser.rs -assertion_line: 637 expression: ast -snapshot_kind: text --- nodes: - Text: diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace_newline.snap.new b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace_newline.snap similarity index 82% rename from crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace_newline.snap.new rename to crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace_newline.snap index 6e6a56d..4886c71 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace_newline.snap.new +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__whitespace__parse_with_trailing_whitespace_newline.snap @@ -1,8 +1,6 @@ --- source: crates/djls-template-ast/src/parser.rs -assertion_line: 647 expression: ast -snapshot_kind: text --- nodes: - Text: