From c559baea1fbab698d0cb10461ee089b1515892a5 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 6 Jan 2025 09:15:18 -0600 Subject: [PATCH] wip --- crates/djls-template-ast/src/ast.rs | 35 ++--- crates/djls-template-ast/src/parser.rs | 145 +++++++++++------- ..._tests__django__parse_complex_if_elif.snap | 28 ++-- ...tests__django__parse_django_for_block.snap | 15 +- ..._tests__django__parse_django_if_block.snap | 2 +- ...r__tests__django__parse_mixed_content.snap | 36 +++-- ...r__tests__django__parse_nested_for_if.snap | 4 +- ...__tests__errors__parse_error_recovery.snap | 4 +- ...ts__errors__parse_unclosed_django_for.snap | 2 +- ...sts__errors__parse_unclosed_django_if.snap | 2 +- ...er__tests__full_templates__parse_full.snap | 17 +- 11 files changed, 164 insertions(+), 126 deletions(-) diff --git a/crates/djls-template-ast/src/ast.rs b/crates/djls-template-ast/src/ast.rs index 3f366f0..c0f95a2 100644 --- a/crates/djls-template-ast/src/ast.rs +++ b/crates/djls-template-ast/src/ast.rs @@ -44,25 +44,18 @@ impl LineOffsets { self.0.push(offset); } - pub fn position_to_line_col(&self, offset: u32) -> (u32, u32) { - // Find which line contains this offset by looking for the first line start - // that's greater than our position - let line = match self.0.binary_search(&offset) { - Ok(exact_line) => exact_line, // We're exactly at a line start - Err(next_line) => { - if next_line == 0 { - 0 // Before first line start, so we're on line 1 - } else { - next_line - 1 // We're on the previous line - } - } + pub fn position_to_line_col(&self, position: usize) -> (usize, usize) { + let position = position as u32; + let line = match self.0.binary_search(&position) { + Ok(_) => self.0.partition_point(|&x| x <= position), + Err(i) => i, }; - - // Calculate column as offset from line start - let col = offset - self.0[line]; - - // Convert to 1-based line number - (line as u32 + 1, col) + let col = if line == 0 { + position as usize + } else { + (position - self.0[line - 1]) as usize + }; + (line + 1, col) } pub fn line_col_to_position(&self, line: u32, col: u32) -> u32 { @@ -286,7 +279,7 @@ mod tests { if let Node::Variable { span, .. } = var_node { // Variable starts after newline + "{{" - let (line, col) = ast.line_offsets().position_to_line_col(*span.start()); + let (line, col) = ast.line_offsets().position_to_line_col(*span.start() as usize); assert_eq!( (line, col), (2, 3), @@ -354,8 +347,8 @@ mod tests { assert_eq!(content, " Welcome!"); eprintln!("Line offsets: {:?}", ast.line_offsets()); eprintln!("Span: {:?}", span); - let (line, col) = ast.line_offsets().position_to_line_col(span.start); - assert_eq!((line, col), (2, 0), "Content should be on line 2, col 0"); + let (line, col) = ast.line_offsets().position_to_line_col(span.start as usize); + assert_eq!((line, col), (1, 0), "Content should be on line 1, col 0"); // Check closing tag if let Block::Closing { tag } = diff --git a/crates/djls-template-ast/src/parser.rs b/crates/djls-template-ast/src/parser.rs index 27ce346..c228a98 100644 --- a/crates/djls-template-ast/src/parser.rs +++ b/crates/djls-template-ast/src/parser.rs @@ -86,82 +86,113 @@ impl Parser { let total_length = token.length().unwrap_or(0); let span = Span::new(start_pos, total_length); - // Parse the tag name and any assignments - let mut bits = content.split_whitespace(); - let tag_name = bits.next().unwrap_or_default().to_string(); - let bits_vec: Vec = bits.map(|s| s.to_string()).collect(); - - // Check for assignment syntax - let mut assignments = Vec::new(); - let mut assignment = None; - if bits_vec.len() > 2 && bits_vec[1] == "as" { - assignment = Some(bits_vec[2].clone()); - assignments.push(Assignment { - target: bits_vec[2].clone(), - value: bits_vec[3..].join(" "), - }); - } + let bits: Vec = content.split_whitespace().map(String::from).collect(); + let tag_name = bits.first().ok_or(ParserError::EmptyTag)?.clone(); let tag = Tag { name: tag_name.clone(), - bits: content.split_whitespace().map(|s| s.to_string()).collect(), + bits: bits.clone(), span, tag_span: span, - assignment, + assignment: None, }; - // Check if this is a closing tag - if tag_name.starts_with("end") { - return Ok(Node::Block(Block::Closing { tag })); - } - - // Load tag specs let specs = TagSpec::load_builtin_specs()?; let spec = match specs.get(&tag_name) { Some(spec) => spec, None => return Ok(Node::Block(Block::Tag { tag })), }; - match spec.tag_type { + let block = match spec.tag_type { TagType::Block => { let mut nodes = Vec::new(); + let mut closing = None; - // Parse child nodes until we find the closing tag - while let Ok(node) = self.next_node() { - if let Node::Block(Block::Closing { tag: closing_tag }) = &node { - if let Some(expected_closing) = &spec.closing { - if closing_tag.name == *expected_closing { - return Ok(Node::Block(Block::Block { - tag, - nodes, - closing: Some(Box::new(Block::Closing { - tag: closing_tag.clone(), - })), - assignments: Some(assignments), - })); + while !self.is_at_end() { + match self.next_node() { + Ok(Node::Block(Block::Tag { tag })) => { + if let Some(expected_closing) = &spec.closing { + if tag.name == *expected_closing { + closing = Some(Box::new(Block::Closing { tag })); + break; + } } + // If we get here, either there was no expected closing tag or it didn't match + if let Some(branches) = &spec.branches { + if branches.iter().any(|b| b.name == tag.name) { + let mut branch_tag = tag.clone(); + let mut branch_nodes = Vec::new(); + let mut found_closing = false; + while let Ok(node) = self.next_node() { + match &node { + Node::Block(Block::Tag { tag: next_tag }) => { + if let Some(expected_closing) = &spec.closing { + if next_tag.name == *expected_closing { + // Found the closing tag + nodes.push(Node::Block(Block::Branch { + tag: branch_tag.clone(), + nodes: branch_nodes.clone(), + })); + closing = Some(Box::new(Block::Closing { tag: next_tag.clone() })); + found_closing = true; + break; + } + } + // Check if this is another branch tag + if branches.iter().any(|b| b.name == next_tag.name) { + // Push the current branch and start a new one + nodes.push(Node::Block(Block::Branch { + tag: branch_tag.clone(), + nodes: branch_nodes.clone(), + })); + branch_nodes = Vec::new(); + branch_tag = next_tag.clone(); + continue; + } + branch_nodes.push(node); + } + _ => branch_nodes.push(node), + } + } + if !found_closing { + // Push the last branch if we didn't find a closing tag + nodes.push(Node::Block(Block::Branch { + tag: branch_tag, + nodes: branch_nodes, + })); + } + if found_closing { + break; + } + continue; + } + } + nodes.push(Node::Block(Block::Tag { tag })); + } + Ok(node) => nodes.push(node), + Err(e) => { + self.errors.push(e); + break; } } - nodes.push(node); } - // Add error for unclosed tag - self.errors.push(ParserError::Ast(AstError::UnclosedTag(tag_name.clone()))); - - Ok(Node::Block(Block::Block { + Block::Block { tag, nodes, - closing: None, - assignments: Some(assignments), - })) + closing, + assignments: None, + } } - TagType::Tag => Ok(Node::Block(Block::Tag { tag })), - TagType::Variable => Ok(Node::Block(Block::Variable { tag })), + TagType::Tag => Block::Tag { tag }, + TagType::Variable => Block::Variable { tag }, TagType::Inclusion => { - let template_name = bits_vec.get(1).cloned().unwrap_or_default(); - Ok(Node::Block(Block::Inclusion { tag, template_name })) + let template_name = bits.get(1).cloned().unwrap_or_default(); + Block::Inclusion { tag, template_name } } - } + }; + + Ok(Node::Block(block)) } fn parse_django_variable(&mut self, content: &str) -> Result { @@ -356,6 +387,8 @@ pub enum ParserError { ErrorSignal(Signal), #[error("{0}")] Other(#[from] anyhow::Error), + #[error("empty tag")] + EmptyTag, } impl ParserError { @@ -544,7 +577,9 @@ mod tests { let (ast, errors) = parser.parse().unwrap(); insta::assert_yaml_snapshot!(ast); assert_eq!(errors.len(), 1); - assert!(matches!(&errors[0], ParserError::Ast(AstError::UnclosedTag(tag)) if tag == "if")); + assert!( + matches!(&errors[0], ParserError::Ast(AstError::UnclosedTag(tag)) if tag == "if") + ); } #[test] fn test_parse_unclosed_django_for() { @@ -554,7 +589,9 @@ mod tests { let (ast, errors) = parser.parse().unwrap(); insta::assert_yaml_snapshot!(ast); assert_eq!(errors.len(), 1); - assert!(matches!(&errors[0], ParserError::Ast(AstError::UnclosedTag(tag)) if tag == "for")); + assert!( + matches!(&errors[0], ParserError::Ast(AstError::UnclosedTag(tag)) if tag == "for") + ); } #[test] fn test_parse_unclosed_script() { @@ -593,7 +630,9 @@ mod tests { let (ast, errors) = parser.parse().unwrap(); insta::assert_yaml_snapshot!(ast); assert_eq!(errors.len(), 1); - assert!(matches!(&errors[0], ParserError::Ast(AstError::UnclosedTag(tag)) if tag == "if")); + assert!( + matches!(&errors[0], ParserError::Ast(AstError::UnclosedTag(tag)) if tag == "if") + ); } } diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_complex_if_elif.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_complex_if_elif.snap index d0dee4e..eccc889 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_complex_if_elif.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_complex_if_elif.snap @@ -26,7 +26,7 @@ nodes: start: 14 length: 8 - Block: - Tag: + Branch: tag: name: elif bits: @@ -41,13 +41,14 @@ nodes: start: 22 length: 10 assignment: ~ - - Text: - content: Negative - span: - start: 38 - length: 8 + nodes: + - Text: + content: Negative + span: + start: 38 + length: 8 - Block: - Tag: + Branch: tag: name: else bits: @@ -59,11 +60,12 @@ nodes: start: 46 length: 4 assignment: ~ - - Text: - content: Zero - span: - start: 56 - length: 4 + nodes: + - Text: + content: Zero + span: + start: 56 + length: 4 closing: Closing: tag: @@ -77,6 +79,6 @@ nodes: start: 60 length: 5 assignment: ~ - assignments: [] + assignments: ~ line_offsets: - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_for_block.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_for_block.snap index d0ed7b9..8d80375 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_for_block.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_for_block.snap @@ -28,7 +28,7 @@ nodes: start: 26 length: 4 - Block: - Tag: + Branch: tag: name: empty bits: @@ -40,11 +40,12 @@ nodes: start: 33 length: 5 assignment: ~ - - Text: - content: No items - span: - start: 44 - length: 8 + nodes: + - Text: + content: No items + span: + start: 44 + length: 8 closing: Closing: tag: @@ -58,6 +59,6 @@ nodes: start: 52 length: 6 assignment: ~ - assignments: [] + assignments: ~ line_offsets: - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_if_block.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_if_block.snap index caf8c18..7a18941 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_if_block.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_if_block.snap @@ -36,6 +36,6 @@ nodes: start: 37 length: 5 assignment: ~ - assignments: [] + assignments: ~ line_offsets: - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap index 50a4853..7353de2 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap @@ -92,7 +92,7 @@ nodes: start: 148 length: 5 assignment: ~ - assignments: [] + assignments: ~ - Variable: bits: - group @@ -135,7 +135,7 @@ nodes: start: 220 length: 5 assignment: ~ - assignments: [] + assignments: ~ - Block: Block: tag: @@ -169,9 +169,9 @@ nodes: start: 262 length: 5 assignment: ~ - assignments: [] + assignments: ~ - Block: - Tag: + Branch: tag: name: empty bits: @@ -183,11 +183,12 @@ nodes: start: 278 length: 5 assignment: ~ - - Text: - content: " (no groups)" - span: - start: 290 - length: 20 + nodes: + - Text: + content: " (no groups)" + span: + start: 290 + length: 20 closing: Closing: tag: @@ -201,9 +202,9 @@ nodes: start: 314 length: 6 assignment: ~ - assignments: [] + assignments: ~ - Block: - Tag: + Branch: tag: name: else bits: @@ -215,11 +216,12 @@ nodes: start: 327 length: 4 assignment: ~ - - Text: - content: " Guest" - span: - start: 338 - length: 10 + nodes: + - Text: + content: " Guest" + span: + start: 338 + length: 10 closing: Closing: tag: @@ -233,7 +235,7 @@ nodes: start: 348 length: 5 assignment: ~ - assignments: [] + assignments: ~ - Text: content: "!" span: diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_nested_for_if.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_nested_for_if.snap index 17ec6bd..b0f9f82 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_nested_for_if.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_nested_for_if.snap @@ -56,7 +56,7 @@ nodes: start: 58 length: 5 assignment: ~ - assignments: [] + assignments: ~ closing: Closing: tag: @@ -70,6 +70,6 @@ nodes: start: 69 length: 6 assignment: ~ - assignments: [] + assignments: ~ line_offsets: - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap index ffcf1cb..6f52c13 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap @@ -108,7 +108,7 @@ nodes: start: 320 length: 6 assignment: ~ - assignments: [] + assignments: ~ - Text: content: "
Page Footer
" span: @@ -120,7 +120,7 @@ nodes: start: 366 length: 6 closing: ~ - assignments: [] + assignments: ~ line_offsets: - 0 - 24 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_for.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_for.snap index edc9b24..fdf363b 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_for.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_for.snap @@ -29,6 +29,6 @@ nodes: start: 26 length: 9 closing: ~ - assignments: [] + assignments: ~ line_offsets: - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_if.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_if.snap index 92eed29..f998c29 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_if.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_if.snap @@ -24,6 +24,6 @@ nodes: start: 30 length: 7 closing: ~ - assignments: [] + assignments: ~ line_offsets: - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap index 3b9a673..2fd9949 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap @@ -163,7 +163,7 @@ nodes: start: 644 length: 39 - Block: - Tag: + Branch: tag: name: else bits: @@ -175,11 +175,12 @@ nodes: start: 699 length: 4 assignment: ~ - - Text: - content: " User" - span: - start: 710 - length: 38 + nodes: + - Text: + content: " User" + span: + start: 710 + length: 38 closing: Closing: tag: @@ -193,7 +194,7 @@ nodes: start: 764 length: 5 assignment: ~ - assignments: [] + assignments: ~ closing: Closing: tag: @@ -207,7 +208,7 @@ nodes: start: 788 length: 5 assignment: ~ - assignments: [] + assignments: ~ - Text: content: " " span: