From a078faabf71dfa526ccca5fdf5384c080cd38ac0 Mon Sep 17 00:00:00 2001 From: Josh Date: Mon, 6 Jan 2025 16:06:47 -0600 Subject: [PATCH] fix --- crates/djls-template-ast/src/ast.rs | 2 +- crates/djls-template-ast/src/lexer.rs | 250 +++++++++--------- ...rser__tests__comments__parse_comments.snap | 2 +- ..._tests__django__parse_complex_if_elif.snap | 16 +- ...tests__django__parse_django_for_block.snap | 12 +- ..._tests__django__parse_django_if_block.snap | 8 +- ...go__parse_django_variable_with_filter.snap | 4 +- ...r__tests__django__parse_filter_chains.snap | 12 +- ...r__tests__django__parse_mixed_content.snap | 56 ++-- ...r__tests__django__parse_nested_for_if.snap | 16 +- ...__tests__errors__parse_error_recovery.snap | 16 +- ...ts__errors__parse_unclosed_django_for.snap | 4 +- ...sts__errors__parse_unclosed_django_if.snap | 4 +- ...er__tests__full_templates__parse_full.snap | 30 +-- 14 files changed, 214 insertions(+), 218 deletions(-) diff --git a/crates/djls-template-ast/src/ast.rs b/crates/djls-template-ast/src/ast.rs index 138808b..6f70324 100644 --- a/crates/djls-template-ast/src/ast.rs +++ b/crates/djls-template-ast/src/ast.rs @@ -97,7 +97,7 @@ impl From for Span { let token_start = token.start().unwrap_or(0); match token.token_type() { TokenType::Comment(_, start, _) => token_start + start.len() as u32, - TokenType::DjangoBlock(_) | TokenType::DjangoVariable(_) => token_start + 2, + TokenType::DjangoBlock(_) | TokenType::DjangoVariable(_) => token_start + 3, _ => token_start, } }; diff --git a/crates/djls-template-ast/src/lexer.rs b/crates/djls-template-ast/src/lexer.rs index 83b55b8..4833183 100644 --- a/crates/djls-template-ast/src/lexer.rs +++ b/crates/djls-template-ast/src/lexer.rs @@ -22,140 +22,136 @@ impl Lexer { pub fn tokenize(&mut self) -> Result { let mut tokens = TokenStream::default(); + while !self.is_at_end() { - let token = self.next_token()?; + self.start = self.current; + + let token_type = match self.peek()? { + '{' => match self.peek_next()? { + '%' => { + self.consume_n(2)?; // {% + let content = self.consume_until("%}")?; + self.consume_n(2)?; // %} + TokenType::DjangoBlock(content) + } + '{' => { + self.consume_n(2)?; // {{ + let content = self.consume_until("}}")?; + self.consume_n(2)?; // }} + TokenType::DjangoVariable(content) + } + '#' => { + self.consume_n(2)?; // {# + let content = self.consume_until("#}")?; + self.consume_n(2)?; // #} + TokenType::Comment(content, "{#".to_string(), Some("#}".to_string())) + } + _ => { + self.consume()?; // { + TokenType::Text(String::from("{")) + } + }, + + '<' => match self.peek_next()? { + '/' => { + self.consume_n(2)?; // ")?; + self.consume()?; // > + TokenType::HtmlTagClose(tag) + } + '!' if self.matches("")?; + self.consume_n(3)?; // --> + TokenType::Comment(content, "".to_string())) + } + _ => { + self.consume()?; // consume < + let tag = self.consume_until(">")?; + self.consume()?; // consume > + if tag.starts_with("script") { + TokenType::ScriptTagOpen(tag) + } else if tag.starts_with("style") { + TokenType::StyleTagOpen(tag) + } else if tag.ends_with("/") { + TokenType::HtmlTagVoid(tag.trim_end_matches("/").to_string()) + } else { + TokenType::HtmlTagOpen(tag) + } + } + }, + + '/' => match self.peek_next()? { + '/' => { + self.consume_n(2)?; // // + let content = self.consume_until("\n")?; + TokenType::Comment(content, "//".to_string(), None) + } + '*' => { + self.consume_n(2)?; // /* + let content = self.consume_until("*/")?; + self.consume_n(2)?; // */ + TokenType::Comment(content, "/*".to_string(), Some("*/".to_string())) + } + _ => { + self.consume()?; + TokenType::Text("/".to_string()) + } + }, + + c if c.is_whitespace() => { + if c == '\n' || c == '\r' { + self.consume()?; // \r or \n + if c == '\r' && self.peek()? == '\n' { + self.consume()?; // \n of \r\n + } + TokenType::Newline + } else { + self.consume()?; // Consume the first whitespace + while !self.is_at_end() && self.peek()?.is_whitespace() { + if self.peek()? == '\n' || self.peek()? == '\r' { + break; + } + self.consume()?; + } + let whitespace_count = self.current - self.start; + TokenType::Whitespace(whitespace_count) + } + } + + _ => { + let mut text = String::new(); + while !self.is_at_end() { + let c = self.peek()?; + if c == '{' || c == '<' || c == '\n' { + break; + } + text.push(c); + self.consume()?; + } + TokenType::Text(text) + } + }; + + let token = Token::new(token_type, self.line, Some(self.start)); + + match self.peek_previous()? { + '\n' => self.line += 1, + '\r' => { + self.line += 1; + if self.peek()? == '\n' { + self.current += 1; + } + } + _ => {} + } + tokens.add_token(token); } tokens.finalize(self.line); Ok(tokens) } - fn next_token(&mut self) -> Result { - self.start = self.current; - - let token_type = match self.peek()? { - '{' => match self.peek_next()? { - '%' => { - self.consume_n(2)?; // {% - let content = self.consume_until("%}")?; - self.consume_n(2)?; // %} - TokenType::DjangoBlock(content) - } - '{' => { - self.consume_n(2)?; // {{ - let content = self.consume_until("}}")?; - self.consume_n(2)?; // }} - TokenType::DjangoVariable(content) - } - '#' => { - self.consume_n(2)?; // {# - let content = self.consume_until("#}")?; - self.consume_n(2)?; // #} - TokenType::Comment(content, "{#".to_string(), Some("#}".to_string())) - } - _ => { - self.consume()?; // { - TokenType::Text(String::from("{")) - } - }, - - '<' => match self.peek_next()? { - '/' => { - self.consume_n(2)?; // ")?; - self.consume()?; // > - TokenType::HtmlTagClose(tag) - } - '!' if self.matches("")?; - self.consume_n(3)?; // --> - TokenType::Comment(content, "".to_string())) - } - _ => { - self.consume()?; // consume < - let tag = self.consume_until(">")?; - self.consume()?; // consume > - if tag.starts_with("script") { - TokenType::ScriptTagOpen(tag) - } else if tag.starts_with("style") { - TokenType::StyleTagOpen(tag) - } else if tag.ends_with("/") { - TokenType::HtmlTagVoid(tag.trim_end_matches("/").to_string()) - } else { - TokenType::HtmlTagOpen(tag) - } - } - }, - - '/' => match self.peek_next()? { - '/' => { - self.consume_n(2)?; // // - let content = self.consume_until("\n")?; - TokenType::Comment(content, "//".to_string(), None) - } - '*' => { - self.consume_n(2)?; // /* - let content = self.consume_until("*/")?; - self.consume_n(2)?; // */ - TokenType::Comment(content, "/*".to_string(), Some("*/".to_string())) - } - _ => { - self.consume()?; - TokenType::Text("/".to_string()) - } - }, - - c if c.is_whitespace() => { - if c == '\n' || c == '\r' { - self.consume()?; // \r or \n - if c == '\r' && self.peek()? == '\n' { - self.consume()?; // \n of \r\n - } - TokenType::Newline - } else { - self.consume()?; // Consume the first whitespace - while !self.is_at_end() && self.peek()?.is_whitespace() { - if self.peek()? == '\n' || self.peek()? == '\r' { - break; - } - self.consume()?; - } - let whitespace_count = self.current - self.start; - TokenType::Whitespace(whitespace_count) - } - } - - _ => { - let mut text = String::new(); - while !self.is_at_end() { - let c = self.peek()?; - if c == '{' || c == '<' || c == '\n' { - break; - } - text.push(c); - self.consume()?; - } - TokenType::Text(text) - } - }; - - let token = Token::new(token_type, self.line, Some(self.start)); - - match self.peek_previous()? { - '\n' => self.line += 1, - '\r' => { - self.line += 1; - if self.peek()? == '\n' { - self.current += 1; - } - } - _ => {} - } - - Ok(token) - } - fn peek(&self) -> Result { self.peek_at(0) } diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap index d3852fe..a3665d6 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__comments__parse_comments.snap @@ -11,7 +11,7 @@ nodes: - Comment: content: Django comment span: - start: 21 + start: 23 length: 14 line_offsets: - 0 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_complex_if_elif.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_complex_if_elif.snap index eccc889..640a9aa 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_complex_if_elif.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_complex_if_elif.snap @@ -13,10 +13,10 @@ nodes: - ">" - "0" span: - start: 0 + start: 3 length: 8 tag_span: - start: 0 + start: 3 length: 8 assignment: ~ nodes: @@ -35,10 +35,10 @@ nodes: - "<" - "0" span: - start: 22 + start: 25 length: 10 tag_span: - start: 22 + start: 25 length: 10 assignment: ~ nodes: @@ -54,10 +54,10 @@ nodes: bits: - else span: - start: 46 + start: 49 length: 4 tag_span: - start: 46 + start: 49 length: 4 assignment: ~ nodes: @@ -73,10 +73,10 @@ nodes: bits: - endif span: - start: 60 + start: 63 length: 5 tag_span: - start: 60 + start: 63 length: 5 assignment: ~ assignments: ~ diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_for_block.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_for_block.snap index 8d80375..8bc778e 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_for_block.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_for_block.snap @@ -13,10 +13,10 @@ nodes: - in - items span: - start: 0 + start: 3 length: 17 tag_span: - start: 0 + start: 3 length: 17 assignment: ~ nodes: @@ -34,10 +34,10 @@ nodes: bits: - empty span: - start: 33 + start: 36 length: 5 tag_span: - start: 33 + start: 36 length: 5 assignment: ~ nodes: @@ -53,10 +53,10 @@ nodes: bits: - endfor span: - start: 52 + start: 55 length: 6 tag_span: - start: 52 + start: 55 length: 6 assignment: ~ assignments: ~ diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_if_block.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_if_block.snap index 7a18941..298526a 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_if_block.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_if_block.snap @@ -11,10 +11,10 @@ nodes: - if - user.is_authenticated span: - start: 0 + start: 3 length: 24 tag_span: - start: 0 + start: 3 length: 24 assignment: ~ nodes: @@ -30,10 +30,10 @@ nodes: bits: - endif span: - start: 37 + start: 40 length: 5 tag_span: - start: 37 + start: 40 length: 5 assignment: ~ assignments: ~ diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_variable_with_filter.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_variable_with_filter.snap index 79cf610..490e866 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_variable_with_filter.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_django_variable_with_filter.snap @@ -11,8 +11,8 @@ nodes: - name: title args: [] span: - start: 4 - length: 15 + start: 10 + length: 5 span: start: 3 length: 15 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_filter_chains.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_filter_chains.snap index 5c907df..460f2ca 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_filter_chains.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_filter_chains.snap @@ -11,18 +11,18 @@ nodes: args: - "'nothing'" span: - start: 4 - length: 35 + start: 6 + length: 17 - name: title args: [] span: - start: 4 - length: 35 + start: 24 + length: 5 - name: upper args: [] span: - start: 4 - length: 35 + start: 30 + length: 5 span: start: 3 length: 35 diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap index e74051d..0df1a6a 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_mixed_content.snap @@ -16,10 +16,10 @@ nodes: - if - user.is_authenticated span: - start: 9 + start: 12 length: 24 tag_span: - start: 9 + start: 12 length: 24 assignment: ~ nodes: @@ -31,14 +31,14 @@ nodes: - name: title args: [] span: - start: 48 - length: 31 + start: 54 + length: 5 - name: default args: - "'Guest'" span: - start: 48 - length: 31 + start: 60 + length: 15 span: start: 47 length: 31 @@ -52,10 +52,10 @@ nodes: - in - user.groups span: - start: 86 + start: 89 length: 24 tag_span: - start: 86 + start: 89 length: 24 assignment: ~ nodes: @@ -67,10 +67,10 @@ nodes: - if - forloop.first span: - start: 125 + start: 128 length: 16 tag_span: - start: 125 + start: 128 length: 16 assignment: ~ nodes: @@ -86,10 +86,10 @@ nodes: bits: - endif span: - start: 148 + start: 151 length: 5 tag_span: - start: 148 + start: 151 length: 5 assignment: ~ assignments: ~ @@ -110,10 +110,10 @@ nodes: - not - forloop.last span: - start: 193 + start: 196 length: 19 tag_span: - start: 193 + start: 196 length: 19 assignment: ~ nodes: @@ -129,10 +129,10 @@ nodes: bits: - endif span: - start: 220 + start: 223 length: 5 tag_span: - start: 220 + start: 223 length: 5 assignment: ~ assignments: ~ @@ -144,10 +144,10 @@ nodes: - if - forloop.last span: - start: 240 + start: 243 length: 15 tag_span: - start: 240 + start: 243 length: 15 assignment: ~ nodes: @@ -163,10 +163,10 @@ nodes: bits: - endif span: - start: 262 + start: 265 length: 5 tag_span: - start: 262 + start: 265 length: 5 assignment: ~ assignments: ~ @@ -177,10 +177,10 @@ nodes: bits: - empty span: - start: 278 + start: 281 length: 5 tag_span: - start: 278 + start: 281 length: 5 assignment: ~ nodes: @@ -196,10 +196,10 @@ nodes: bits: - endfor span: - start: 314 + start: 317 length: 6 tag_span: - start: 314 + start: 317 length: 6 assignment: ~ assignments: ~ @@ -210,10 +210,10 @@ nodes: bits: - else span: - start: 327 + start: 330 length: 4 tag_span: - start: 327 + start: 330 length: 4 assignment: ~ nodes: @@ -229,10 +229,10 @@ nodes: bits: - endif span: - start: 348 + start: 351 length: 5 tag_span: - start: 348 + start: 351 length: 5 assignment: ~ assignments: ~ diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_nested_for_if.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_nested_for_if.snap index b0f9f82..ab81746 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_nested_for_if.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__django__parse_nested_for_if.snap @@ -13,10 +13,10 @@ nodes: - in - items span: - start: 0 + start: 3 length: 17 tag_span: - start: 0 + start: 3 length: 17 assignment: ~ nodes: @@ -28,10 +28,10 @@ nodes: - if - item.active span: - start: 23 + start: 26 length: 14 tag_span: - start: 23 + start: 26 length: 14 assignment: ~ nodes: @@ -50,10 +50,10 @@ nodes: bits: - endif span: - start: 58 + start: 61 length: 5 tag_span: - start: 58 + start: 61 length: 5 assignment: ~ assignments: ~ @@ -64,10 +64,10 @@ nodes: bits: - endfor span: - start: 69 + start: 72 length: 6 tag_span: - start: 69 + start: 72 length: 6 assignment: ~ assignments: ~ diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap index c314a91..d663c12 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_error_recovery.snap @@ -21,17 +21,17 @@ nodes: - if - user.is_authenticated span: - start: 48 + start: 51 length: 24 tag_span: - start: 48 + start: 51 length: 24 assignment: ~ nodes: - Comment: content: This if is unclosed which does matter span: - start: 87 + start: 89 length: 37 - Text: content: "

Welcome" @@ -59,7 +59,7 @@ nodes: - Comment: content: "This div is unclosed which doesn't matter" span: - start: 196 + start: 198 length: 41 - Block: Block: @@ -71,10 +71,10 @@ nodes: - in - items span: - start: 252 + start: 255 length: 17 tag_span: - start: 252 + start: 255 length: 17 assignment: ~ nodes: @@ -102,10 +102,10 @@ nodes: bits: - endfor span: - start: 320 + start: 323 length: 6 tag_span: - start: 320 + start: 323 length: 6 assignment: ~ assignments: ~ diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_for.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_for.snap index fdf363b..39188cf 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_for.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_for.snap @@ -13,10 +13,10 @@ nodes: - in - items span: - start: 0 + start: 3 length: 17 tag_span: - start: 0 + start: 3 length: 17 assignment: ~ nodes: diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_if.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_if.snap index f998c29..04839d7 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_if.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__errors__parse_unclosed_django_if.snap @@ -11,10 +11,10 @@ nodes: - if - user.is_authenticated span: - start: 0 + start: 3 length: 24 tag_span: - start: 0 + start: 3 length: 24 assignment: ~ nodes: diff --git a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap index fdd15b9..64cf292 100644 --- a/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap +++ b/crates/djls-template-ast/src/snapshots/djls_template_ast__parser__tests__full_templates__parse_full.snap @@ -101,17 +101,17 @@ nodes: - if - user.is_authenticated span: - start: 463 + start: 466 length: 24 tag_span: - start: 463 + start: 466 length: 24 assignment: ~ nodes: - Comment: content: Welcome message span: - start: 510 + start: 512 length: 15 - Text: content: "

Welcome," @@ -126,14 +126,14 @@ nodes: - name: title args: [] span: - start: 565 - length: 31 + start: 571 + length: 5 - name: default args: - "'Guest'" span: - start: 565 - length: 31 + start: 577 + length: 15 span: start: 564 length: 31 @@ -150,10 +150,10 @@ nodes: - if - user.is_staff span: - start: 621 + start: 624 length: 16 tag_span: - start: 621 + start: 624 length: 16 assignment: ~ nodes: @@ -169,10 +169,10 @@ nodes: bits: - else span: - start: 699 + start: 702 length: 4 tag_span: - start: 699 + start: 702 length: 4 assignment: ~ nodes: @@ -188,10 +188,10 @@ nodes: bits: - endif span: - start: 764 + start: 767 length: 5 tag_span: - start: 764 + start: 767 length: 5 assignment: ~ assignments: ~ @@ -202,10 +202,10 @@ nodes: bits: - endif span: - start: 788 + start: 791 length: 5 tag_span: - start: 788 + start: 791 length: 5 assignment: ~ assignments: ~