checkpoint

This commit is contained in:
Josh Thomas 2025-01-06 11:06:26 -06:00
parent 6d26d78099
commit ad9d6e56b0
13 changed files with 83 additions and 831 deletions

View file

@ -261,12 +261,10 @@ impl Parser {
let start_token = self.peek_previous()?;
let start_pos = start_token.start().unwrap_or(0);
// If we start with a newline, skip it
if matches!(start_token.token_type(), TokenType::Newline) {
return self.next_node();
}
// Use TokenType's Display implementation for formatting
let mut text = start_token.token_type().to_string();
let mut total_length: u32 = u32::try_from(text.len()).unwrap();
@ -286,7 +284,6 @@ impl Parser {
}
}
// Skip empty text nodes
if text.trim().is_empty() {
self.next_node()
} else {
@ -303,14 +300,18 @@ impl Parser {
start: &str,
end: Option<&str>,
) -> Result<Node, ParserError> {
let start_token = self.peek_previous()?;
let start_pos = start_token.start().unwrap_or(0);
let total_length = (content.len() + start.len() + end.map_or(0, |e| e.len())) as u32;
let span = Span::new(start_pos, total_length);
Ok(Node::Comment {
content: content.to_string(),
span,
})
let token = self.peek_previous()?;
let start_pos = token.start().unwrap_or(0);
// Only treat Django comments as Comment nodes
if start == "{#" && end == Some("#}") {
Ok(Node::Comment {
content: content.to_string(),
span: Span::new(start_pos, token.token_type().len().unwrap_or(0) as u32),
})
} else {
self.parse_text()
}
}
fn peek(&self) -> Result<Token, ParserError> {

View file

@ -3,15 +3,15 @@ source: crates/djls-template-ast/src/parser.rs
expression: ast
---
nodes:
- Comment:
content: HTML comment
- Text:
content: "<!-- HTML comment -->"
span:
start: 0
length: 19
length: 21
- Comment:
content: Django comment
span:
start: 21
length: 18
length: 20
line_offsets:
- 0

View file

@ -188,7 +188,7 @@ nodes:
content: " (no groups)"
span:
start: 290
length: 20
length: 19
closing:
Closing:
tag:
@ -221,7 +221,7 @@ nodes:
content: " Guest"
span:
start: 338
length: 10
length: 9
closing:
Closing:
tag:

View file

@ -1,259 +0,0 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 516
expression: ast
snapshot_kind: text
---
nodes:
- Text:
content: "Welcome, "
span:
start: 0
length: 9
- Block:
Block:
tag:
name: if
bits:
- if
- user.is_authenticated
span:
start: 9
length: 24
tag_span:
start: 9
length: 24
assignment: ~
nodes:
- Variable:
bits:
- user
- name
filters:
- name: title
args: []
span:
start: 48
length: 31
- name: default
args:
- "'Guest'"
span:
start: 48
length: 31
span:
start: 47
length: 31
- Block:
Block:
tag:
name: for
bits:
- for
- group
- in
- user.groups
span:
start: 86
length: 24
tag_span:
start: 86
length: 24
assignment: ~
nodes:
- Block:
Block:
tag:
name: if
bits:
- if
- forloop.first
span:
start: 125
length: 16
tag_span:
start: 125
length: 16
assignment: ~
nodes:
- Text:
content: (
span:
start: 147
length: 1
closing:
Closing:
tag:
name: endif
bits:
- endif
span:
start: 148
length: 5
tag_span:
start: 148
length: 5
assignment: ~
assignments: ~
- Variable:
bits:
- group
- name
filters: []
span:
start: 171
length: 10
- Block:
Block:
tag:
name: if
bits:
- if
- not
- forloop.last
span:
start: 193
length: 19
tag_span:
start: 193
length: 19
assignment: ~
nodes:
- Text:
content: ", "
span:
start: 218
length: 2
closing:
Closing:
tag:
name: endif
bits:
- endif
span:
start: 220
length: 5
tag_span:
start: 220
length: 5
assignment: ~
assignments: ~
- Block:
Block:
tag:
name: if
bits:
- if
- forloop.last
span:
start: 240
length: 15
tag_span:
start: 240
length: 15
assignment: ~
nodes:
- Text:
content: )
span:
start: 261
length: 1
closing:
Closing:
tag:
name: endif
bits:
- endif
span:
start: 262
length: 5
tag_span:
start: 262
length: 5
assignment: ~
assignments: ~
- Block:
Branch:
tag:
name: empty
bits:
- empty
span:
start: 278
length: 5
tag_span:
start: 278
length: 5
assignment: ~
nodes:
- Text:
content: " (no groups)"
span:
start: 290
length: 19
closing:
Closing:
tag:
name: endfor
bits:
- endfor
span:
start: 314
length: 6
tag_span:
start: 314
length: 6
assignment: ~
assignments: ~
- Block:
Branch:
tag:
name: else
bits:
- else
span:
start: 327
length: 4
tag_span:
start: 327
length: 4
assignment: ~
nodes:
- Text:
content: " Guest"
span:
start: 338
length: 9
closing:
Closing:
tag:
name: endif
bits:
- endif
span:
start: 348
length: 5
tag_span:
start: 348
length: 5
assignment: ~
assignments: ~
- Text:
content: "!"
span:
start: 359
length: 1
line_offsets:
- 0
- 40
- 82
- 117
- 160
- 185
- 232
- 274
- 290
- 310
- 327
- 338
- 348

View file

@ -7,12 +7,12 @@ nodes:
content: "<div class=\"container\">"
span:
start: 0
length: 24
length: 23
- Text:
content: " <h1>Header</h1>"
span:
start: 24
length: 20
length: 19
- Block:
Block:
tag:
@ -32,7 +32,7 @@ nodes:
content: This if is unclosed which does matter
span:
start: 87
length: 41
length: 43
- Text:
content: " <p>Welcome "
span:
@ -50,17 +50,17 @@ nodes:
content: "</p>"
span:
start: 165
length: 5
length: 4
- Text:
content: " <div>"
span:
start: 170
length: 14
length: 13
- Comment:
content: "This div is unclosed which doesn't matter"
span:
start: 196
length: 45
length: 47
- Block:
Block:
tag:
@ -94,7 +94,7 @@ nodes:
content: "</span>"
span:
start: 304
length: 8
length: 7
closing:
Closing:
tag:
@ -113,7 +113,7 @@ nodes:
content: " <footer>Page Footer</footer>"
span:
start: 333
length: 33
length: 32
- Text:
content: "</div>"
span:

View file

@ -1,138 +0,0 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 637
expression: ast
snapshot_kind: text
---
nodes:
- Text:
content: "<div class=\"container\">"
span:
start: 0
length: 23
- Text:
content: " <h1>Header</h1>"
span:
start: 24
length: 19
- Block:
Block:
tag:
name: if
bits:
- if
- user.is_authenticated
span:
start: 48
length: 24
tag_span:
start: 48
length: 24
assignment: ~
nodes:
- Comment:
content: This if is unclosed which does matter
span:
start: 87
length: 41
- Text:
content: " <p>Welcome "
span:
start: 131
length: 19
- Variable:
bits:
- user
- name
filters: []
span:
start: 153
length: 9
- Text:
content: "</p>"
span:
start: 165
length: 4
- Text:
content: " <div>"
span:
start: 170
length: 13
- Comment:
content: "This div is unclosed which doesn't matter"
span:
start: 196
length: 45
- Block:
Block:
tag:
name: for
bits:
- for
- item
- in
- items
span:
start: 252
length: 17
tag_span:
start: 252
length: 17
assignment: ~
nodes:
- Text:
content: " <span>"
span:
start: 276
length: 18
- Variable:
bits:
- item
filters: []
span:
start: 297
length: 4
- Text:
content: "</span>"
span:
start: 304
length: 7
closing:
Closing:
tag:
name: endfor
bits:
- endfor
span:
start: 320
length: 6
tag_span:
start: 320
length: 6
assignment: ~
assignments: ~
- Text:
content: " <footer>Page Footer</footer>"
span:
start: 333
length: 32
- Text:
content: "</div>"
span:
start: 366
length: 6
closing: ~
assignments: ~
line_offsets:
- 0
- 24
- 44
- 79
- 131
- 170
- 184
- 244
- 276
- 312
- 333
- 366

View file

@ -7,92 +7,92 @@ nodes:
content: "<!DOCTYPE html>"
span:
start: 0
length: 16
length: 15
- Text:
content: "<html>"
span:
start: 16
length: 7
length: 6
- Text:
content: " <head>"
span:
start: 23
length: 11
length: 10
- Text:
content: " <style type=\"text/css\">"
span:
start: 34
length: 32
- Text:
content: " /* Style header */"
span:
start: 66
length: 31
- Text:
content: /* Style header */
span:
start: 78
length: 18
- Text:
content: " .header { color: blue; }"
span:
start: 97
length: 37
length: 36
- Text:
content: " </style>"
span:
start: 134
length: 17
length: 16
- Text:
content: " <script type=\"text/javascript\">"
span:
start: 151
length: 40
length: 39
- Text:
content: " // Init app"
content: // Init app
span:
start: 191
length: 24
start: 203
length: 11
- Text:
content: " const app = {"
span:
start: 215
length: 26
length: 25
- Text:
content: " /* Config */"
content: /* Config */
span:
start: 241
length: 29
start: 257
length: 12
- Text:
content: " debug: true"
span:
start: 270
length: 28
length: 27
- Text:
content: " };"
span:
start: 298
length: 15
length: 14
- Text:
content: " </script>"
span:
start: 313
length: 18
length: 17
- Text:
content: " </head>"
span:
start: 331
length: 12
length: 11
- Text:
content: " <body>"
span:
start: 343
length: 11
length: 10
- Text:
content: " <!-- Header section -->"
content: "<!-- Header section -->"
span:
start: 354
length: 32
start: 362
length: 23
- Text:
content: " <div class=\"header\" id=\"main\" data-value=\"123\" disabled>"
span:
start: 386
length: 65
length: 64
- Block:
Block:
tag:
@ -112,7 +112,7 @@ nodes:
content: Welcome message
span:
start: 510
length: 19
length: 21
- Text:
content: " <h1>Welcome, "
span:
@ -141,7 +141,7 @@ nodes:
content: "!</h1>"
span:
start: 598
length: 7
length: 6
- Block:
Block:
tag:
@ -161,7 +161,7 @@ nodes:
content: " <span>Admin</span>"
span:
start: 644
length: 39
length: 38
- Block:
Branch:
tag:
@ -180,7 +180,7 @@ nodes:
content: " <span>User</span>"
span:
start: 710
length: 38
length: 37
closing:
Closing:
tag:
@ -213,12 +213,12 @@ nodes:
content: " </div>"
span:
start: 800
length: 15
length: 14
- Text:
content: " </body>"
span:
start: 815
length: 12
length: 11
- Text:
content: "</html>"
span:

View file

@ -1,259 +0,0 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 682
expression: ast
snapshot_kind: text
---
nodes:
- Text:
content: "<!DOCTYPE html>"
span:
start: 0
length: 15
- Text:
content: "<html>"
span:
start: 16
length: 6
- Text:
content: " <head>"
span:
start: 23
length: 10
- Text:
content: " <style type=\"text/css\">"
span:
start: 34
length: 31
- Comment:
content: Style header
span:
start: 78
length: 16
- Text:
content: " .header { color: blue; }"
span:
start: 97
length: 36
- Text:
content: " </style>"
span:
start: 134
length: 16
- Text:
content: " <script type=\"text/javascript\">"
span:
start: 151
length: 39
- Comment:
content: Init app
span:
start: 203
length: 10
- Text:
content: " const app = {"
span:
start: 215
length: 25
- Comment:
content: Config
span:
start: 257
length: 10
- Text:
content: " debug: true"
span:
start: 270
length: 27
- Text:
content: " };"
span:
start: 298
length: 14
- Text:
content: " </script>"
span:
start: 313
length: 17
- Text:
content: " </head>"
span:
start: 331
length: 11
- Text:
content: " <body>"
span:
start: 343
length: 10
- Comment:
content: Header section
span:
start: 362
length: 21
- Text:
content: " <div class=\"header\" id=\"main\" data-value=\"123\" disabled>"
span:
start: 386
length: 64
- Block:
Block:
tag:
name: if
bits:
- if
- user.is_authenticated
span:
start: 463
length: 24
tag_span:
start: 463
length: 24
assignment: ~
nodes:
- Comment:
content: Welcome message
span:
start: 510
length: 19
- Text:
content: " <h1>Welcome, "
span:
start: 532
length: 29
- Variable:
bits:
- user
- name
filters:
- name: title
args: []
span:
start: 565
length: 31
- name: default
args:
- "'Guest'"
span:
start: 565
length: 31
span:
start: 564
length: 31
- Text:
content: "!</h1>"
span:
start: 598
length: 6
- Block:
Block:
tag:
name: if
bits:
- if
- user.is_staff
span:
start: 621
length: 16
tag_span:
start: 621
length: 16
assignment: ~
nodes:
- Text:
content: " <span>Admin</span>"
span:
start: 644
length: 38
- Block:
Branch:
tag:
name: else
bits:
- else
span:
start: 699
length: 4
tag_span:
start: 699
length: 4
assignment: ~
nodes:
- Text:
content: " <span>User</span>"
span:
start: 710
length: 37
closing:
Closing:
tag:
name: endif
bits:
- endif
span:
start: 764
length: 5
tag_span:
start: 764
length: 5
assignment: ~
assignments: ~
closing:
Closing:
tag:
name: endif
bits:
- endif
span:
start: 788
length: 5
tag_span:
start: 788
length: 5
assignment: ~
assignments: ~
- Text:
content: " </div>"
span:
start: 800
length: 14
- Text:
content: " </body>"
span:
start: 815
length: 11
- Text:
content: "</html>"
span:
start: 827
length: 7
line_offsets:
- 0
- 16
- 23
- 34
- 66
- 97
- 134
- 151
- 191
- 215
- 241
- 270
- 298
- 313
- 331
- 343
- 354
- 386
- 451
- 494
- 532
- 605
- 644
- 683
- 710
- 748
- 776
- 800
- 815
- 827

View file

@ -7,32 +7,27 @@ nodes:
content: "<script type=\"text/javascript\">"
span:
start: 0
length: 32
length: 31
- Text:
content: " // Single line comment"
content: // Single line comment
span:
start: 32
length: 27
start: 36
length: 22
- Text:
content: " const x = 1;"
span:
start: 59
length: 17
length: 16
- Text:
content: " /* Multi-line"
content: "/* Multi-line\n comment */"
span:
start: 76
length: 18
- Text:
content: " comment */"
span:
start: 94
length: 19
start: 80
length: 32
- Text:
content: " console.log(x);"
span:
start: 113
length: 20
length: 19
- Text:
content: "</script>"
span:
@ -43,6 +38,5 @@ line_offsets:
- 32
- 59
- 76
- 94
- 113
- 133

View file

@ -1,44 +0,0 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 534
expression: ast
snapshot_kind: text
---
nodes:
- Text:
content: "<script type=\"text/javascript\">"
span:
start: 0
length: 31
- Comment:
content: Single line comment
span:
start: 36
length: 21
- Text:
content: " const x = 1;"
span:
start: 59
length: 16
- Comment:
content: "Multi-line\n comment"
span:
start: 80
length: 30
- Text:
content: " console.log(x);"
span:
start: 113
length: 19
- Text:
content: "</script>"
span:
start: 133
length: 9
line_offsets:
- 0
- 32
- 59
- 76
- 113
- 133

View file

@ -7,27 +7,27 @@ nodes:
content: "<style type=\"text/css\">"
span:
start: 0
length: 24
length: 23
- Text:
content: " /* Header styles */"
content: /* Header styles */
span:
start: 24
length: 24
start: 28
length: 19
- Text:
content: " .header {"
span:
start: 48
length: 14
length: 13
- Text:
content: " color: blue;"
span:
start: 62
length: 21
length: 20
- Text:
content: " }"
span:
start: 83
length: 6
length: 5
- Text:
content: "</style>"
span:

View file

@ -1,44 +0,0 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 551
expression: ast
snapshot_kind: text
---
nodes:
- Text:
content: "<style type=\"text/css\">"
span:
start: 0
length: 23
- Comment:
content: Header styles
span:
start: 28
length: 17
- Text:
content: " .header {"
span:
start: 48
length: 13
- Text:
content: " color: blue;"
span:
start: 62
length: 20
- Text:
content: " }"
span:
start: 83
length: 5
- Text:
content: "</style>"
span:
start: 89
length: 8
line_offsets:
- 0
- 24
- 48
- 62
- 83
- 89

View file

@ -33,10 +33,11 @@ impl TokenType {
| TokenType::StyleTagOpen(s)
| TokenType::StyleTagClose(s)
| TokenType::Text(s) => Some(s.len()),
TokenType::Comment(content, start, end) => {
Some(content.len() + start.len() + end.as_ref().map_or(0, |e| e.len()))
}
TokenType::Whitespace(len) => Some(len.clone()),
TokenType::Comment(content, start, end) => match end {
Some(end) => Some(start.len() + 1 + content.len() + 1 + end.len()),
None => Some(start.len() + 1 + content.len()),
},
TokenType::Whitespace(len) => Some(*len),
TokenType::Newline => Some(1),
TokenType::Eof => None,
}
@ -54,7 +55,7 @@ impl TokenType {
| TokenType::StyleTagOpen(s)
| TokenType::StyleTagClose(s)
| TokenType::Text(s) => s,
TokenType::Comment(content, _, _) => content, // Just return the content
TokenType::Comment(content, _, _) => content,
TokenType::Whitespace(_) => " ",
TokenType::Newline => "\n",
TokenType::Eof => "",
@ -68,8 +69,8 @@ impl fmt::Display for TokenType {
match self {
Comment(content, start, end) => match end {
Some(end) => write!(f, "{}{}{}", start, content, end),
None => write!(f, "{}{}", start, content),
Some(end) => write!(f, "{} {} {}", start, content, end),
None => write!(f, "{} {}", start, content),
},
DjangoBlock(s) => write!(f, "{{% {} %}}", s),
DjangoVariable(s) => write!(f, "{{{{ {} }}}}", s),