checkpoint

This commit is contained in:
Josh Thomas 2025-01-06 10:35:05 -06:00
parent bc54814229
commit 6d26d78099
13 changed files with 710 additions and 114 deletions

View file

@ -52,9 +52,6 @@ impl Parser {
}
fn next_node(&mut self) -> Result<Node, ParserError> {
if self.is_at_end() {
return Err(ParserError::Ast(AstError::StreamError("AtEnd".to_string())));
}
let token = self.peek()?;
match token.token_type() {
TokenType::DjangoBlock(content) => {
@ -65,6 +62,10 @@ impl Parser {
self.consume()?;
self.parse_django_variable(content)
}
TokenType::Comment(content, start, end) => {
self.consume()?;
self.parse_comment(content, start, end.as_deref())
}
TokenType::Text(_)
| TokenType::Whitespace(_)
| TokenType::Newline
@ -74,10 +75,9 @@ impl Parser {
| TokenType::ScriptTagOpen(_)
| TokenType::ScriptTagClose(_)
| TokenType::StyleTagOpen(_)
| TokenType::StyleTagClose(_) => self.parse_text(),
TokenType::Comment(content, start, end) => {
| TokenType::StyleTagClose(_) => {
self.consume()?;
self.parse_comment(content, start, end.as_deref())
self.parse_text()
}
TokenType::Eof => Err(ParserError::Ast(AstError::StreamError("AtEnd".to_string()))),
}
@ -136,13 +136,16 @@ impl Parser {
tag: branch_tag.clone(),
nodes: branch_nodes.clone(),
}));
closing = Some(Box::new(Block::Closing { tag: next_tag.clone() }));
closing = Some(Box::new(Block::Closing {
tag: next_tag.clone(),
}));
found_closing = true;
break;
}
}
// Check if this is another branch tag
if branches.iter().any(|b| b.name == next_tag.name) {
if branches.iter().any(|b| b.name == next_tag.name)
{
// Push the current branch and start a new one
nodes.push(Node::Block(Block::Branch {
tag: branch_tag.clone(),
@ -164,7 +167,9 @@ impl Parser {
nodes: branch_nodes.clone(),
}));
// Add error for unclosed tag
self.errors.push(ParserError::Ast(AstError::UnclosedTag(tag_name.clone())));
self.errors.push(ParserError::Ast(AstError::UnclosedTag(
tag_name.clone(),
)));
}
if found_closing {
break;
@ -198,9 +203,16 @@ impl Parser {
};
// Add error if we didn't find a closing tag for a block
if let Block::Block { closing: None, tag: tag_ref, .. } = &block {
if let Block::Block {
closing: None,
tag: tag_ref,
..
} = &block
{
if let Some(expected_closing) = &spec.closing {
self.errors.push(ParserError::Ast(AstError::UnclosedTag(tag_ref.name.clone())));
self.errors.push(ParserError::Ast(AstError::UnclosedTag(
tag_ref.name.clone(),
)));
}
}
@ -246,58 +258,40 @@ impl Parser {
}
fn parse_text(&mut self) -> Result<Node, ParserError> {
let start_token = self.peek()?;
let start_token = self.peek_previous()?;
let start_pos = start_token.start().unwrap_or(0);
let mut total_length = start_token.length().unwrap_or(0);
// Handle newlines by returning next node
// If we start with a newline, skip it
if matches!(start_token.token_type(), TokenType::Newline) {
self.consume()?;
let node = self.next_node()?;
return Ok(node);
return self.next_node();
}
let mut content = match start_token.token_type() {
TokenType::Text(text) => text.to_string(),
TokenType::Whitespace(count) => " ".repeat(*count),
_ => {
return Err(ParserError::Ast(AstError::InvalidTag(
"Expected text or whitespace token".to_string(),
)))
}
};
self.consume()?;
// Use TokenType's Display implementation for formatting
let mut text = start_token.token_type().to_string();
let mut total_length: u32 = u32::try_from(text.len()).unwrap();
// Look ahead for more tokens until newline
while let Ok(next_token) = self.peek() {
match next_token.token_type() {
TokenType::Text(text) => {
content.push_str(text);
total_length += next_token.length().unwrap_or(0);
while let Ok(token) = self.peek() {
match token.token_type() {
TokenType::DjangoBlock(_)
| TokenType::DjangoVariable(_)
| TokenType::Comment(_, _, _)
| TokenType::Newline
| TokenType::Eof => break,
_ => {
let token_text = token.token_type().to_string();
text.push_str(&token_text);
total_length += u32::try_from(token_text.len()).unwrap();
self.consume()?;
}
TokenType::Whitespace(count) => {
content.push_str(&" ".repeat(*count));
total_length += next_token.length().unwrap_or(0);
self.consume()?;
}
TokenType::Newline => {
// Include newline in span but not content
total_length += next_token.length().unwrap_or(0);
self.consume()?;
break;
}
_ => break,
}
}
// Skip empty text nodes
if content.trim().is_empty() {
let node = self.next_node()?;
Ok(node)
if text.trim().is_empty() {
self.next_node()
} else {
Ok(Node::Text {
content,
content: text,
span: Span::new(start_pos, total_length),
})
}

View file

@ -0,0 +1,259 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 516
expression: ast
snapshot_kind: text
---
nodes:
- Text:
content: "Welcome, "
span:
start: 0
length: 9
- Block:
Block:
tag:
name: if
bits:
- if
- user.is_authenticated
span:
start: 9
length: 24
tag_span:
start: 9
length: 24
assignment: ~
nodes:
- Variable:
bits:
- user
- name
filters:
- name: title
args: []
span:
start: 48
length: 31
- name: default
args:
- "'Guest'"
span:
start: 48
length: 31
span:
start: 47
length: 31
- Block:
Block:
tag:
name: for
bits:
- for
- group
- in
- user.groups
span:
start: 86
length: 24
tag_span:
start: 86
length: 24
assignment: ~
nodes:
- Block:
Block:
tag:
name: if
bits:
- if
- forloop.first
span:
start: 125
length: 16
tag_span:
start: 125
length: 16
assignment: ~
nodes:
- Text:
content: (
span:
start: 147
length: 1
closing:
Closing:
tag:
name: endif
bits:
- endif
span:
start: 148
length: 5
tag_span:
start: 148
length: 5
assignment: ~
assignments: ~
- Variable:
bits:
- group
- name
filters: []
span:
start: 171
length: 10
- Block:
Block:
tag:
name: if
bits:
- if
- not
- forloop.last
span:
start: 193
length: 19
tag_span:
start: 193
length: 19
assignment: ~
nodes:
- Text:
content: ", "
span:
start: 218
length: 2
closing:
Closing:
tag:
name: endif
bits:
- endif
span:
start: 220
length: 5
tag_span:
start: 220
length: 5
assignment: ~
assignments: ~
- Block:
Block:
tag:
name: if
bits:
- if
- forloop.last
span:
start: 240
length: 15
tag_span:
start: 240
length: 15
assignment: ~
nodes:
- Text:
content: )
span:
start: 261
length: 1
closing:
Closing:
tag:
name: endif
bits:
- endif
span:
start: 262
length: 5
tag_span:
start: 262
length: 5
assignment: ~
assignments: ~
- Block:
Branch:
tag:
name: empty
bits:
- empty
span:
start: 278
length: 5
tag_span:
start: 278
length: 5
assignment: ~
nodes:
- Text:
content: " (no groups)"
span:
start: 290
length: 19
closing:
Closing:
tag:
name: endfor
bits:
- endfor
span:
start: 314
length: 6
tag_span:
start: 314
length: 6
assignment: ~
assignments: ~
- Block:
Branch:
tag:
name: else
bits:
- else
span:
start: 327
length: 4
tag_span:
start: 327
length: 4
assignment: ~
nodes:
- Text:
content: " Guest"
span:
start: 338
length: 9
closing:
Closing:
tag:
name: endif
bits:
- endif
span:
start: 348
length: 5
tag_span:
start: 348
length: 5
assignment: ~
assignments: ~
- Text:
content: "!"
span:
start: 359
length: 1
line_offsets:
- 0
- 40
- 82
- 117
- 160
- 185
- 232
- 274
- 290
- 310
- 327
- 338
- 348

View file

@ -1,10 +1,128 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 643
assertion_line: 637
expression: ast
snapshot_kind: text
---
nodes: []
nodes:
- Text:
content: "<div class=\"container\">"
span:
start: 0
length: 23
- Text:
content: " <h1>Header</h1>"
span:
start: 24
length: 19
- Block:
Block:
tag:
name: if
bits:
- if
- user.is_authenticated
span:
start: 48
length: 24
tag_span:
start: 48
length: 24
assignment: ~
nodes:
- Comment:
content: This if is unclosed which does matter
span:
start: 87
length: 41
- Text:
content: " <p>Welcome "
span:
start: 131
length: 19
- Variable:
bits:
- user
- name
filters: []
span:
start: 153
length: 9
- Text:
content: "</p>"
span:
start: 165
length: 4
- Text:
content: " <div>"
span:
start: 170
length: 13
- Comment:
content: "This div is unclosed which doesn't matter"
span:
start: 196
length: 45
- Block:
Block:
tag:
name: for
bits:
- for
- item
- in
- items
span:
start: 252
length: 17
tag_span:
start: 252
length: 17
assignment: ~
nodes:
- Text:
content: " <span>"
span:
start: 276
length: 18
- Variable:
bits:
- item
filters: []
span:
start: 297
length: 4
- Text:
content: "</span>"
span:
start: 304
length: 7
closing:
Closing:
tag:
name: endfor
bits:
- endfor
span:
start: 320
length: 6
tag_span:
start: 320
length: 6
assignment: ~
assignments: ~
- Text:
content: " <footer>Page Footer</footer>"
span:
start: 333
length: 32
- Text:
content: "</div>"
span:
start: 366
length: 6
closing: ~
assignments: ~
line_offsets:
- 0
- 24

View file

@ -1,9 +0,0 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 581
expression: ast
snapshot_kind: text
---
nodes: []
line_offsets:
- 0

View file

@ -1,9 +0,0 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 614
expression: ast
snapshot_kind: text
---
nodes: []
line_offsets:
- 0

View file

@ -1,9 +0,0 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 623
expression: ast
snapshot_kind: text
---
nodes: []
line_offsets:
- 0

View file

@ -1,10 +1,231 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 688
assertion_line: 682
expression: ast
snapshot_kind: text
---
nodes: []
nodes:
- Text:
content: "<!DOCTYPE html>"
span:
start: 0
length: 15
- Text:
content: "<html>"
span:
start: 16
length: 6
- Text:
content: " <head>"
span:
start: 23
length: 10
- Text:
content: " <style type=\"text/css\">"
span:
start: 34
length: 31
- Comment:
content: Style header
span:
start: 78
length: 16
- Text:
content: " .header { color: blue; }"
span:
start: 97
length: 36
- Text:
content: " </style>"
span:
start: 134
length: 16
- Text:
content: " <script type=\"text/javascript\">"
span:
start: 151
length: 39
- Comment:
content: Init app
span:
start: 203
length: 10
- Text:
content: " const app = {"
span:
start: 215
length: 25
- Comment:
content: Config
span:
start: 257
length: 10
- Text:
content: " debug: true"
span:
start: 270
length: 27
- Text:
content: " };"
span:
start: 298
length: 14
- Text:
content: " </script>"
span:
start: 313
length: 17
- Text:
content: " </head>"
span:
start: 331
length: 11
- Text:
content: " <body>"
span:
start: 343
length: 10
- Comment:
content: Header section
span:
start: 362
length: 21
- Text:
content: " <div class=\"header\" id=\"main\" data-value=\"123\" disabled>"
span:
start: 386
length: 64
- Block:
Block:
tag:
name: if
bits:
- if
- user.is_authenticated
span:
start: 463
length: 24
tag_span:
start: 463
length: 24
assignment: ~
nodes:
- Comment:
content: Welcome message
span:
start: 510
length: 19
- Text:
content: " <h1>Welcome, "
span:
start: 532
length: 29
- Variable:
bits:
- user
- name
filters:
- name: title
args: []
span:
start: 565
length: 31
- name: default
args:
- "'Guest'"
span:
start: 565
length: 31
span:
start: 564
length: 31
- Text:
content: "!</h1>"
span:
start: 598
length: 6
- Block:
Block:
tag:
name: if
bits:
- if
- user.is_staff
span:
start: 621
length: 16
tag_span:
start: 621
length: 16
assignment: ~
nodes:
- Text:
content: " <span>Admin</span>"
span:
start: 644
length: 38
- Block:
Branch:
tag:
name: else
bits:
- else
span:
start: 699
length: 4
tag_span:
start: 699
length: 4
assignment: ~
nodes:
- Text:
content: " <span>User</span>"
span:
start: 710
length: 37
closing:
Closing:
tag:
name: endif
bits:
- endif
span:
start: 764
length: 5
tag_span:
start: 764
length: 5
assignment: ~
assignments: ~
closing:
Closing:
tag:
name: endif
bits:
- endif
span:
start: 788
length: 5
tag_span:
start: 788
length: 5
assignment: ~
assignments: ~
- Text:
content: " </div>"
span:
start: 800
length: 14
- Text:
content: " </body>"
span:
start: 815
length: 11
- Text:
content: "</html>"
span:
start: 827
length: 7
line_offsets:
- 0
- 16

View file

@ -1,9 +0,0 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 425
expression: ast
snapshot_kind: text
---
nodes: []
line_offsets:
- 0

View file

@ -1,9 +0,0 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 434
expression: ast
snapshot_kind: text
---
nodes: []
line_offsets:
- 0

View file

@ -1,9 +0,0 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 443
expression: ast
snapshot_kind: text
---
nodes: []
line_offsets:
- 0

View file

@ -1,10 +1,40 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 540
assertion_line: 534
expression: ast
snapshot_kind: text
---
nodes: []
nodes:
- Text:
content: "<script type=\"text/javascript\">"
span:
start: 0
length: 31
- Comment:
content: Single line comment
span:
start: 36
length: 21
- Text:
content: " const x = 1;"
span:
start: 59
length: 16
- Comment:
content: "Multi-line\n comment"
span:
start: 80
length: 30
- Text:
content: " console.log(x);"
span:
start: 113
length: 19
- Text:
content: "</script>"
span:
start: 133
length: 9
line_offsets:
- 0
- 32

View file

@ -1,10 +1,40 @@
---
source: crates/djls-template-ast/src/parser.rs
assertion_line: 557
assertion_line: 551
expression: ast
snapshot_kind: text
---
nodes: []
nodes:
- Text:
content: "<style type=\"text/css\">"
span:
start: 0
length: 23
- Comment:
content: Header styles
span:
start: 28
length: 17
- Text:
content: " .header {"
span:
start: 48
length: 13
- Text:
content: " color: blue;"
span:
start: 62
length: 20
- Text:
content: " }"
span:
start: 83
length: 5
- Text:
content: "</style>"
span:
start: 89
length: 8
line_offsets:
- 0
- 24

View file

@ -74,13 +74,11 @@ impl fmt::Display for TokenType {
DjangoBlock(s) => write!(f, "{{% {} %}}", s),
DjangoVariable(s) => write!(f, "{{{{ {} }}}}", s),
Eof => Ok(()),
HtmlTagOpen(s) => write!(f, "<{}>", s),
HtmlTagOpen(s) | ScriptTagOpen(s) | StyleTagOpen(s) => write!(f, "<{}>", s),
HtmlTagClose(s) => write!(f, "</{}>", s),
HtmlTagVoid(s) => write!(f, "<{}/>", s),
Newline => f.write_str("\n"),
ScriptTagOpen(s) => write!(f, "<script{}>", s),
ScriptTagClose(_) => f.write_str("</script>"),
StyleTagOpen(s) => write!(f, "<style{}>", s),
StyleTagClose(_) => f.write_str("</style>"),
Text(s) => f.write_str(s),
Whitespace(len) => f.write_str(&" ".repeat(*len)),