mirrors/django-components
1
0
Fork 0
mirror of https://github.com/django-components/django-components.git synced 2025-08-17 12:40:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: list and dict literals in tags + fix tag parser (#898)

Browse source 
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Juro Oravec 2025-01-09 21:15:23 +01:00 • committed by GitHub
parent 6c2a687d69
commit a79b24b692
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 2800 additions and 300 deletions
Download patch file Download diff file Expand all files Collapse all files

37
src/django_components/templatetags/component_tags.py
View file

@ -12,7 +12,7 @@
# During documentation generation, we access the `fn._tag_spec`.
import functools
from typing import Any, Callable, Dict, List, Literal, NamedTuple, Optional, Set, Tuple, Union
from typing import Any, Callable, Dict, List, Literal, NamedTuple, Optional, Set, Tuple, Union, cast
import django.template
from django.template.base import FilterExpression, NodeList, Parser, TextNode, Token, TokenType
@ -34,7 +34,6 @@ from django_components.expression import (
SpreadOperator,
is_aggregate_key,
is_dynamic_expression,
is_spread_operator,
)
from django_components.provide import PROVIDE_NAME_KWARG, ProvideNode
from django_components.slots import (
@ -49,7 +48,7 @@ from django_components.slots import (
from django_components.tag_formatter import get_tag_formatter
from django_components.util.logger import trace_msg
from django_components.util.misc import gen_id
from django_components.util.tag_parser import TagAttr, parse_tag_attrs
from django_components.util.tag_parser import TagAttr, TagValue, parse_tag
# NOTE: Variable name `register` is required by Django to recognize this as a template tag library
# See https://docs.djangoproject.com/en/dev/howto/custom-template-tags
@ -856,11 +855,11 @@ def _parse_tag_preprocess(
) -> Tuple[str, List[str], List[TagKwarg], Set[str], bool]:
_fix_nested_tags(parser, token)
_, attrs = parse_tag_attrs(token.contents)
_, attrs = parse_tag(token.contents)
# First token is tag name, e.g. `slot` in `{% slot <name> ... %}`
tag_name_attr = attrs.pop(0)
tag_name = tag_name_attr.formatted_value()
tag_name = tag_name_attr.serialize(omit_key=True)
# Sanity check
if tag_name != tag_spec.tag:
@ -872,7 +871,8 @@ def _parse_tag_preprocess(
# Otherwise, depending on the tag spec, the tag may be:
# 2. Block tag - With corresponding end tag, e.g. `{% endslot %}`
# 3. Inlined tag - Without the end tag.
last_token = attrs[-1].formatted_value() if len(attrs) else None
last_token = attrs[-1].serialize(omit_key=True) if len(attrs) else None
if last_token == "/":
attrs.pop()
is_inline = True
@ -901,10 +901,10 @@ def _parse_tag_input(tag_name: str, attrs: List[TagAttr]) -> Tuple[List[str], Li
flags = set()
seen_spreads = 0
for attr in attrs:
value = attr.formatted_value()
value = attr.serialize(omit_key=True)
# Spread
if is_spread_operator(value):
if attr.value.spread:
if value == "...":
raise TemplateSyntaxError("Syntax operator is missing a value")
@ -1161,20 +1161,27 @@ def _fix_nested_tags(parser: Parser, block_token: Token) -> None:
#
# We can parse the tag's tokens so we can find the last one, and so we consider
# the unclosed `{%` only for the last bit.
_, attrs = parse_tag_attrs(block_token.contents)
_, attrs = parse_tag(block_token.contents)
# If there are no attributes, then there are no nested tags
if not attrs:
return
last_attr = attrs[-1]
last_attr_part = last_attr.parts[-1]
last_token = last_attr_part.value
# TODO: Currently, using a nested template inside a list or dict
# e.g. `{% component ... key=["{% nested %}"] %}` is NOT supported.
# Hence why we leave if value is not "simple" (which means the value is list or dict).
if last_attr.value.type != "simple":
return
last_attr_value = cast(TagValue, last_attr.value.entries[0])
last_token = last_attr_value.parts[-1]
# User probably forgot to wrap the nested tag in quotes, or this is the end of the input.
# `{% component ... key={% nested %} %}`
# `{% component ... key= %}`
if not last_token:
if not last_token.value:
return
# When our template tag contains a nested tag, e.g.:
@ -1187,7 +1194,7 @@ def _fix_nested_tags(parser: Parser, block_token: Token) -> None:
# and includes `{%`. So that's what we use to identify if we need to fix
# nested tags or not.
has_unclosed_tag = (
(last_token.count("{%") > last_token.count("%}"))
(last_token.value.count("{%") > last_token.value.count("%}"))
# Moreover we need to also check for unclosed quotes for this edge case:
# `{% component 'test' "{%}" %}`
#
@ -1200,12 +1207,12 @@ def _fix_nested_tags(parser: Parser, block_token: Token) -> None:
# only within the last 'bit'. Consider this:
# `{% component 'test' '"' "{%}" %}`
#
or (last_token in ("'{", '"{'))
or (last_token.value in ("'{", '"{'))
)
# There is 3 double quotes, but if the contents get split at the first `%}`
# then there will be a single unclosed double quote in the last bit.
has_unclosed_quote = not last_attr_part.quoted and last_token and last_token[0] in ('"', "'")
has_unclosed_quote = not last_token.quoted and last_token.value and last_token.value[0] in ('"', "'")
needs_fixing = has_unclosed_tag and has_unclosed_quote

807
src/django_components/util/tag_parser.py
View file

@ -1,14 +1,102 @@
from dataclasses import dataclass
from typing import List, Optional, Sequence, Tuple, Union
"""
Parser for Django template tags.
TAG_WHITESPACES = (" ", "\t", "\n", "\r", "\f")
TAG_QUOTES = ("'", '"', '_("', "_('")
TAG_FILTER_JOINERS = ("|", ":")
TAG_SPREAD = "..."
The parser reads a tag like this (without the `{%` `%}`):
```django
{% component 'my_comp' key=val key2='val2 two' %}
```
and returns an AST representation of the tag:
```py
[
TagAttr(
key=None,
value=TagValueStruct(
type="simple",
spread=None,
meta={},
entries=[
TagValue(
parts=[
TagValuePart(
value="component", quoted=None, spread=None, translation=False, filter=None
),
]
),
],
),
start_index=0,
),
...
]
```
See `parse_tag()` for details.
"""
from dataclasses import dataclass
from typing import Any, Dict, List, Literal, NamedTuple, Optional, Sequence, Tuple, Union, cast
from django.template.exceptions import TemplateSyntaxError
TAG_WHITESPACE = (" ", "\t", "\n", "\r", "\f")
TAG_FILTER = ("|", ":")
TAG_SPREAD = ("*", "**", "...")
@dataclass
class TagAttrPart:
class TagAttr:
"""
A tag attribute represents a single token of a tag.
E.g. the following tag:
```django
{% component "my_comp" key=val key2='val2 two' %}
```
Has 4 attributes: `component`, `my_comp`, `key=val` and `key2='val2 two'`.
"""
key: Optional[str]
value: "TagValueStruct"
start_index: int
def serialize(self, omit_key: bool = False) -> str:
s = self.value.serialize()
if not omit_key and self.key:
return f"{self.key}={s}"
return s
class TagValue(NamedTuple):
"""
A tag value represents the text to the right of the `=` in a tag attribute.
E.g. in the following tag:
```django
{% component "my_comp" key=val2|filter1:"one" %}
```
The `key` attribute has the TagValue `val2|filter1:"one"`.
"""
parts: List["TagValuePart"]
@property
def is_spread(self) -> bool:
if not self.parts:
return False
return self.parts[0].spread is not None
def serialize(self) -> str:
return "".join(part.serialize() for part in self.parts)
@dataclass
class TagValuePart:
"""
Django tag attributes may consist of multiple parts, being separated by filter pipes (`|`)
or filter arguments (`:`). This class represents a single part of the attribute value.
@ -18,98 +106,204 @@ class TagAttrPart:
{% component "my_comp" key="my val's" key2=val2|filter1:"one" %}
```
The `key2` attribute has three parts: `val2`, `filter1` and `"one"`.
The value of attribute `key2` has three parts: `val2`, `filter1` and `"one"`.
"""
value: str
"""The actual value of the part, e.g. `val2` in `key2=val2` or `my string` in `_("my string")`."""
prefix: Optional[str]
"""
If this part is filter or filter arguent, `prefix` is the string that connects it to the previous part.
E.g. the `|` and `:` in `key2=val2|filter1:"one"`.
"""
"""The textual value"""
quoted: Optional[str]
"""Whether the value is quoted, and the character that's used for the quotation"""
spread: Optional[str]
"""
The prefix used by a spread syntax, e.g. `...`, `*`, or `**`. If present, it means
this values should be spread into the parent tag / list / dict.
"""
translation: bool
"""Whether the value is a translation string, e.g. `_("my string")`"""
filter: Optional[str]
"""The prefix of the filter, e.g. `|` or `:`"""
def __post_init__(self) -> None:
if self.translation and not self.quoted:
raise ValueError("Translation value must be quoted")
raise TemplateSyntaxError("Translation value must be quoted")
if self.translation and self.spread:
raise TemplateSyntaxError("Cannot combine translation and spread syntax")
if self.spread and self.filter:
raise TemplateSyntaxError("Cannot define spread syntax inside a filter")
if self.filter and self.filter not in TAG_FILTER:
raise TemplateSyntaxError(f"Invalid filter character: {self.filter}")
def formatted(self) -> str:
"""
Format the part as a string that can be used in a Django template tag.
E.g. `val2`, `|filter1:"one"`, `_("my string")`.
"""
def serialize(self) -> str:
value = f"{self.quoted}{self.value}{self.quoted}" if self.quoted else self.value
if self.translation:
value = f"_({value})"
if self.prefix:
value = f"{self.prefix}{value}"
elif self.spread:
value = f"{self.spread}{value}"
if self.filter:
value = f"{self.filter}{value}"
return value
# NOTE: dataclass is used so we can validate the input. But dataclasses are not hashable,
# by default, hence these methods.
def __hash__(self) -> int:
# Create a hash based on the attributes that define object equality
return hash((self.value, self.quoted, self.spread, self.translation, self.filter))
def __eq__(self, other: Any) -> bool:
if not isinstance(other, TagValuePart):
return False
return (
self.value == other.value
and self.quoted == other.quoted
and self.spread == other.spread
and self.translation == other.translation
and self.filter == other.filter
)
@dataclass
class TagAttr:
key: Optional[str]
parts: List[TagAttrPart]
start_index: int
class TagValueStruct:
"""
Start index of the attribute (include both key and value),
relative to the start of the owner Tag.
TagValueStruct represents a potential container (list or dict) that holds other tag values.
Types:
- `root`: Plain tag value
- `list`: A list of tag values
- `dict`: A dictionary of tag values
"""
spread: bool
"""Whether the value is a spread syntax, e.g. `...my_var`"""
def formatted_value(self) -> str:
value = "".join(part.formatted() for part in self.parts)
if self.spread:
value = f"{TAG_SPREAD}{value}"
return value
type: Literal["list", "dict", "simple"]
entries: List[Union["TagValueStruct", TagValue]]
spread: Optional[str]
"""
The prefix used by a spread syntax, e.g. `...`, `*`, or `**`. If present, it means
this values should be spread into the parent tag / list / dict.
"""
meta: Dict[str, Any]
def formatted(self) -> str:
s = self.formatted_value()
if self.key:
return f"{self.key}={s}"
return s
# Recursively walks down the value of TagAttr and serializes it to a string.
# This is effectively the inverse of `parse_tag()`.
def serialize(self) -> str:
def render_value(value: Union[TagValue, TagValueStruct]) -> str:
if isinstance(value, TagValue):
return value.serialize()
else:
return value.serialize()
if self.type == "simple":
value = self.entries[0]
return render_value(value)
elif self.type == "list":
prefix = self.spread or ""
return prefix + "[" + ", ".join([render_value(entry) for entry in self.entries]) + "]"
elif self.type == "dict":
prefix = self.spread or ""
dict_pairs = []
dict_pair: List[str] = []
# NOTE: Here we assume that the dict pairs have been validated by the parser and
# that the pairs line up.
for entry in self.entries:
rendered = render_value(entry)
if isinstance(entry, TagValueStruct):
if entry.spread:
if dict_pair:
raise TemplateSyntaxError("Malformed dict: spread operator cannot be used as a dict key")
dict_pairs.append(rendered)
else:
dict_pair.append(rendered)
else:
if entry.is_spread:
if dict_pair:
raise TemplateSyntaxError("Malformed dict: spread operator cannot be used as a dict key")
dict_pairs.append(rendered)
else:
dict_pair.append(rendered)
if len(dict_pair) == 2:
dict_pairs.append(": ".join(dict_pair))
dict_pair = []
return prefix + "{" + ", ".join(dict_pairs) + "}"
# Parse the content of a Django template tag like this:
#
# ```django
# {% component "my_comp" key="my val's" key2=val2 %}
# ```
#
# into a tag name and a list of attributes:
#
# ```python
# {
# "component": "component",
# }
# ```
def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]:
def parse_tag(text: str) -> Tuple[str, List[TagAttr]]:
"""
Parse the content of a Django template tag like this:
```django
{% component 'my_comp' key=val key2='val2 two' %}
```
into an AST representation:
[
TagAttr(
key=None,
start_index=0,
value=TagValue(
parts=tuple([
TagValuePart(value="component", quoted=None, spread=None, translation=False, filter=None)
])
),
),
TagAttr(
key=None,
start_index=10,
value=TagValue(
parts=tuple([
TagValuePart(value="my_comp", quoted="'", spread=None, translation=False, filter=None)
])
),
),
...
]
```
Supported syntax:
- Variables: `val`, `key`
- Kwargs (attributes): `key=val`, `key2='val2 two'`
- Quoted strings: `"my string"`, `'my string'`
- Translation: `_("my string")`
- Filters: `val|filter`, `val|filter:arg`
- List literals: `[value1, value2]`, `key=[value1, [1, 2, 3]]`
- Dict literals: `{"key1": value1, "key2": value2}`, `key={"key1": value1, "key2": {"nested": "value"}}`
- Trailing commas: `[1, 2, 3,]`, `{"key": "value", "key2": "value2",}`
- Spread operators: `...`, `*`, `**`
- Spread inside lists and dicts: `key=[1, *val, 3]`, `key={"key": val, **kwargs, "key2": 3}`
- Spread with list and dict literals: `{**{"key": val2}, "key": val1}`, `[ ...[val1], val2 ]`
- Spread list and dict literals as attributes: `{% ...[val1] %}`, `{% ...{"key" val1 } %}`
Invalid syntax:
- Spread inside a filter: `val|...filter`
- Spread inside a dictionary key: `attr={...attrs: "value"}`
- Spread inside a dictionary value: `attr={"key": ...val}`
- Misplaced spread: `attr=[...val]`, `attr={...val}`, `attr=[**val]`, `attr={*val}`
- Spreading lists and dicts: `...[1, 2, 3]`, `...{"key": "value"}`
"""
index = 0
normalized = ""
def add_token(token: Union[str, Tuple[str, ...]]) -> None:
def add_token(token: str) -> None:
nonlocal normalized
nonlocal index
text = "".join(token)
normalized += text
index += len(text)
normalized += token
index += len(token)
def is_next_token(*tokens: Union[str, Tuple[str, ...]]) -> bool:
def is_at_end(offset: int = 0) -> bool:
return index + offset >= len(text)
def is_next_token(tokens: Union[List[str], Tuple[str, ...]]) -> bool:
if not tokens:
raise ValueError("No tokens provided")
raise TemplateSyntaxError("No tokens provided")
def is_token_match(token: Union[str, Tuple[str, ...]]) -> bool:
def is_token_match(token: str) -> bool:
if not token:
raise ValueError("Empty token")
raise TemplateSyntaxError("Empty token")
for token_index, token_char in enumerate(token):
text_char = text[index + token_index] if index + token_index < len(text) else None
text_char = text[index + token_index] if not is_at_end(token_index) else None
if text_char is None or text_char != token_char:
return False
return True
@ -128,19 +322,19 @@ def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]:
# tag_name = take_until([" ", "\t", "\n", "\r", "\f", ">", "/>"])
def take_until(
tokens: Sequence[Union[str, Tuple[str, ...]]],
ignore: Optional[Sequence[Union[str, Tuple[str, ...]],]] = None,
tokens: Union[List[str], Tuple[str, ...]],
ignore: Optional[Sequence[str]] = None,
) -> str:
nonlocal index
nonlocal text
result = ""
while index < len(text):
while not is_at_end():
char = text[index]
ignore_token_match: Optional[Union[str, Tuple[str, ...]]] = None
ignore_token_match: Optional[str] = None
for ignore_token in ignore or []:
if is_next_token(ignore_token):
if is_next_token([ignore_token]):
ignore_token_match = ignore_token
if ignore_token_match:
@ -148,7 +342,7 @@ def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]:
add_token(ignore_token_match)
continue
if any(is_next_token(token) for token in tokens):
if is_next_token(tokens):
return result
result += char
@ -156,15 +350,15 @@ def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]:
return result
# tag_name = take_while([" ", "\t", "\n", "\r", "\f"])
def take_while(tokens: Sequence[Union[str, Tuple[str, ...]]]) -> str:
def take_while(tokens: Union[List[str], Tuple[str, ...]]) -> str:
nonlocal index
nonlocal text
result = ""
while index < len(text):
while not is_at_end():
char = text[index]
if any(is_next_token(token) for token in tokens):
if is_next_token(tokens):
result += char
add_token(char)
else:
@ -172,109 +366,426 @@ def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]:
return result
def parse_attr_parts() -> List[TagAttrPart]:
parts: List[TagAttrPart] = []
while index < len(text) and not is_next_token("=", *TAG_WHITESPACES):
is_translation = False
value: str = ""
quoted: Optional[str] = None
prefix: Optional[str] = None
if is_next_token(*TAG_FILTER_JOINERS):
prefix = taken_n(1) # | or :
# E.g. `height="20"` or `height=_("my_text")` or `height="my_text"|fil1:"one"`
if is_next_token(*TAG_QUOTES):
# NOTE: Strings may be wrapped in `_()` to allow for translation.
# See https://docs.djangoproject.com/en/5.1/topics/i18n/translation/#string-literals-passed-to-tags-and-filters # noqa: E501
if is_next_token("_("):
taken_n(2) # _(
is_translation = True
# NOTE: We assume no space between the translation syntax and the quote.
quote_char = taken_n(1) # " or '
# NOTE: Handle escaped quotes like \" or \', and continue until we reach the closing quote.
value = take_until([quote_char], ignore=["\\" + quote_char])
# Handle the case when there is a trailing quote, e.g. when a text value is not closed.
# `{% component 'my_comp' text="organis %}`
if is_next_token(quote_char):
add_token(quote_char)
if is_translation:
taken_n(1) # )
quoted = quote_char
else:
quoted = None
value = quote_char + value
# E.g. `height=20` or `height=my_var` or or `height=my_var|fil1:"one"`
def extract_spread_token(curr_struct: TagValueStruct, filter_token: Optional[str]) -> Optional[str]:
# Move the spread syntax out of the way, so that we properly handle what's next.
# Spread syntax MUST NOT be part of a filter, so that will raise if so.
#
# NOTE: To be consistent with Python API, the spread operator is marked with `*` or `**`
# inside lists and dicts:
# - `...` - Outside: `{% component ...spread %}`
# - `*` - Inside lists: `{% component key=[ *spread ] %}`
# - `**` - Inside dicts: `{% component key={ **spread } %}`
spread_token: Optional[str] = None
is_spread = is_next_token(TAG_SPREAD)
if is_spread:
if is_next_token(["..."]):
if curr_struct.type != "simple":
raise TemplateSyntaxError(
f"Spread syntax '...' found in {curr_struct.type}. It must be used on tag attributes only"
)
spread_token = "..."
elif is_next_token(["**"]):
if curr_struct.type != "dict":
raise TemplateSyntaxError("Spread syntax '**' found outside of a dictionary")
spread_token = "**"
elif is_next_token(["*"]):
if curr_struct.type != "list":
raise TemplateSyntaxError("Spread syntax '*' found outside of a list")
spread_token = "*"
else:
value = take_until(["=", *TAG_WHITESPACES, *TAG_FILTER_JOINERS])
quoted = None
raise TemplateSyntaxError("Invalid spread syntax")
parts.append(
TagAttrPart(
value=value,
prefix=prefix,
quoted=quoted,
translation=is_translation,
)
)
if spread_token is not None:
# Check for usage like `args|...filter`
if filter_token:
raise TemplateSyntaxError("Spread syntax cannot be used inside of a filter")
return parts
# Check for usage like `key=...attrs`
if curr_struct.type == "simple" and key is not None:
raise TemplateSyntaxError("Spread syntax '...' cannot follow a key ('key=...attrs')")
taken_n(len(cast(str, spread_token))) # ... or * or **
# Allow whitespace between spread and the variable, but only for the Python-like syntax
# (lists and dicts). E.g.:
# `{% component key=[ * spread ] %}` or `{% component key={ ** spread } %}`
#
# But not for the template tag syntax, because template tags rely on the whitespace
# to determine the end of the attribute value. E.g.:
# `{% component key=val ...spread key2=val2 %}`
if spread_token != "...":
take_while(TAG_WHITESPACE)
else:
if is_next_token(TAG_WHITESPACE) or is_at_end():
raise TemplateSyntaxError("Spread syntax '...' is missing a value")
return spread_token
# Parse attributes
attrs: List[TagAttr] = []
while index < len(text):
while not is_at_end():
# Skip whitespace
take_while(TAG_WHITESPACES)
take_while(TAG_WHITESPACE)
start_index = len(normalized)
key = None
# If token starts with a quote, we assume it's a value without key part.
# If token starts with any of these, we assume it's a value without key part.
# e.g. `component 'my_comp'`
# Otherwise, parse the key.
if is_next_token(*TAG_QUOTES, TAG_SPREAD):
# Otherwise, try to parse the key.
if is_next_token(["'", '"', '_("', "_('", "[", "{", *TAG_SPREAD]):
key = None
else:
parts = parse_attr_parts()
key = take_until(["=", "'", '"', '_("', "_('", "|", "[", "{", *TAG_SPREAD, *TAG_WHITESPACE])
# We've reached the end of the text
if not parts:
if not key and is_at_end():
break
# Has value
if is_next_token("="):
add_token("=")
key = "".join(part.formatted() for part in parts)
else:
# Actually was a value without key part
if not is_next_token(["="]):
# This was actually a value (variable) without the key part
index -= len(key)
normalized = normalized[:start_index]
key = None
attrs.append(
TagAttr(
key=key,
parts=parts,
start_index=start_index,
spread=False,
)
)
else:
add_token("=")
# NOTE: We put a fake root item, so we can modify the list in place.
# At the end, we'll unwrap the list to get the actual value.
total_value = TagValueStruct(type="simple", entries=[], spread=None, meta={})
stack = [total_value]
while len(stack) > 0:
take_while(TAG_WHITESPACE)
curr_value = stack[-1]
# Manage state with regards to lists and dictionaries
if is_next_token(["[", "...[", "*[", "**["]):
spread_token = extract_spread_token(curr_value, None)
if spread_token is not None:
if curr_value.type == "simple" and key is not None:
raise TemplateSyntaxError("Spread syntax '...' cannot follow a key ('key=...attrs')")
# NOTE: The `...`, `**`, `*` are "taken" in `extract_spread_token()`
taken_n(1) # [
struct = TagValueStruct(type="list", entries=[], spread=spread_token, meta={})
curr_value.entries.append(struct)
stack.append(struct)
continue
# Move the spread syntax out of the way, so that we properly handle what's next.
is_spread = is_next_token(TAG_SPREAD)
if is_spread:
taken_n(len(TAG_SPREAD)) # ...
elif is_next_token(["]"]):
if curr_value.type != "list":
raise TemplateSyntaxError("Unexpected closing bracket")
taken_n(1) # ]
stack.pop()
# Allow only 1 top-level list, similar to JSON
if stack[-1].type == "simple":
stack.pop()
continue
parts = parse_attr_parts()
elif is_next_token(["{", "...{", "*{", "**{"]):
spread_token = extract_spread_token(curr_value, None)
if spread_token is not None:
if curr_value.type == "simple" and key is not None:
raise TemplateSyntaxError("Spread syntax '...' cannot follow a key ('key=...attrs')")
# NOTE: The `...`, `**`, `*` are "taken" in `extract_spread_token()`
taken_n(1) # {
# Disallow nested structs on the position of a key
# E.g. `{ [val1, val2]: value }` or `{ {key: val}: value }`
# However, technically, we could allow this if the spread syntax is used.
# E.g. `{ ...{"key": val2} }`
if curr_value.type == "dict" and curr_value.meta["expects_key"]:
if spread_token:
curr_value.meta["expects_key"] = True
else:
raise TemplateSyntaxError("Dictionary cannot be used as a dictionary key")
struct = TagValueStruct(type="dict", entries=[], spread=spread_token, meta={})
curr_value.entries.append(struct)
struct.meta["expects_key"] = True
stack.append(struct)
continue
elif is_next_token(["}"]):
if curr_value.type != "dict":
raise TemplateSyntaxError("Unexpected closing bracket")
# Validate that the dicts contains only key-value pairs and spread entries
dict_pair: List[Union[TagValueStruct, TagValue]] = []
for entry in curr_value.entries:
# Dicts and lists can be used only as values, not as keys
if isinstance(entry, TagValueStruct):
if entry.spread:
# Case: `{ "key": **{"key2": val2} }`
if dict_pair:
raise TemplateSyntaxError(
"Spread syntax cannot be used in place of a dictionary value"
)
# Case: `{ **{"key": val2} }`
continue
else:
# Case: `{ {"key": val2}: value }`
if not dict_pair:
val_type = "Dictionary" if curr_value.type == "dict" else "List"
raise TemplateSyntaxError(f"{val_type} cannot be used as a dictionary key")
# Case: `{ "key": {"key2": val2} }`
else:
pass
dict_pair.append(entry)
if len(dict_pair) == 2:
dict_pair = []
else:
# Spread is fine when on its own, but cannot be used after a dict key
if entry.is_spread:
# Case: `{ "key": **my_attrs }`
if dict_pair:
raise TemplateSyntaxError(
"Spread syntax cannot be used in place of a dictionary value"
)
# Case: `{ **my_attrs }`
continue
# Non-spread value can be both key and value.
else:
# Cases: `{ my_attrs: "value" }` or `{ "key": my_attrs }`
dict_pair.append(entry)
if len(dict_pair) == 2:
dict_pair = []
# If, at the end, there an unmatched key-value pair, raise an error
if dict_pair:
raise TemplateSyntaxError("Dictionary key is missing a value")
del curr_value.meta["expects_key"]
taken_n(1) # }
stack.pop()
# Allow only 1 top-level dict, similar to JSON
if stack[-1].type == "simple":
stack.pop()
continue
elif is_next_token([","]):
if curr_value.type not in ("list", "dict"):
raise TemplateSyntaxError("Unexpected comma")
taken_n(1) # ,
if curr_value.type == "dict":
curr_value.meta["expects_key"] = True
continue
# NOTE: Altho `:` is used also in filter syntax, the "value" part
# that the filter is part of is parsed as a whole block. So if we got
# here, we know we're NOT in filter.
elif is_next_token([":"]):
if curr_value.type != "dict":
raise TemplateSyntaxError("Unexpected colon")
if not curr_value.meta["expects_key"]:
raise TemplateSyntaxError("Unexpected colon")
taken_n(1) # :
curr_value.meta["expects_key"] = False
continue
else:
# Allow only 1 top-level plain value, similar to JSON
if curr_value.type == "simple":
stack.pop()
else:
if is_at_end():
raise TemplateSyntaxError("Unexpected end of text")
# Once we got here, we know that next token is NOT a list nor dict.
# So we can now parse the value.
# Parse all filter parts of a value, e.g. `height="20" | yesno : "1,2,3" | lower`
# should be parsed as `"20" | yesno : "1,2,3" | lower`
values_parts: List[TagValuePart] = []
is_first_part = True
end_of_value = False
while not end_of_value:
is_translation = False
take_while(TAG_WHITESPACE)
if is_at_end():
if is_first_part:
raise TemplateSyntaxError("Unexpected end of text")
else:
end_of_value = True
continue
# In this case we've reached the end of a filter sequence
# e.g. image: `height="20"|lower key1=value1`
# and we're here: ^
# such that the next token already belongs to the next attribute.
if not is_first_part and not is_next_token(TAG_FILTER):
end_of_value = True
continue
# Catch cases like `|filter` or `:arg`, which should be `var|filter` or `filter:arg`
elif is_first_part and is_next_token(TAG_FILTER):
raise TemplateSyntaxError("Filter is missing a value")
# Get past the filter tokens like `|` or `:`, until the next value part.
# E.g. imagine: `height="20" | yesno : "1,2,3" | lower`
# and we're here: ^
# and we want to parse `yesno` next
if not is_first_part:
filter_token = taken_n(1) # | or :
take_while(TAG_WHITESPACE) # Allow whitespace after filter
else:
filter_token = None
is_first_part = False
# Move the spread syntax out of the way, so that we properly handle what's next.
# Spread syntax MUST NOT be part of a filter, so that will raise if so.
#
# NOTE: To be consistent with Python API, the spread operator is marked with `*` or `**`
# inside lists and dicts:
# - `...` - Outside: `{% component ...spread %}`
# - `*` - Inside lists: `{% component key=[ *spread ] %}`
# - `**` - Inside dicts: `{% component key={ **spread } %}`
spread_token = extract_spread_token(curr_value, filter_token)
# Handle top-level spread `{% component ...attrs %}`
if curr_value.type == "simple":
curr_value.spread = spread_token
# IMPORTANT!!! Depending on whether we're in a list or dict, there may be extra terminal tokens.
#
# E.g. in `[value | filter : argument, value2 | filter2 : argument2]`, the two values
# are separated by a comma, and terminated by `]`.
#
# And in `{key1: value1 | filter1 : argument1, key2: value2 | filter2 : argument2}`
# the two key-value pairs are separated by a comma, and terminated by `}`.
#
# But as you can see, the dictionary also uses `:` syntax to separate the key from value.
# This effectively means that when we're parsing a dictionary KEY, we're unable to tell
# if the next `:` is the key-value syntax versus filter argument syntax.
#
# THUS, to resolve this, in dictionary we don't allow KEY to have a filter argument syntax `:`.
# So if we see `:`, we end the key part of key-value syntax, and start parsing the value.
if curr_value.type == "dict":
if curr_value.meta["expects_key"]:
terminal_tokens: Tuple[str, ...] = (":", ",", "}")
else:
if spread_token:
raise TemplateSyntaxError("Spread syntax cannot be used in place of a dictionary value")
terminal_tokens = (",", "}")
elif curr_value.type == "list":
terminal_tokens = (",", "]")
else:
terminal_tokens = tuple()
# Parse the value
#
# E.g. imagine: `height="20" | yesno : "1,2,3" | lower`
# and we're here: ^
# or here: ^
# or here: ^
# or here: ^
if is_next_token(["'", '"', "_("]):
# NOTE: Strings may be wrapped in `_()` to allow for translation.
# See https://docs.djangoproject.com/en/5.1/topics/i18n/translation/#string-literals-passed-to-tags-and-filters # noqa: E501
# NOTE 2: We could potentially raise if this token is supposed to be a filter
# name (after `|`) and we got a translation or a quoted string instead. But we
# leave that up for Django.
if is_next_token(["_("]):
taken_n(2) # _(
# There may be whitespace between the translation syntax and the quote.
# E.g. `_("20")` vs `_( "20" )`
take_while(TAG_WHITESPACE) # Allow whitespace after translation
is_translation = True
quote_char = taken_n(1) # " or '
# NOTE: Handle escaped quotes like \" or \', and continue until we reach the closing quote.
value = take_until([quote_char], ignore=["\\" + quote_char])
if is_next_token([quote_char]):
add_token(quote_char)
if is_translation:
# There may be whitespace between the translation syntax and the quote.
# E.g. `_("20")` vs `_( "20" )`
take_while(TAG_WHITESPACE) # Allow whitespace after translation
taken_n(1) # )
quoted = quote_char
# Handle the case when there is a trailing quote, e.g. when a text value is not closed.
# `{% component 'my_comp' text="organis %}`
else:
quoted = None
value = quote_char + value
# E.g. the `20` or `lower` of `height=20|lower`
# Since this is not a string, we know that it CANNOT contain whitespace.
#
# NOTE: This branch is also taken by terminal tokens like `]` or `}`.
else:
quoted = None
value = take_until(TAG_WHITESPACE + TAG_FILTER + terminal_tokens)
take_while(TAG_WHITESPACE)
if terminal_tokens and is_next_token(terminal_tokens):
end_of_value = True
values_parts.append(
TagValuePart(
value=value,
quoted=quoted,
spread=spread_token,
translation=is_translation,
filter=filter_token,
)
)
# Here we're done with the value (+ a sequence of filters)
# E.g. `height="20" | yesno : "1,2,3" | lower`
# we're here: ^
#
# This whole sequence could be part of a list or a dict,
# E.g. `[my_comp, 'height="20" | yesno : "1,2,3" | lower']`
#
# So we add it to the parent struct
curr_value.entries.append(TagValue(parts=values_parts))
if curr_value.type == "dict":
if values_parts[0].spread:
# Validation for `{"key": **spread }`
if not curr_value.meta["expects_key"]:
raise TemplateSyntaxError(
"Got spread syntax on the position of a value inside a dictionary key-value pair"
)
# Validation for `{**spread: value }`
take_while(TAG_WHITESPACE)
if is_next_token([":"]):
raise TemplateSyntaxError("Spread syntax cannot be used in place of a dictionary key")
else:
# Validation for `{"key", value }`
if curr_value.meta["expects_key"]:
take_while(TAG_WHITESPACE)
if not is_next_token([":"]):
raise TemplateSyntaxError("Dictionary key is missing a value")
# And at this point, we have the full representation of the tag value,
# including any lists or dictionaries (even nested). E.g.
# ```py
# TagValueStruct(type="simple", entries=[
# TagValueStruct(type="list", entries=[
# TagValuePart(value="my_comp", quoted=None, spread=False, translation=False, filter=None),
# TagValuePart(value="'height=\"20\" | yesno : \"1,2,3\" | lower'", quoted="'", spread=False, translation=False, filter=None), # noqa: E501
# TagValueStruct(type="dict", entries=[
# TagValuePart(value="key1", quoted=None, spread=False, translation=False, filter=None),
# TagValuePart(value="value1|filter2 : \"1,2,3\"", quoted="'", spread=False, translation=False, filter=None), # noqa: E501
# ]),
# ]),
# ])
# ```
# Unwrap top-level list / dict
if isinstance(total_value.entries[0], TagValueStruct) and total_value.entries[0].type != "simple":
total_value = total_value.entries[0]
attrs.append(
TagAttr(
key=key,
parts=parts,
start_index=start_index,
spread=is_spread,
value=total_value,
)
)

2
tests/test_expression.py
View file

@ -827,7 +827,7 @@ class SpreadOperatorTests(BaseTestCase):
)
)
with self.assertRaisesMessage(TemplateSyntaxError, "Syntax operator is missing a value"):
with self.assertRaisesMessage(TemplateSyntaxError, "Spread syntax '...' is missing a value"):
Template(template_str)
@parametrize_context_behavior(["django", "isolated"])

2254
tests/test_tag_parser.py
View file

File diff suppressed because it is too large Load diff