From a79b24b692eb3a2f7ab814d1b7d6e922ec17a8a5 Mon Sep 17 00:00:00 2001 From: Juro Oravec Date: Thu, 9 Jan 2025 21:15:23 +0100 Subject: [PATCH] feat: list and dict literals in tags + fix tag parser (#898) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../templatetags/component_tags.py | 37 +- src/django_components/util/tag_parser.py | 807 ++++-- tests/test_expression.py | 2 +- tests/test_tag_parser.py | 2254 ++++++++++++++++- 4 files changed, 2800 insertions(+), 300 deletions(-) diff --git a/src/django_components/templatetags/component_tags.py b/src/django_components/templatetags/component_tags.py index e9aeb73d..121d7bb1 100644 --- a/src/django_components/templatetags/component_tags.py +++ b/src/django_components/templatetags/component_tags.py @@ -12,7 +12,7 @@ # During documentation generation, we access the `fn._tag_spec`. import functools -from typing import Any, Callable, Dict, List, Literal, NamedTuple, Optional, Set, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, NamedTuple, Optional, Set, Tuple, Union, cast import django.template from django.template.base import FilterExpression, NodeList, Parser, TextNode, Token, TokenType @@ -34,7 +34,6 @@ from django_components.expression import ( SpreadOperator, is_aggregate_key, is_dynamic_expression, - is_spread_operator, ) from django_components.provide import PROVIDE_NAME_KWARG, ProvideNode from django_components.slots import ( @@ -49,7 +48,7 @@ from django_components.slots import ( from django_components.tag_formatter import get_tag_formatter from django_components.util.logger import trace_msg from django_components.util.misc import gen_id -from django_components.util.tag_parser import TagAttr, parse_tag_attrs +from django_components.util.tag_parser import TagAttr, TagValue, parse_tag # NOTE: Variable name `register` is required by Django to recognize this as a template tag library # See https://docs.djangoproject.com/en/dev/howto/custom-template-tags @@ -856,11 +855,11 @@ def _parse_tag_preprocess( ) -> Tuple[str, List[str], List[TagKwarg], Set[str], bool]: _fix_nested_tags(parser, token) - _, attrs = parse_tag_attrs(token.contents) + _, attrs = parse_tag(token.contents) # First token is tag name, e.g. `slot` in `{% slot ... %}` tag_name_attr = attrs.pop(0) - tag_name = tag_name_attr.formatted_value() + tag_name = tag_name_attr.serialize(omit_key=True) # Sanity check if tag_name != tag_spec.tag: @@ -872,7 +871,8 @@ def _parse_tag_preprocess( # Otherwise, depending on the tag spec, the tag may be: # 2. Block tag - With corresponding end tag, e.g. `{% endslot %}` # 3. Inlined tag - Without the end tag. - last_token = attrs[-1].formatted_value() if len(attrs) else None + last_token = attrs[-1].serialize(omit_key=True) if len(attrs) else None + if last_token == "/": attrs.pop() is_inline = True @@ -901,10 +901,10 @@ def _parse_tag_input(tag_name: str, attrs: List[TagAttr]) -> Tuple[List[str], Li flags = set() seen_spreads = 0 for attr in attrs: - value = attr.formatted_value() + value = attr.serialize(omit_key=True) # Spread - if is_spread_operator(value): + if attr.value.spread: if value == "...": raise TemplateSyntaxError("Syntax operator is missing a value") @@ -1161,20 +1161,27 @@ def _fix_nested_tags(parser: Parser, block_token: Token) -> None: # # We can parse the tag's tokens so we can find the last one, and so we consider # the unclosed `{%` only for the last bit. - _, attrs = parse_tag_attrs(block_token.contents) + _, attrs = parse_tag(block_token.contents) # If there are no attributes, then there are no nested tags if not attrs: return last_attr = attrs[-1] - last_attr_part = last_attr.parts[-1] - last_token = last_attr_part.value + + # TODO: Currently, using a nested template inside a list or dict + # e.g. `{% component ... key=["{% nested %}"] %}` is NOT supported. + # Hence why we leave if value is not "simple" (which means the value is list or dict). + if last_attr.value.type != "simple": + return + + last_attr_value = cast(TagValue, last_attr.value.entries[0]) + last_token = last_attr_value.parts[-1] # User probably forgot to wrap the nested tag in quotes, or this is the end of the input. # `{% component ... key={% nested %} %}` # `{% component ... key= %}` - if not last_token: + if not last_token.value: return # When our template tag contains a nested tag, e.g.: @@ -1187,7 +1194,7 @@ def _fix_nested_tags(parser: Parser, block_token: Token) -> None: # and includes `{%`. So that's what we use to identify if we need to fix # nested tags or not. has_unclosed_tag = ( - (last_token.count("{%") > last_token.count("%}")) + (last_token.value.count("{%") > last_token.value.count("%}")) # Moreover we need to also check for unclosed quotes for this edge case: # `{% component 'test' "{%}" %}` # @@ -1200,12 +1207,12 @@ def _fix_nested_tags(parser: Parser, block_token: Token) -> None: # only within the last 'bit'. Consider this: # `{% component 'test' '"' "{%}" %}` # - or (last_token in ("'{", '"{')) + or (last_token.value in ("'{", '"{')) ) # There is 3 double quotes, but if the contents get split at the first `%}` # then there will be a single unclosed double quote in the last bit. - has_unclosed_quote = not last_attr_part.quoted and last_token and last_token[0] in ('"', "'") + has_unclosed_quote = not last_token.quoted and last_token.value and last_token.value[0] in ('"', "'") needs_fixing = has_unclosed_tag and has_unclosed_quote diff --git a/src/django_components/util/tag_parser.py b/src/django_components/util/tag_parser.py index abdc6ac4..df206e15 100644 --- a/src/django_components/util/tag_parser.py +++ b/src/django_components/util/tag_parser.py @@ -1,14 +1,102 @@ -from dataclasses import dataclass -from typing import List, Optional, Sequence, Tuple, Union +""" +Parser for Django template tags. -TAG_WHITESPACES = (" ", "\t", "\n", "\r", "\f") -TAG_QUOTES = ("'", '"', '_("', "_('") -TAG_FILTER_JOINERS = ("|", ":") -TAG_SPREAD = "..." +The parser reads a tag like this (without the `{%` `%}`): + +```django +{% component 'my_comp' key=val key2='val2 two' %} +``` + +and returns an AST representation of the tag: + +```py +[ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ), + ] + ), + ], + ), + start_index=0, + ), + ... +] +``` + +See `parse_tag()` for details. +""" + +from dataclasses import dataclass +from typing import Any, Dict, List, Literal, NamedTuple, Optional, Sequence, Tuple, Union, cast + +from django.template.exceptions import TemplateSyntaxError + +TAG_WHITESPACE = (" ", "\t", "\n", "\r", "\f") +TAG_FILTER = ("|", ":") +TAG_SPREAD = ("*", "**", "...") @dataclass -class TagAttrPart: +class TagAttr: + """ + A tag attribute represents a single token of a tag. + + E.g. the following tag: + + ```django + {% component "my_comp" key=val key2='val2 two' %} + ``` + + Has 4 attributes: `component`, `my_comp`, `key=val` and `key2='val2 two'`. + """ + + key: Optional[str] + value: "TagValueStruct" + start_index: int + + def serialize(self, omit_key: bool = False) -> str: + s = self.value.serialize() + if not omit_key and self.key: + return f"{self.key}={s}" + return s + + +class TagValue(NamedTuple): + """ + A tag value represents the text to the right of the `=` in a tag attribute. + + E.g. in the following tag: + ```django + {% component "my_comp" key=val2|filter1:"one" %} + ``` + + The `key` attribute has the TagValue `val2|filter1:"one"`. + """ + + parts: List["TagValuePart"] + + @property + def is_spread(self) -> bool: + if not self.parts: + return False + return self.parts[0].spread is not None + + def serialize(self) -> str: + return "".join(part.serialize() for part in self.parts) + + +@dataclass +class TagValuePart: """ Django tag attributes may consist of multiple parts, being separated by filter pipes (`|`) or filter arguments (`:`). This class represents a single part of the attribute value. @@ -18,98 +106,204 @@ class TagAttrPart: {% component "my_comp" key="my val's" key2=val2|filter1:"one" %} ``` - The `key2` attribute has three parts: `val2`, `filter1` and `"one"`. + The value of attribute `key2` has three parts: `val2`, `filter1` and `"one"`. """ value: str - """The actual value of the part, e.g. `val2` in `key2=val2` or `my string` in `_("my string")`.""" - prefix: Optional[str] - """ - If this part is filter or filter arguent, `prefix` is the string that connects it to the previous part. - E.g. the `|` and `:` in `key2=val2|filter1:"one"`. - """ + """The textual value""" quoted: Optional[str] """Whether the value is quoted, and the character that's used for the quotation""" + spread: Optional[str] + """ + The prefix used by a spread syntax, e.g. `...`, `*`, or `**`. If present, it means + this values should be spread into the parent tag / list / dict. + """ translation: bool """Whether the value is a translation string, e.g. `_("my string")`""" + filter: Optional[str] + """The prefix of the filter, e.g. `|` or `:`""" def __post_init__(self) -> None: if self.translation and not self.quoted: - raise ValueError("Translation value must be quoted") + raise TemplateSyntaxError("Translation value must be quoted") + if self.translation and self.spread: + raise TemplateSyntaxError("Cannot combine translation and spread syntax") + if self.spread and self.filter: + raise TemplateSyntaxError("Cannot define spread syntax inside a filter") + if self.filter and self.filter not in TAG_FILTER: + raise TemplateSyntaxError(f"Invalid filter character: {self.filter}") - def formatted(self) -> str: - """ - Format the part as a string that can be used in a Django template tag. - E.g. `val2`, `|filter1:"one"`, `_("my string")`. - """ + def serialize(self) -> str: value = f"{self.quoted}{self.value}{self.quoted}" if self.quoted else self.value if self.translation: value = f"_({value})" - if self.prefix: - value = f"{self.prefix}{value}" + elif self.spread: + value = f"{self.spread}{value}" + + if self.filter: + value = f"{self.filter}{value}" + return value + # NOTE: dataclass is used so we can validate the input. But dataclasses are not hashable, + # by default, hence these methods. + def __hash__(self) -> int: + # Create a hash based on the attributes that define object equality + return hash((self.value, self.quoted, self.spread, self.translation, self.filter)) + + def __eq__(self, other: Any) -> bool: + if not isinstance(other, TagValuePart): + return False + return ( + self.value == other.value + and self.quoted == other.quoted + and self.spread == other.spread + and self.translation == other.translation + and self.filter == other.filter + ) + @dataclass -class TagAttr: - key: Optional[str] - parts: List[TagAttrPart] - start_index: int +class TagValueStruct: """ - Start index of the attribute (include both key and value), - relative to the start of the owner Tag. + TagValueStruct represents a potential container (list or dict) that holds other tag values. + + Types: + + - `root`: Plain tag value + - `list`: A list of tag values + - `dict`: A dictionary of tag values """ - spread: bool - """Whether the value is a spread syntax, e.g. `...my_var`""" - def formatted_value(self) -> str: - value = "".join(part.formatted() for part in self.parts) - if self.spread: - value = f"{TAG_SPREAD}{value}" - return value + type: Literal["list", "dict", "simple"] + entries: List[Union["TagValueStruct", TagValue]] + spread: Optional[str] + """ + The prefix used by a spread syntax, e.g. `...`, `*`, or `**`. If present, it means + this values should be spread into the parent tag / list / dict. + """ + meta: Dict[str, Any] - def formatted(self) -> str: - s = self.formatted_value() - if self.key: - return f"{self.key}={s}" - return s + # Recursively walks down the value of TagAttr and serializes it to a string. + # This is effectively the inverse of `parse_tag()`. + def serialize(self) -> str: + def render_value(value: Union[TagValue, TagValueStruct]) -> str: + if isinstance(value, TagValue): + return value.serialize() + else: + return value.serialize() + + if self.type == "simple": + value = self.entries[0] + return render_value(value) + elif self.type == "list": + prefix = self.spread or "" + return prefix + "[" + ", ".join([render_value(entry) for entry in self.entries]) + "]" + elif self.type == "dict": + prefix = self.spread or "" + dict_pairs = [] + dict_pair: List[str] = [] + # NOTE: Here we assume that the dict pairs have been validated by the parser and + # that the pairs line up. + for entry in self.entries: + rendered = render_value(entry) + if isinstance(entry, TagValueStruct): + if entry.spread: + if dict_pair: + raise TemplateSyntaxError("Malformed dict: spread operator cannot be used as a dict key") + dict_pairs.append(rendered) + else: + dict_pair.append(rendered) + else: + if entry.is_spread: + if dict_pair: + raise TemplateSyntaxError("Malformed dict: spread operator cannot be used as a dict key") + dict_pairs.append(rendered) + else: + dict_pair.append(rendered) + if len(dict_pair) == 2: + dict_pairs.append(": ".join(dict_pair)) + dict_pair = [] + return prefix + "{" + ", ".join(dict_pairs) + "}" -# Parse the content of a Django template tag like this: -# -# ```django -# {% component "my_comp" key="my val's" key2=val2 %} -# ``` -# -# into a tag name and a list of attributes: -# -# ```python -# { -# "component": "component", -# } -# ``` -def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]: +def parse_tag(text: str) -> Tuple[str, List[TagAttr]]: + """ + Parse the content of a Django template tag like this: + + ```django + {% component 'my_comp' key=val key2='val2 two' %} + ``` + + into an AST representation: + + [ + TagAttr( + key=None, + start_index=0, + value=TagValue( + parts=tuple([ + TagValuePart(value="component", quoted=None, spread=None, translation=False, filter=None) + ]) + ), + ), + TagAttr( + key=None, + start_index=10, + value=TagValue( + parts=tuple([ + TagValuePart(value="my_comp", quoted="'", spread=None, translation=False, filter=None) + ]) + ), + ), + ... + ] + ``` + + Supported syntax: + - Variables: `val`, `key` + - Kwargs (attributes): `key=val`, `key2='val2 two'` + - Quoted strings: `"my string"`, `'my string'` + - Translation: `_("my string")` + - Filters: `val|filter`, `val|filter:arg` + - List literals: `[value1, value2]`, `key=[value1, [1, 2, 3]]` + - Dict literals: `{"key1": value1, "key2": value2}`, `key={"key1": value1, "key2": {"nested": "value"}}` + - Trailing commas: `[1, 2, 3,]`, `{"key": "value", "key2": "value2",}` + - Spread operators: `...`, `*`, `**` + - Spread inside lists and dicts: `key=[1, *val, 3]`, `key={"key": val, **kwargs, "key2": 3}` + - Spread with list and dict literals: `{**{"key": val2}, "key": val1}`, `[ ...[val1], val2 ]` + - Spread list and dict literals as attributes: `{% ...[val1] %}`, `{% ...{"key" val1 } %}` + + Invalid syntax: + - Spread inside a filter: `val|...filter` + - Spread inside a dictionary key: `attr={...attrs: "value"}` + - Spread inside a dictionary value: `attr={"key": ...val}` + - Misplaced spread: `attr=[...val]`, `attr={...val}`, `attr=[**val]`, `attr={*val}` + - Spreading lists and dicts: `...[1, 2, 3]`, `...{"key": "value"}` + """ index = 0 normalized = "" - def add_token(token: Union[str, Tuple[str, ...]]) -> None: + def add_token(token: str) -> None: nonlocal normalized nonlocal index - text = "".join(token) - normalized += text - index += len(text) + normalized += token + index += len(token) - def is_next_token(*tokens: Union[str, Tuple[str, ...]]) -> bool: + def is_at_end(offset: int = 0) -> bool: + return index + offset >= len(text) + + def is_next_token(tokens: Union[List[str], Tuple[str, ...]]) -> bool: if not tokens: - raise ValueError("No tokens provided") + raise TemplateSyntaxError("No tokens provided") - def is_token_match(token: Union[str, Tuple[str, ...]]) -> bool: + def is_token_match(token: str) -> bool: if not token: - raise ValueError("Empty token") + raise TemplateSyntaxError("Empty token") for token_index, token_char in enumerate(token): - text_char = text[index + token_index] if index + token_index < len(text) else None + text_char = text[index + token_index] if not is_at_end(token_index) else None if text_char is None or text_char != token_char: return False return True @@ -128,19 +322,19 @@ def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]: # tag_name = take_until([" ", "\t", "\n", "\r", "\f", ">", "/>"]) def take_until( - tokens: Sequence[Union[str, Tuple[str, ...]]], - ignore: Optional[Sequence[Union[str, Tuple[str, ...]],]] = None, + tokens: Union[List[str], Tuple[str, ...]], + ignore: Optional[Sequence[str]] = None, ) -> str: nonlocal index nonlocal text result = "" - while index < len(text): + while not is_at_end(): char = text[index] - ignore_token_match: Optional[Union[str, Tuple[str, ...]]] = None + ignore_token_match: Optional[str] = None for ignore_token in ignore or []: - if is_next_token(ignore_token): + if is_next_token([ignore_token]): ignore_token_match = ignore_token if ignore_token_match: @@ -148,7 +342,7 @@ def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]: add_token(ignore_token_match) continue - if any(is_next_token(token) for token in tokens): + if is_next_token(tokens): return result result += char @@ -156,15 +350,15 @@ def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]: return result # tag_name = take_while([" ", "\t", "\n", "\r", "\f"]) - def take_while(tokens: Sequence[Union[str, Tuple[str, ...]]]) -> str: + def take_while(tokens: Union[List[str], Tuple[str, ...]]) -> str: nonlocal index nonlocal text result = "" - while index < len(text): + while not is_at_end(): char = text[index] - if any(is_next_token(token) for token in tokens): + if is_next_token(tokens): result += char add_token(char) else: @@ -172,109 +366,426 @@ def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]: return result - def parse_attr_parts() -> List[TagAttrPart]: - parts: List[TagAttrPart] = [] - - while index < len(text) and not is_next_token("=", *TAG_WHITESPACES): - is_translation = False - value: str = "" - quoted: Optional[str] = None - prefix: Optional[str] = None - - if is_next_token(*TAG_FILTER_JOINERS): - prefix = taken_n(1) # | or : - - # E.g. `height="20"` or `height=_("my_text")` or `height="my_text"|fil1:"one"` - if is_next_token(*TAG_QUOTES): - # NOTE: Strings may be wrapped in `_()` to allow for translation. - # See https://docs.djangoproject.com/en/5.1/topics/i18n/translation/#string-literals-passed-to-tags-and-filters # noqa: E501 - if is_next_token("_("): - taken_n(2) # _( - is_translation = True - - # NOTE: We assume no space between the translation syntax and the quote. - quote_char = taken_n(1) # " or ' - - # NOTE: Handle escaped quotes like \" or \', and continue until we reach the closing quote. - value = take_until([quote_char], ignore=["\\" + quote_char]) - # Handle the case when there is a trailing quote, e.g. when a text value is not closed. - # `{% component 'my_comp' text="organis %}` - - if is_next_token(quote_char): - add_token(quote_char) - if is_translation: - taken_n(1) # ) - quoted = quote_char - else: - quoted = None - value = quote_char + value - # E.g. `height=20` or `height=my_var` or or `height=my_var|fil1:"one"` + def extract_spread_token(curr_struct: TagValueStruct, filter_token: Optional[str]) -> Optional[str]: + # Move the spread syntax out of the way, so that we properly handle what's next. + # Spread syntax MUST NOT be part of a filter, so that will raise if so. + # + # NOTE: To be consistent with Python API, the spread operator is marked with `*` or `**` + # inside lists and dicts: + # - `...` - Outside: `{% component ...spread %}` + # - `*` - Inside lists: `{% component key=[ *spread ] %}` + # - `**` - Inside dicts: `{% component key={ **spread } %}` + spread_token: Optional[str] = None + is_spread = is_next_token(TAG_SPREAD) + if is_spread: + if is_next_token(["..."]): + if curr_struct.type != "simple": + raise TemplateSyntaxError( + f"Spread syntax '...' found in {curr_struct.type}. It must be used on tag attributes only" + ) + spread_token = "..." + elif is_next_token(["**"]): + if curr_struct.type != "dict": + raise TemplateSyntaxError("Spread syntax '**' found outside of a dictionary") + spread_token = "**" + elif is_next_token(["*"]): + if curr_struct.type != "list": + raise TemplateSyntaxError("Spread syntax '*' found outside of a list") + spread_token = "*" else: - value = take_until(["=", *TAG_WHITESPACES, *TAG_FILTER_JOINERS]) - quoted = None + raise TemplateSyntaxError("Invalid spread syntax") - parts.append( - TagAttrPart( - value=value, - prefix=prefix, - quoted=quoted, - translation=is_translation, - ) - ) + if spread_token is not None: + # Check for usage like `args|...filter` + if filter_token: + raise TemplateSyntaxError("Spread syntax cannot be used inside of a filter") - return parts + # Check for usage like `key=...attrs` + if curr_struct.type == "simple" and key is not None: + raise TemplateSyntaxError("Spread syntax '...' cannot follow a key ('key=...attrs')") + + taken_n(len(cast(str, spread_token))) # ... or * or ** + # Allow whitespace between spread and the variable, but only for the Python-like syntax + # (lists and dicts). E.g.: + # `{% component key=[ * spread ] %}` or `{% component key={ ** spread } %}` + # + # But not for the template tag syntax, because template tags rely on the whitespace + # to determine the end of the attribute value. E.g.: + # `{% component key=val ...spread key2=val2 %}` + if spread_token != "...": + take_while(TAG_WHITESPACE) + else: + if is_next_token(TAG_WHITESPACE) or is_at_end(): + raise TemplateSyntaxError("Spread syntax '...' is missing a value") + return spread_token # Parse attributes attrs: List[TagAttr] = [] - while index < len(text): + while not is_at_end(): # Skip whitespace - take_while(TAG_WHITESPACES) + take_while(TAG_WHITESPACE) start_index = len(normalized) key = None - # If token starts with a quote, we assume it's a value without key part. + # If token starts with any of these, we assume it's a value without key part. # e.g. `component 'my_comp'` - # Otherwise, parse the key. - if is_next_token(*TAG_QUOTES, TAG_SPREAD): + # Otherwise, try to parse the key. + if is_next_token(["'", '"', '_("', "_('", "[", "{", *TAG_SPREAD]): key = None else: - parts = parse_attr_parts() + key = take_until(["=", "'", '"', '_("', "_('", "|", "[", "{", *TAG_SPREAD, *TAG_WHITESPACE]) # We've reached the end of the text - if not parts: + if not key and is_at_end(): break - # Has value - if is_next_token("="): - add_token("=") - key = "".join(part.formatted() for part in parts) - else: - # Actually was a value without key part + if not is_next_token(["="]): + # This was actually a value (variable) without the key part + index -= len(key) + normalized = normalized[:start_index] key = None - attrs.append( - TagAttr( - key=key, - parts=parts, - start_index=start_index, - spread=False, - ) - ) + else: + add_token("=") + + # NOTE: We put a fake root item, so we can modify the list in place. + # At the end, we'll unwrap the list to get the actual value. + total_value = TagValueStruct(type="simple", entries=[], spread=None, meta={}) + stack = [total_value] + + while len(stack) > 0: + take_while(TAG_WHITESPACE) + + curr_value = stack[-1] + + # Manage state with regards to lists and dictionaries + if is_next_token(["[", "...[", "*[", "**["]): + spread_token = extract_spread_token(curr_value, None) + if spread_token is not None: + if curr_value.type == "simple" and key is not None: + raise TemplateSyntaxError("Spread syntax '...' cannot follow a key ('key=...attrs')") + # NOTE: The `...`, `**`, `*` are "taken" in `extract_spread_token()` + taken_n(1) # [ + struct = TagValueStruct(type="list", entries=[], spread=spread_token, meta={}) + curr_value.entries.append(struct) + stack.append(struct) continue - # Move the spread syntax out of the way, so that we properly handle what's next. - is_spread = is_next_token(TAG_SPREAD) - if is_spread: - taken_n(len(TAG_SPREAD)) # ... + elif is_next_token(["]"]): + if curr_value.type != "list": + raise TemplateSyntaxError("Unexpected closing bracket") + taken_n(1) # ] + stack.pop() + # Allow only 1 top-level list, similar to JSON + if stack[-1].type == "simple": + stack.pop() + continue - parts = parse_attr_parts() + elif is_next_token(["{", "...{", "*{", "**{"]): + spread_token = extract_spread_token(curr_value, None) + if spread_token is not None: + if curr_value.type == "simple" and key is not None: + raise TemplateSyntaxError("Spread syntax '...' cannot follow a key ('key=...attrs')") + # NOTE: The `...`, `**`, `*` are "taken" in `extract_spread_token()` + taken_n(1) # { + + # Disallow nested structs on the position of a key + # E.g. `{ [val1, val2]: value }` or `{ {key: val}: value }` + # However, technically, we could allow this if the spread syntax is used. + # E.g. `{ ...{"key": val2} }` + if curr_value.type == "dict" and curr_value.meta["expects_key"]: + if spread_token: + curr_value.meta["expects_key"] = True + else: + raise TemplateSyntaxError("Dictionary cannot be used as a dictionary key") + + struct = TagValueStruct(type="dict", entries=[], spread=spread_token, meta={}) + curr_value.entries.append(struct) + struct.meta["expects_key"] = True + stack.append(struct) + continue + + elif is_next_token(["}"]): + if curr_value.type != "dict": + raise TemplateSyntaxError("Unexpected closing bracket") + + # Validate that the dicts contains only key-value pairs and spread entries + dict_pair: List[Union[TagValueStruct, TagValue]] = [] + for entry in curr_value.entries: + # Dicts and lists can be used only as values, not as keys + if isinstance(entry, TagValueStruct): + if entry.spread: + # Case: `{ "key": **{"key2": val2} }` + if dict_pair: + raise TemplateSyntaxError( + "Spread syntax cannot be used in place of a dictionary value" + ) + # Case: `{ **{"key": val2} }` + continue + else: + # Case: `{ {"key": val2}: value }` + if not dict_pair: + val_type = "Dictionary" if curr_value.type == "dict" else "List" + raise TemplateSyntaxError(f"{val_type} cannot be used as a dictionary key") + # Case: `{ "key": {"key2": val2} }` + else: + pass + dict_pair.append(entry) + if len(dict_pair) == 2: + dict_pair = [] + else: + # Spread is fine when on its own, but cannot be used after a dict key + if entry.is_spread: + # Case: `{ "key": **my_attrs }` + if dict_pair: + raise TemplateSyntaxError( + "Spread syntax cannot be used in place of a dictionary value" + ) + # Case: `{ **my_attrs }` + continue + # Non-spread value can be both key and value. + else: + # Cases: `{ my_attrs: "value" }` or `{ "key": my_attrs }` + dict_pair.append(entry) + if len(dict_pair) == 2: + dict_pair = [] + # If, at the end, there an unmatched key-value pair, raise an error + if dict_pair: + raise TemplateSyntaxError("Dictionary key is missing a value") + + del curr_value.meta["expects_key"] + + taken_n(1) # } + stack.pop() + # Allow only 1 top-level dict, similar to JSON + if stack[-1].type == "simple": + stack.pop() + continue + + elif is_next_token([","]): + if curr_value.type not in ("list", "dict"): + raise TemplateSyntaxError("Unexpected comma") + taken_n(1) # , + if curr_value.type == "dict": + curr_value.meta["expects_key"] = True + continue + + # NOTE: Altho `:` is used also in filter syntax, the "value" part + # that the filter is part of is parsed as a whole block. So if we got + # here, we know we're NOT in filter. + elif is_next_token([":"]): + if curr_value.type != "dict": + raise TemplateSyntaxError("Unexpected colon") + if not curr_value.meta["expects_key"]: + raise TemplateSyntaxError("Unexpected colon") + taken_n(1) # : + curr_value.meta["expects_key"] = False + continue + + else: + # Allow only 1 top-level plain value, similar to JSON + if curr_value.type == "simple": + stack.pop() + else: + if is_at_end(): + raise TemplateSyntaxError("Unexpected end of text") + + # Once we got here, we know that next token is NOT a list nor dict. + # So we can now parse the value. + + # Parse all filter parts of a value, e.g. `height="20" | yesno : "1,2,3" | lower` + # should be parsed as `"20" | yesno : "1,2,3" | lower` + values_parts: List[TagValuePart] = [] + is_first_part = True + end_of_value = False + while not end_of_value: + is_translation = False + + take_while(TAG_WHITESPACE) + + if is_at_end(): + if is_first_part: + raise TemplateSyntaxError("Unexpected end of text") + else: + end_of_value = True + continue + + # In this case we've reached the end of a filter sequence + # e.g. image: `height="20"|lower key1=value1` + # and we're here: ^ + # such that the next token already belongs to the next attribute. + if not is_first_part and not is_next_token(TAG_FILTER): + end_of_value = True + continue + + # Catch cases like `|filter` or `:arg`, which should be `var|filter` or `filter:arg` + elif is_first_part and is_next_token(TAG_FILTER): + raise TemplateSyntaxError("Filter is missing a value") + + # Get past the filter tokens like `|` or `:`, until the next value part. + # E.g. imagine: `height="20" | yesno : "1,2,3" | lower` + # and we're here: ^ + # and we want to parse `yesno` next + if not is_first_part: + filter_token = taken_n(1) # | or : + take_while(TAG_WHITESPACE) # Allow whitespace after filter + else: + filter_token = None + is_first_part = False + + # Move the spread syntax out of the way, so that we properly handle what's next. + # Spread syntax MUST NOT be part of a filter, so that will raise if so. + # + # NOTE: To be consistent with Python API, the spread operator is marked with `*` or `**` + # inside lists and dicts: + # - `...` - Outside: `{% component ...spread %}` + # - `*` - Inside lists: `{% component key=[ *spread ] %}` + # - `**` - Inside dicts: `{% component key={ **spread } %}` + spread_token = extract_spread_token(curr_value, filter_token) + # Handle top-level spread `{% component ...attrs %}` + if curr_value.type == "simple": + curr_value.spread = spread_token + + # IMPORTANT!!! Depending on whether we're in a list or dict, there may be extra terminal tokens. + # + # E.g. in `[value | filter : argument, value2 | filter2 : argument2]`, the two values + # are separated by a comma, and terminated by `]`. + # + # And in `{key1: value1 | filter1 : argument1, key2: value2 | filter2 : argument2}` + # the two key-value pairs are separated by a comma, and terminated by `}`. + # + # But as you can see, the dictionary also uses `:` syntax to separate the key from value. + # This effectively means that when we're parsing a dictionary KEY, we're unable to tell + # if the next `:` is the key-value syntax versus filter argument syntax. + # + # THUS, to resolve this, in dictionary we don't allow KEY to have a filter argument syntax `:`. + # So if we see `:`, we end the key part of key-value syntax, and start parsing the value. + if curr_value.type == "dict": + if curr_value.meta["expects_key"]: + terminal_tokens: Tuple[str, ...] = (":", ",", "}") + else: + if spread_token: + raise TemplateSyntaxError("Spread syntax cannot be used in place of a dictionary value") + terminal_tokens = (",", "}") + elif curr_value.type == "list": + terminal_tokens = (",", "]") + else: + terminal_tokens = tuple() + + # Parse the value + # + # E.g. imagine: `height="20" | yesno : "1,2,3" | lower` + # and we're here: ^ + # or here: ^ + # or here: ^ + # or here: ^ + if is_next_token(["'", '"', "_("]): + # NOTE: Strings may be wrapped in `_()` to allow for translation. + # See https://docs.djangoproject.com/en/5.1/topics/i18n/translation/#string-literals-passed-to-tags-and-filters # noqa: E501 + # NOTE 2: We could potentially raise if this token is supposed to be a filter + # name (after `|`) and we got a translation or a quoted string instead. But we + # leave that up for Django. + if is_next_token(["_("]): + taken_n(2) # _( + # There may be whitespace between the translation syntax and the quote. + # E.g. `_("20")` vs `_( "20" )` + take_while(TAG_WHITESPACE) # Allow whitespace after translation + is_translation = True + + quote_char = taken_n(1) # " or ' + + # NOTE: Handle escaped quotes like \" or \', and continue until we reach the closing quote. + value = take_until([quote_char], ignore=["\\" + quote_char]) + + if is_next_token([quote_char]): + add_token(quote_char) + if is_translation: + # There may be whitespace between the translation syntax and the quote. + # E.g. `_("20")` vs `_( "20" )` + take_while(TAG_WHITESPACE) # Allow whitespace after translation + taken_n(1) # ) + quoted = quote_char + # Handle the case when there is a trailing quote, e.g. when a text value is not closed. + # `{% component 'my_comp' text="organis %}` + else: + quoted = None + value = quote_char + value + # E.g. the `20` or `lower` of `height=20|lower` + # Since this is not a string, we know that it CANNOT contain whitespace. + # + # NOTE: This branch is also taken by terminal tokens like `]` or `}`. + else: + quoted = None + value = take_until(TAG_WHITESPACE + TAG_FILTER + terminal_tokens) + + take_while(TAG_WHITESPACE) + + if terminal_tokens and is_next_token(terminal_tokens): + end_of_value = True + + values_parts.append( + TagValuePart( + value=value, + quoted=quoted, + spread=spread_token, + translation=is_translation, + filter=filter_token, + ) + ) + + # Here we're done with the value (+ a sequence of filters) + # E.g. `height="20" | yesno : "1,2,3" | lower` + # we're here: ^ + # + # This whole sequence could be part of a list or a dict, + # E.g. `[my_comp, 'height="20" | yesno : "1,2,3" | lower']` + # + # So we add it to the parent struct + curr_value.entries.append(TagValue(parts=values_parts)) + + if curr_value.type == "dict": + if values_parts[0].spread: + # Validation for `{"key": **spread }` + if not curr_value.meta["expects_key"]: + raise TemplateSyntaxError( + "Got spread syntax on the position of a value inside a dictionary key-value pair" + ) + + # Validation for `{**spread: value }` + take_while(TAG_WHITESPACE) + if is_next_token([":"]): + raise TemplateSyntaxError("Spread syntax cannot be used in place of a dictionary key") + else: + # Validation for `{"key", value }` + if curr_value.meta["expects_key"]: + take_while(TAG_WHITESPACE) + if not is_next_token([":"]): + raise TemplateSyntaxError("Dictionary key is missing a value") + + # And at this point, we have the full representation of the tag value, + # including any lists or dictionaries (even nested). E.g. + # ```py + # TagValueStruct(type="simple", entries=[ + # TagValueStruct(type="list", entries=[ + # TagValuePart(value="my_comp", quoted=None, spread=False, translation=False, filter=None), + # TagValuePart(value="'height=\"20\" | yesno : \"1,2,3\" | lower'", quoted="'", spread=False, translation=False, filter=None), # noqa: E501 + # TagValueStruct(type="dict", entries=[ + # TagValuePart(value="key1", quoted=None, spread=False, translation=False, filter=None), + # TagValuePart(value="value1|filter2 : \"1,2,3\"", quoted="'", spread=False, translation=False, filter=None), # noqa: E501 + # ]), + # ]), + # ]) + # ``` + + # Unwrap top-level list / dict + if isinstance(total_value.entries[0], TagValueStruct) and total_value.entries[0].type != "simple": + total_value = total_value.entries[0] attrs.append( TagAttr( key=key, - parts=parts, start_index=start_index, - spread=is_spread, + value=total_value, ) ) diff --git a/tests/test_expression.py b/tests/test_expression.py index 864b1f2e..54d87b4b 100644 --- a/tests/test_expression.py +++ b/tests/test_expression.py @@ -827,7 +827,7 @@ class SpreadOperatorTests(BaseTestCase): ) ) - with self.assertRaisesMessage(TemplateSyntaxError, "Syntax operator is missing a value"): + with self.assertRaisesMessage(TemplateSyntaxError, "Spread syntax '...' is missing a value"): Template(template_str) @parametrize_context_behavior(["django", "isolated"]) diff --git a/tests/test_tag_parser.py b/tests/test_tag_parser.py index 7797415a..79a005b1 100644 --- a/tests/test_tag_parser.py +++ b/tests/test_tag_parser.py @@ -1,4 +1,6 @@ -from django_components.util.tag_parser import TagAttr, TagAttrPart, parse_tag_attrs +from django.template import TemplateSyntaxError + +from django_components.util.tag_parser import TagAttr, TagValue, TagValuePart, TagValueStruct, parse_tag from .django_test_setup import setup_test_config from .testutils import BaseTestCase @@ -8,38 +10,78 @@ setup_test_config({"autodiscover": False}) class TagParserTests(BaseTestCase): def test_args_kwargs(self): - _, attrs = parse_tag_attrs("component 'my_comp' key=val key2='val2 two' ") + _, attrs = parse_tag("component 'my_comp' key=val key2='val2 two' ") expected_attrs = [ TagAttr( key=None, start_index=0, - spread=False, - parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], + value=TagValueStruct( + type="simple", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), ), TagAttr( key=None, start_index=10, - spread=False, - parts=[TagAttrPart(value="my_comp", prefix=None, quoted="'", translation=False)], + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="my_comp", quoted="'", spread=None, translation=False, filter=None) + ] + ) + ], + ), ), TagAttr( key="key", start_index=20, - spread=False, - parts=[TagAttrPart(value="val", prefix=None, quoted=None, translation=False)], + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[TagValuePart(value="val", quoted=None, spread=None, translation=False, filter=None)] + ) + ], + ), ), TagAttr( key="key2", start_index=28, - spread=False, - parts=[TagAttrPart(value="val2 two", prefix=None, quoted="'", translation=False)], + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="val2 two", quoted="'", spread=None, translation=False, filter=None) + ] + ) + ], + ), ), ] self.assertEqual(attrs, expected_attrs) self.assertEqual( - [a.formatted() for a in attrs], + [a.serialize() for a in attrs], [ "component", "'my_comp'", @@ -49,44 +91,98 @@ class TagParserTests(BaseTestCase): ) def test_nested_quotes(self): - _, attrs = parse_tag_attrs("component 'my_comp' key=val key2='val2 \"two\"' text=\"organisation's\" ") + _, attrs = parse_tag("component 'my_comp' key=val key2='val2 \"two\"' text=\"organisation's\" ") expected_attrs = [ TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=0, - spread=False, - parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], ), TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="my_comp", quoted="'", spread=None, translation=False, filter=None) + ] + ) + ], + ), start_index=10, - spread=False, - parts=[TagAttrPart(value="my_comp", prefix=None, quoted="'", translation=False)], ), TagAttr( key="key", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[TagValuePart(value="val", quoted=None, spread=None, translation=False, filter=None)] + ) + ], + ), start_index=20, - spread=False, - parts=[TagAttrPart(value="val", prefix=None, quoted=None, translation=False)], ), TagAttr( key="key2", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value='val2 "two"', quoted="'", spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=28, - spread=False, - parts=[TagAttrPart(value='val2 "two"', prefix=None, quoted="'", translation=False)], ), TagAttr( key="text", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="organisation's", quoted='"', spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=46, - spread=False, - parts=[TagAttrPart(value="organisation's", prefix=None, quoted='"', translation=False)], ), ] self.assertEqual(attrs, expected_attrs) self.assertEqual( - [a.formatted() for a in attrs], + [a.serialize() for a in attrs], [ "component", "'my_comp'", @@ -97,50 +193,114 @@ class TagParserTests(BaseTestCase): ) def test_trailing_quote_single(self): - _, attrs = parse_tag_attrs("component 'my_comp' key=val key2='val2 \"two\"' text=\"organisation's\" 'abc") + _, attrs = parse_tag("component 'my_comp' key=val key2='val2 \"two\"' text=\"organisation's\" 'abc") expected_attrs = [ TagAttr( key=None, + value=TagValueStruct( + type="simple", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=0, - spread=False, - parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], ), TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="my_comp", quoted="'", spread=None, translation=False, filter=None) + ] + ) + ], + ), start_index=10, - spread=False, - parts=[TagAttrPart(value="my_comp", prefix=None, quoted="'", translation=False)], ), TagAttr( key="key", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[TagValuePart(value="val", quoted=None, spread=None, translation=False, filter=None)] + ) + ], + ), start_index=20, - spread=False, - parts=[TagAttrPart(value="val", prefix=None, quoted=None, translation=False)], ), TagAttr( key="key2", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value='val2 "two"', quoted="'", spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=28, - spread=False, - parts=[TagAttrPart(value='val2 "two"', prefix=None, quoted="'", translation=False)], ), TagAttr( key="text", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="organisation's", quoted='"', spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=46, - spread=False, - parts=[TagAttrPart(value="organisation's", prefix=None, quoted='"', translation=False)], ), TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="'abc", quoted=None, spread=None, translation=False, filter=None) + ] + ) + ], + ), start_index=68, - spread=False, - parts=[TagAttrPart(value="'abc", prefix=None, quoted=None, translation=False)], ), ] self.assertEqual(attrs, expected_attrs) self.assertEqual( - [a.formatted() for a in attrs], + [a.serialize() for a in attrs], [ "component", "'my_comp'", @@ -152,50 +312,114 @@ class TagParserTests(BaseTestCase): ) def test_trailing_quote_double(self): - _, attrs = parse_tag_attrs('component "my_comp" key=val key2="val2 \'two\'" text=\'organisation"s\' "abc') + _, attrs = parse_tag('component "my_comp" key=val key2="val2 \'two\'" text=\'organisation"s\' "abc') expected_attrs = [ TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=0, - spread=False, - parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], ), TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="my_comp", quoted='"', spread=None, translation=False, filter=None) + ] + ) + ], + ), start_index=10, - spread=False, - parts=[TagAttrPart(value="my_comp", prefix=None, quoted='"', translation=False)], ), TagAttr( key="key", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[TagValuePart(value="val", quoted=None, spread=None, translation=False, filter=None)] + ) + ], + ), start_index=20, - spread=False, - parts=[TagAttrPart(value="val", prefix=None, quoted=None, translation=False)], ), TagAttr( key="key2", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="val2 'two'", quoted='"', spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=28, - spread=False, - parts=[TagAttrPart(value="val2 'two'", prefix=None, quoted='"', translation=False)], ), TagAttr( key="text", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value='organisation"s', quoted="'", spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=46, - spread=False, - parts=[TagAttrPart(value='organisation"s', prefix=None, quoted="'", translation=False)], - ), + ), # noqa: E501 TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value='"abc', quoted=None, spread=None, translation=False, filter=None) + ] + ) + ], + ), start_index=68, - spread=False, - parts=[TagAttrPart(value='"abc', prefix=None, quoted=None, translation=False)], ), ] self.assertEqual(attrs, expected_attrs) self.assertEqual( - [a.formatted() for a in attrs], + [a.serialize() for a in attrs], [ "component", '"my_comp"', @@ -207,52 +431,114 @@ class TagParserTests(BaseTestCase): ) def test_trailing_quote_as_value_single(self): - _, attrs = parse_tag_attrs( - "component 'my_comp' key=val key2='val2 \"two\"' text=\"organisation's\" value='abc" - ) + _, attrs = parse_tag("component 'my_comp' key=val key2='val2 \"two\"' text=\"organisation's\" value='abc") expected_attrs = [ TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=0, - spread=False, - parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], ), TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="my_comp", quoted="'", spread=None, translation=False, filter=None) + ] + ) + ], + ), start_index=10, - spread=False, - parts=[TagAttrPart(value="my_comp", prefix=None, quoted="'", translation=False)], ), TagAttr( key="key", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[TagValuePart(value="val", quoted=None, spread=None, translation=False, filter=None)] + ) + ], + ), start_index=20, - spread=False, - parts=[TagAttrPart(value="val", prefix=None, quoted=None, translation=False)], ), TagAttr( key="key2", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value='val2 "two"', quoted="'", spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=28, - spread=False, - parts=[TagAttrPart(value='val2 "two"', prefix=None, quoted="'", translation=False)], ), TagAttr( key="text", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="organisation's", quoted='"', spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=46, - spread=False, - parts=[TagAttrPart(value="organisation's", prefix=None, quoted='"', translation=False)], ), TagAttr( key="value", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="'abc", quoted=None, spread=None, translation=False, filter=None) + ] + ) + ], + ), start_index=68, - spread=False, - parts=[TagAttrPart(value="'abc", prefix=None, quoted=None, translation=False)], ), ] self.assertEqual(attrs, expected_attrs) self.assertEqual( - [a.formatted() for a in attrs], + [a.serialize() for a in attrs], [ "component", "'my_comp'", @@ -264,52 +550,114 @@ class TagParserTests(BaseTestCase): ) def test_trailing_quote_as_value_double(self): - _, attrs = parse_tag_attrs( - 'component "my_comp" key=val key2="val2 \'two\'" text=\'organisation"s\' value="abc' - ) + _, attrs = parse_tag('component "my_comp" key=val key2="val2 \'two\'" text=\'organisation"s\' value="abc') expected_attrs = [ TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=0, - spread=False, - parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], ), TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="my_comp", quoted='"', spread=None, translation=False, filter=None) + ] + ) + ], + ), start_index=10, - spread=False, - parts=[TagAttrPart(value="my_comp", prefix=None, quoted='"', translation=False)], ), TagAttr( key="key", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[TagValuePart(value="val", quoted=None, spread=None, translation=False, filter=None)] + ) + ], + ), start_index=20, - spread=False, - parts=[TagAttrPart(value="val", prefix=None, quoted=None, translation=False)], ), TagAttr( key="key2", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="val2 'two'", quoted='"', spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=28, - spread=False, - parts=[TagAttrPart(value="val2 'two'", prefix=None, quoted='"', translation=False)], ), TagAttr( key="text", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value='organisation"s', quoted="'", spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=46, - spread=False, - parts=[TagAttrPart(value='organisation"s', prefix=None, quoted="'", translation=False)], ), TagAttr( key="value", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value='"abc', quoted=None, spread=None, translation=False, filter=None) + ] + ) + ], + ), start_index=68, - spread=False, - parts=[TagAttrPart(value='"abc', prefix=None, quoted=None, translation=False)], ), ] self.assertEqual(attrs, expected_attrs) self.assertEqual( - [a.formatted() for a in attrs], + [a.serialize() for a in attrs], [ "component", '"my_comp"', @@ -321,38 +669,76 @@ class TagParserTests(BaseTestCase): ) def test_translation(self): - _, attrs = parse_tag_attrs('component "my_comp" _("one") key=_("two")') + _, attrs = parse_tag('component "my_comp" _("one") key=_("two")') expected_attrs = [ TagAttr( key=None, - parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=0, - spread=False, ), TagAttr( key=None, - parts=[TagAttrPart(value="my_comp", prefix=None, quoted='"', translation=False)], + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="my_comp", quoted='"', spread=None, translation=False, filter=None) + ] + ) + ], + ), start_index=10, - spread=False, ), TagAttr( key=None, - parts=[TagAttrPart(value="one", prefix=None, quoted='"', translation=True)], + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[TagValuePart(value="one", quoted='"', spread=None, translation=True, filter=None)] + ) + ], + ), start_index=20, - spread=False, ), TagAttr( key="key", - parts=[TagAttrPart(value="two", prefix=None, quoted='"', translation=True)], + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[TagValuePart(value="two", quoted='"', spread=None, translation=True, filter=None)] + ) + ], + ), start_index=29, - spread=False, ), ] self.assertEqual(attrs, expected_attrs) self.assertEqual( - [a.formatted() for a in attrs], + [a.serialize() for a in attrs], [ "component", '"my_comp"', @@ -361,74 +747,1670 @@ class TagParserTests(BaseTestCase): ], ) - def test_filter(self): - _, attrs = parse_tag_attrs( - 'component "my_comp" abc|fil1 key=val|fil2:"one two "|lower|safe "val2 two"|fil3 key2=\'val2 two\'|fil3' - ) + def test_tag_parser_filters(self): + _, attrs = parse_tag('component "my_comp" value|lower key=val|yesno:"yes,no" key2=val2|default:"N/A"|upper') expected_attrs = [ TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), start_index=0, - spread=False, - parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], ), TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="my_comp", quoted='"', spread=None, translation=False, filter=None) + ] + ) + ], + ), start_index=10, - spread=False, - parts=[TagAttrPart(value="my_comp", prefix=None, quoted='"', translation=False)], ), TagAttr( key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="value", quoted=None, spread=None, translation=False, filter=None), + TagValuePart(value="lower", quoted=None, spread=None, translation=False, filter="|"), + ] + ) + ], + ), start_index=20, - spread=False, - parts=[ - TagAttrPart(value="abc", prefix=None, quoted=None, translation=False), - TagAttrPart(value="fil1", prefix="|", quoted=None, translation=False), - ], ), TagAttr( key="key", - start_index=29, - spread=False, - parts=[ - TagAttrPart(value="val", prefix=None, quoted=None, translation=False), - TagAttrPart(value="fil2", prefix="|", quoted=None, translation=False), - TagAttrPart(value="one two ", prefix=":", quoted='"', translation=False), - TagAttrPart(value="lower", prefix="|", quoted=None, translation=False), - TagAttrPart(value="safe", prefix="|", quoted=None, translation=False), - ], - ), - TagAttr( - key=None, - start_index=64, - spread=False, - parts=[ - TagAttrPart(value="val2 two", prefix=None, quoted='"', translation=False), - TagAttrPart(value="fil3", prefix="|", quoted=None, translation=False), - ], + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="val", quoted=None, spread=None, translation=False, filter=None), + TagValuePart(value="yesno", quoted=None, spread=None, translation=False, filter="|"), + TagValuePart(value="yes,no", quoted='"', spread=None, translation=False, filter=":"), + ] + ) + ], + ), + start_index=32, ), TagAttr( key="key2", - start_index=80, - spread=False, - parts=[ - TagAttrPart(value="val2 two", prefix=None, quoted="'", translation=False), - TagAttrPart(value="fil3", prefix="|", quoted=None, translation=False), - ], + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="val2", quoted=None, spread=None, translation=False, filter=None), + TagValuePart(value="default", quoted=None, spread=None, translation=False, filter="|"), + TagValuePart(value="N/A", quoted='"', spread=None, translation=False, filter=":"), + TagValuePart(value="upper", quoted=None, spread=None, translation=False, filter="|"), + ] + ) + ], + ), + start_index=55, ), ] self.assertEqual(attrs, expected_attrs) self.assertEqual( - [a.formatted() for a in attrs], + [a.serialize() for a in attrs], [ "component", '"my_comp"', - "abc|fil1", - 'key=val|fil2:"one two "|lower|safe', - '"val2 two"|fil3', - "key2='val2 two'|fil3", + "value|lower", + 'key=val|yesno:"yes,no"', + 'key2=val2|default:"N/A"|upper', + ], + ) + + def test_translation_whitespace(self): + _, attrs = parse_tag('component value=_( "test" )') + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), + start_index=0, + ), + TagAttr( + key="value", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="test", quoted='"', spread=None, translation=True, filter=None), + ] + ) + ], + ), + start_index=10, + ), + ] + self.assertEqual(attrs, expected_attrs) + + def test_filter_whitespace(self): + _, attrs = parse_tag("component value | lower key=val | upper key2=val2") + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), + start_index=0, + ), + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="value", quoted=None, spread=None, translation=False, filter=None), + TagValuePart(value="lower", quoted=None, spread=None, translation=False, filter="|"), + ] + ) + ], + ), + start_index=10, + ), + TagAttr( + key="key", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="val", quoted=None, spread=None, translation=False, filter=None), + TagValuePart(value="upper", quoted=None, spread=None, translation=False, filter="|"), + ] + ) + ], + ), + start_index=29, + ), + TagAttr( + key="key2", + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="val2", quoted=None, spread=None, translation=False, filter=None), + ] + ) + ], + ), + start_index=50, + ), + ] + + self.assertEqual(attrs, expected_attrs) + + def test_dict_simple(self): + _, attrs = parse_tag('component data={ "key": "val" }') + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), + start_index=0, + ), + TagAttr( + key="data", + value=TagValueStruct( + type="dict", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart(value="key", quoted='"', spread=None, translation=False, filter=None), + ] + ), + TagValue( + parts=[ + TagValuePart(value="val", quoted='"', spread=None, translation=False, filter=None), + ] + ), + ], + ), + start_index=10, + ), + ] + + self.assertEqual(attrs, expected_attrs) + + def test_dict_trailing_comma(self): + _, attrs = parse_tag('component data={ "key": "val", }') + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), + start_index=0, + ), + TagAttr( + key="data", + value=TagValueStruct( + type="dict", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="key", quoted='"', spread=None, translation=False, filter=None), + ] + ), + TagValue( + parts=[ + TagValuePart(value="val", quoted='"', spread=None, translation=False, filter=None), + ] + ), + ], + ), + start_index=10, + ), + ] + + self.assertEqual(attrs, expected_attrs) + + def test_dict_missing_colon(self): + with self.assertRaisesMessage(TemplateSyntaxError, "Dictionary key is missing a value"): + parse_tag('component data={ "key" }') + + def test_dict_missing_colon_2(self): + with self.assertRaisesMessage(TemplateSyntaxError, "Dictionary key is missing a value"): + parse_tag('component data={ "key", "val" }') + + def test_dict_extra_colon(self): + with self.assertRaisesMessage(TemplateSyntaxError, "Unexpected colon"): + _, attrs = parse_tag("component data={ key:: key }") + + def test_dict_spread(self): + _, attrs = parse_tag("component data={ **spread }") + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), + start_index=0, + ), + TagAttr( + key="data", + value=TagValueStruct( + type="dict", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart(value="spread", quoted=None, spread="**", translation=False, filter=None), + ] + ) + ], + ), + start_index=10, + ), + ] + + self.assertEqual(attrs, expected_attrs) + + def test_dict_spread_between_key_value_pairs(self): + _, attrs = parse_tag('component data={ "key": val, **spread, "key2": val2 }') + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + ), + start_index=0, + ), + TagAttr( + key="data", + value=TagValueStruct( + type="dict", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="key", quoted='"', spread=None, translation=False, filter=None), + ] + ), + TagValue( + parts=[ + TagValuePart(value="val", quoted=None, spread=None, translation=False, filter=None), + ] + ), + TagValue( + parts=[ + TagValuePart(value="spread", quoted=None, spread="**", translation=False, filter=None), + ] + ), + TagValue( + parts=[ + TagValuePart(value="key2", quoted='"', spread=None, translation=False, filter=None), + ] + ), + TagValue( + parts=[ + TagValuePart(value="val2", quoted=None, spread=None, translation=False, filter=None), + ] + ), + ], + ), + start_index=10, + ), + ] + + self.assertEqual(attrs, expected_attrs) + + # Test that dictionary keys cannot have filter arguments - The `:` is parsed as dictionary key separator + # So instead, the content below will be parsed as key `"key"|filter`, and value `"arg":"value"` + def test_colon_in_dictionary_keys(self): + _, attrs = parse_tag('component data={"key"|filter:"arg": "value"}') + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ), + ] + ) + ], + ), + start_index=0, + ), + TagAttr( + key="data", + value=TagValueStruct( + type="dict", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="key", quoted='"', spread=None, translation=False, filter=None), + TagValuePart(value="filter", quoted=None, spread=None, translation=False, filter="|"), + ] + ), + TagValue( + parts=[ + TagValuePart(value="arg", quoted='"', spread=None, translation=False, filter=None), + TagValuePart(value="value", quoted='"', spread=None, translation=False, filter=":"), + ] + ), + ], + ), + start_index=10, + ), + ] + self.assertEqual(attrs, expected_attrs) + + def test_list_simple(self): + _, attrs = parse_tag("component data=[1, 2, 3]") + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ), + ] + ) + ], + ), + start_index=0, + ), + TagAttr( + key="data", + value=TagValueStruct( + type="list", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart(value="1", quoted=None, spread=None, translation=False, filter=None), + ] + ), + TagValue( + parts=[ + TagValuePart(value="2", quoted=None, spread=None, translation=False, filter=None), + ] + ), + TagValue( + parts=[ + TagValuePart(value="3", quoted=None, spread=None, translation=False, filter=None), + ] + ), + ], + ), + start_index=10, + ), + ] + + self.assertEqual(attrs, expected_attrs) + + def test_list_trailing_comma(self): + _, attrs = parse_tag("component data=[1, 2, 3, ]") + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ), + ] + ) + ], + ), + start_index=0, + ), + TagAttr( + key="data", + value=TagValueStruct( + type="list", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart(value="1", quoted=None, spread=None, translation=False, filter=None), + ] + ), + TagValue( + parts=[ + TagValuePart(value="2", quoted=None, spread=None, translation=False, filter=None), + ] + ), + TagValue( + parts=[ + TagValuePart(value="3", quoted=None, spread=None, translation=False, filter=None), + ] + ), + ], + ), + start_index=10, + ), + ] + + self.assertEqual(attrs, expected_attrs) + + def test_tag_parser_lists(self): + _, attrs = parse_tag( + """ + component + nums=[ + 1, + 2|add:3, + *spread + ] + items=[ + "a"|upper, + 'b'|lower, + c|default:"d" + ] + mixed=[ + 1, + [*nested], + {"key": "val"} + ] + """ + ) + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ), + ] + ) + ], + ), + start_index=17, + ), + TagAttr( + key="nums", + value=TagValueStruct( + type="list", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[TagValuePart(value="1", quoted=None, spread=None, translation=False, filter=None)] + ), + TagValue( + parts=[ + TagValuePart(value="2", quoted=None, spread=None, translation=False, filter=None), + TagValuePart(value="add", quoted=None, spread=None, translation=False, filter="|"), + TagValuePart(value="3", quoted=None, spread=None, translation=False, filter=":"), + ] + ), + TagValue( + parts=[ + TagValuePart(value="spread", quoted=None, spread="*", translation=False, filter=None) + ] + ), + ], + ), + start_index=43, + ), + TagAttr( + key="items", + value=TagValueStruct( + type="list", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart(value="a", quoted='"', spread=None, translation=False, filter=None), + TagValuePart(value="upper", quoted=None, spread=None, translation=False, filter="|"), + ] + ), + TagValue( + parts=[ + TagValuePart(value="b", quoted="'", spread=None, translation=False, filter=None), + TagValuePart(value="lower", quoted=None, spread=None, translation=False, filter="|"), + ] + ), + TagValue( + parts=[ + TagValuePart(value="c", quoted=None, spread=None, translation=False, filter=None), + TagValuePart(value="default", quoted=None, spread=None, translation=False, filter="|"), + TagValuePart(value="d", quoted='"', spread=None, translation=False, filter=":"), + ] + ), + ], + ), + start_index=164, + ), + TagAttr( + key="mixed", + value=TagValueStruct( + type="list", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[TagValuePart(value="1", quoted=None, spread=None, translation=False, filter=None)] + ), + TagValueStruct( + type="list", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="nested", + quoted=None, + spread="*", + translation=False, + filter=None, + ) + ] + ), + ], + ), + TagValueStruct( + type="dict", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="key", + quoted='"', + spread=None, + translation=False, + filter=None, + ) + ] + ), + TagValue( + parts=[ + TagValuePart( + value="val", + quoted='"', + spread=None, + translation=False, + filter=None, + ) + ] + ), + ], + ), + ], + ), + start_index=302, + ), + ] + + self.assertEqual(attrs, expected_attrs) + self.assertEqual( + [a.serialize() for a in attrs], + [ + "component", + "nums=[1, 2|add:3, *spread]", + 'items=["a"|upper, \'b\'|lower, c|default:"d"]', + 'mixed=[1, [*nested], {"key": "val"}]', + ], + ) + + def test_tag_parser_dicts(self): + # NOTE: The case `c|default:"d": "e"|yesno:"yes,no"` tests that the first `:` + # is interpreted as a separator between key and value, not as a filter. + # So it's as if we wrote `c|default: "d":"e"|yesno:"yes,no"` + _, attrs = parse_tag( + """ + component + simple={ + "a": 1|add:2 + } + nested={ + "key"|upper: val|lower, + **spread, + "obj": {"x": 1|add:2} + } + filters={ + "a"|lower: "b"|upper, + c|default:"d": "e"|yesno:"yes,no" + } + """ + ) + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ), + ] + ), + ], + ), + start_index=13, + ), + TagAttr( + key="simple", + value=TagValueStruct( + type="dict", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[TagValuePart(value="a", quoted='"', spread=None, translation=False, filter=None)] + ), + TagValue( + parts=[ + TagValuePart(value="1", quoted=None, spread=None, translation=False, filter=None), + TagValuePart(value="add", quoted=None, spread=None, translation=False, filter="|"), + TagValuePart(value="2", quoted=None, spread=None, translation=False, filter=":"), + ] + ), + ], + ), + start_index=35, + ), + TagAttr( + key="nested", + value=TagValueStruct( + type="dict", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="key", quoted='"', spread=None, translation=False, filter=None), + TagValuePart(value="upper", quoted=None, spread=None, translation=False, filter="|"), + ] + ), + TagValue( + parts=[ + TagValuePart(value="val", quoted=None, spread=None, translation=False, filter=None), + TagValuePart(value="lower", quoted=None, spread=None, translation=False, filter="|"), + ] + ), + TagValue( + parts=[ + TagValuePart(value="spread", quoted=None, spread="**", translation=False, filter=None) + ] + ), + TagValue( + parts=[TagValuePart(value="obj", quoted='"', spread=None, translation=False, filter=None)] + ), + TagValueStruct( + type="dict", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="x", quoted='"', spread=None, translation=False, filter=None + ) + ] + ), + TagValue( + parts=[ + TagValuePart( + value="1", quoted=None, spread=None, translation=False, filter=None + ), + TagValuePart( + value="add", + quoted=None, + spread=None, + translation=False, + filter="|", + ), + TagValuePart( + value="2", quoted=None, spread=None, translation=False, filter=":" + ), + ] + ), + ], + ), + ], + ), + start_index=99, + ), + TagAttr( + key="filters", + value=TagValueStruct( + type="dict", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart(value="a", quoted='"', spread=None, translation=False, filter=None), + TagValuePart(value="lower", quoted=None, spread=None, translation=False, filter="|"), + ] + ), + TagValue( + parts=[ + TagValuePart(value="b", quoted='"', spread=None, translation=False, filter=None), + TagValuePart(value="upper", quoted=None, spread=None, translation=False, filter="|"), + ] + ), + TagValue( + parts=[ + TagValuePart(value="c", quoted=None, spread=None, translation=False, filter=None), + TagValuePart(value="default", quoted=None, spread=None, translation=False, filter="|"), + ] + ), + TagValue( + parts=[ + TagValuePart(value="d", quoted='"', spread=None, translation=False, filter=None), + TagValuePart(value="e", quoted='"', spread=None, translation=False, filter=":"), + TagValuePart(value="yesno", quoted=None, spread=None, translation=False, filter="|"), + TagValuePart(value="yes,no", quoted='"', spread=None, translation=False, filter=":"), + ] + ), + ], + ), + start_index=238, + ), + ] + + self.assertEqual(attrs, expected_attrs) + self.assertEqual( + [a.serialize() for a in attrs], + [ + "component", + 'simple={"a": 1|add:2}', + 'nested={"key"|upper: val|lower, **spread, "obj": {"x": 1|add:2}}', + 'filters={"a"|lower: "b"|upper, c|default: "d":"e"|yesno:"yes,no"}', + ], + ) + + def test_tag_parser_complex(self): + _, attrs = parse_tag( + """ + component + data={ + "items": [ + 1|add:2, + {"x"|upper: 2|add:3}, + *spread_items|default:"" + ], + "nested": { + "a": [ + 1|add:2, + *nums|default:"" + ], + "b": { + "x": [ + *more|default:"" + ] + } + }, + **rest|default, + "key": _('value')|upper + } + """ + ) + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ), + ] + ), + ], + ), + start_index=13, + ), + TagAttr( + key="data", + value=TagValueStruct( + type="dict", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="items", quoted='"', spread=None, translation=False, filter=None) + ] + ), + TagValueStruct( + type="list", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="1", quoted=None, spread=None, translation=False, filter=None + ), + TagValuePart( + value="add", + quoted=None, + spread=None, + translation=False, + filter="|", + ), + TagValuePart( + value="2", quoted=None, spread=None, translation=False, filter=":" + ), + ] + ), + TagValueStruct( + type="dict", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="x", + quoted='"', + spread=None, + translation=False, + filter=None, + ), + TagValuePart( + value="upper", + quoted=None, + spread=None, + translation=False, + filter="|", + ), + ] + ), + TagValue( + parts=[ + TagValuePart( + value="2", + quoted=None, + spread=None, + translation=False, + filter=None, + ), + TagValuePart( + value="add", + quoted=None, + spread=None, + translation=False, + filter="|", + ), + TagValuePart( + value="3", + quoted=None, + spread=None, + translation=False, + filter=":", + ), + ] + ), + ], + ), + TagValue( + parts=[ + TagValuePart( + value="spread_items", + quoted=None, + spread="*", + translation=False, + filter=None, + ), + TagValuePart( + value="default", + quoted=None, + spread=None, + translation=False, + filter="|", + ), + TagValuePart(value="", quoted='"', spread=None, translation=False, filter=":"), + ] + ), + ], + ), + TagValue( + parts=[ + TagValuePart(value="nested", quoted='"', spread=None, translation=False, filter=None) + ] + ), + TagValueStruct( + type="dict", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="a", quoted='"', spread=None, translation=False, filter=None + ) + ] + ), + TagValueStruct( + type="list", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="1", + quoted=None, + spread=None, + translation=False, + filter=None, + ), + TagValuePart( + value="add", + quoted=None, + spread=None, + translation=False, + filter="|", + ), + TagValuePart( + value="2", + quoted=None, + spread=None, + translation=False, + filter=":", + ), + ], + ), + TagValue( + parts=[ + TagValuePart( + value="nums", + quoted=None, + spread="*", + translation=False, + filter=None, + ), + TagValuePart( + value="default", + quoted=None, + spread=None, + translation=False, + filter="|", + ), + TagValuePart( + value="", + quoted='"', + spread=None, + translation=False, + filter=":", + ), + ] + ), + ], + ), + TagValue( + parts=[ + TagValuePart( + value="b", quoted='"', spread=None, translation=False, filter=None + ) + ] + ), + TagValueStruct( + type="dict", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="x", + quoted='"', + spread=None, + translation=False, + filter=None, + ) + ] + ), + TagValueStruct( + type="list", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="more", + quoted=None, + spread="*", + translation=False, + filter=None, + ), + TagValuePart( + value="default", + quoted=None, + spread=None, + translation=False, + filter="|", + ), + TagValuePart( + value="", + quoted='"', + spread=None, + translation=False, + filter=":", + ), + ] + ), + ], + ), + ], + ), + ], + ), + TagValue( + parts=[ + TagValuePart(value="rest", quoted=None, spread="**", translation=False, filter=None), + TagValuePart(value="default", quoted=None, spread=None, translation=False, filter="|"), + ] + ), + TagValue( + parts=[TagValuePart(value="key", quoted='"', spread=None, translation=False, filter=None)] + ), + TagValue( + parts=[ + TagValuePart(value="value", quoted="'", spread=None, translation=True, filter=None), + TagValuePart(value="upper", quoted=None, spread=None, translation=False, filter="|"), + ] + ), + ], + ), + start_index=35, + ), + ] + + self.assertEqual(attrs, expected_attrs) + self.assertEqual( + [a.serialize() for a in attrs], + [ + "component", + 'data={"items": [1|add:2, {"x"|upper: 2|add:3}, *spread_items|default:""], "nested": {"a": [1|add:2, *nums|default:""], "b": {"x": [*more|default:""]}}, **rest|default, "key": _(\'value\')|upper}', # noqa: E501 + ], + ) + + # Test that spread operator cannot be used as dictionary value + def test_spread_as_dictionary_value(self): + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax cannot be used in place of a dictionary value", + ): + parse_tag('component data={"key": **spread}') + + def test_spread_with_colon_interpreted_as_key(self): + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax cannot be used in place of a dictionary key", + ): + _, attrs = parse_tag("component data={**spread|abc: 123 }") + + def test_spread_in_filter_position(self): + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax cannot be used inside of a filter", + ): + _, attrs = parse_tag("component data=val|...spread|abc }") + + def test_spread_whitespace(self): + # NOTE: Separating `...` from its variable is NOT valid, and will result in error. + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax '...' is missing a value", + ): + _, attrs = parse_tag("component ... attrs") + + _, attrs = parse_tag('component dict={"a": "b", ** my_attr} list=["a", * my_list]') + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ), + ] + ), + ], + ), + start_index=0, + ), + TagAttr( + key="dict", + value=TagValueStruct( + type="dict", + spread=None, + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="a", quoted='"', spread=None, translation=False, filter=None), + ] + ), + TagValue( + parts=[ + TagValuePart(value="b", quoted='"', spread=None, translation=False, filter=None), + ] + ), + TagValue( + parts=[ + TagValuePart( + value="my_attr", quoted=None, spread="**", translation=False, filter=None + ), + ] + ), + ], + ), + start_index=10, + ), + TagAttr( + key="list", + value=TagValueStruct( + type="list", + meta={}, + spread=None, + entries=[ + TagValue( + parts=[ + TagValuePart(value="a", quoted='"', spread=None, translation=False, filter=None), + ] + ), + TagValue( + parts=[ + TagValuePart(value="my_list", quoted=None, spread="*", translation=False, filter=None), + ] + ), + ], + ), + start_index=38, + ), + ] + self.assertEqual(attrs, expected_attrs) + + # Test that one cannot use e.g. `...`, `**`, `*` in wrong places + def test_spread_incorrect_syntax(self): + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax '*' found outside of a list", + ): + _, attrs = parse_tag('component dict={"a": "b", *my_attr}') + + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax '...' found in dict. It must be used on tag attributes only", + ): + _, attrs = parse_tag('component dict={"a": "b", ...my_attr}') + + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax '**' found outside of a dictionary", + ): + _, attrs = parse_tag('component list=["a", "b", **my_list]') + + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax '...' found in list. It must be used on tag attributes only", + ): + _, attrs = parse_tag('component list=["a", "b", ...my_list]') + + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax '*' found outside of a list", + ): + _, attrs = parse_tag("component *attrs") + + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax '**' found outside of a dictionary", + ): + _, attrs = parse_tag("component **attrs") + + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax '*' found outside of a list", + ): + _, attrs = parse_tag("component key=*attrs") + + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax '**' found outside of a dictionary", + ): + _, attrs = parse_tag("component key=**attrs") + + # Test that one cannot do `key=...{"a": "b"}` + def test_spread_onto_key(self): + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax '...' cannot follow a key ('key=...attrs')", + ): + _, attrs = parse_tag('component key=...{"a": "b"}') + + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax '...' cannot follow a key ('key=...attrs')", + ): + _, attrs = parse_tag('component key=...["a", "b"]') + + with self.assertRaisesMessage( + TemplateSyntaxError, + "Spread syntax '...' cannot follow a key ('key=...attrs')", + ): + _, attrs = parse_tag("component key=...attrs") + + def test_spread_dict_literal_nested(self): + _, attrs = parse_tag('component { **{"key": val2}, "key": val1 }') + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + spread=None, + meta={}, + ), + start_index=0, + ), + TagAttr( + key=None, + value=TagValueStruct( + type="dict", + spread=None, + meta={}, + entries=[ + TagValueStruct( + type="dict", + spread="**", + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="key", + quoted='"', + spread=None, + translation=False, + filter=None, + ) + ] + ), + TagValue( + parts=[ + TagValuePart( + value="val2", + quoted=None, + spread=None, + translation=False, + filter=None, + ) + ] + ), + ], + ), + TagValue( + parts=[TagValuePart(value="key", quoted='"', spread=None, translation=False, filter=None)] + ), + TagValue( + parts=[ + TagValuePart(value="val1", quoted=None, spread=None, translation=False, filter=None) + ] + ), + ], + ), + start_index=10, + ), + ] + self.assertEqual(attrs, expected_attrs) + + self.assertEqual( + [a.serialize() for a in attrs], + [ + "component", + '{**{"key": val2}, "key": val1}', + ], + ) + + def test_spread_dict_literal_as_attribute(self): + _, attrs = parse_tag('component ...{"key": val2}') + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + spread=None, + meta={}, + ), + start_index=0, + ), + TagAttr( + key=None, + value=TagValueStruct( + type="dict", + spread="...", + meta={}, + entries=[ + TagValue( + parts=[TagValuePart(value="key", quoted='"', spread=None, translation=False, filter=None)] + ), + TagValue( + parts=[ + TagValuePart(value="val2", quoted=None, spread=None, translation=False, filter=None) + ] + ), + ], + ), + start_index=10, + ), + ] + self.assertEqual(attrs, expected_attrs) + + self.assertEqual( + [a.serialize() for a in attrs], + [ + "component", + '...{"key": val2}', + ], + ) + + def test_spread_list_literal_nested(self): + _, attrs = parse_tag("component [ *[val1], val2 ]") + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + spread=None, + meta={}, + ), + start_index=0, + ), + TagAttr( + key=None, + value=TagValueStruct( + type="list", + spread=None, + meta={}, + entries=[ + TagValueStruct( + type="list", + spread="*", + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart( + value="val1", + quoted=None, + spread=None, + translation=False, + filter=None, + ) + ] + ), + ], + ), + TagValue( + parts=[ + TagValuePart(value="val2", quoted=None, spread=None, translation=False, filter=None) + ] + ), + ], + ), + start_index=10, + ), + ] + self.assertEqual(attrs, expected_attrs) + + self.assertEqual( + [a.serialize() for a in attrs], + [ + "component", + "[*[val1], val2]", + ], + ) + + def test_spread_list_literal_as_attribute(self): + _, attrs = parse_tag("component ...[val1]") + + expected_attrs = [ + TagAttr( + key=None, + value=TagValueStruct( + type="simple", + entries=[ + TagValue( + parts=[ + TagValuePart( + value="component", quoted=None, spread=None, translation=False, filter=None + ) + ] + ) + ], + spread=None, + meta={}, + ), + start_index=0, + ), + TagAttr( + key=None, + value=TagValueStruct( + type="list", + spread="...", + meta={}, + entries=[ + TagValue( + parts=[ + TagValuePart(value="val1", quoted=None, spread=None, translation=False, filter=None) + ] + ), + ], + ), + start_index=10, + ), + ] + self.assertEqual(attrs, expected_attrs) + + self.assertEqual( + [a.serialize() for a in attrs], + [ + "component", + "...[val1]", ], )