From f6f6fcb09726afaec9d231a458f0a5e54c2ea340 Mon Sep 17 00:00:00 2001 From: Juro Oravec Date: Tue, 17 Dec 2024 20:25:56 +0100 Subject: [PATCH] refactor: Fix the use of filters with component inputs (#857) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .../templatetags/component_tags.py | 9 +- src/django_components/util/tag_parser.py | 177 +++++---- tests/test_tag_parser.py | 350 +++++++++++++++--- 3 files changed, 424 insertions(+), 112 deletions(-) diff --git a/src/django_components/templatetags/component_tags.py b/src/django_components/templatetags/component_tags.py index 627d3187..e9aeb73d 100644 --- a/src/django_components/templatetags/component_tags.py +++ b/src/django_components/templatetags/component_tags.py @@ -860,7 +860,7 @@ def _parse_tag_preprocess( # First token is tag name, e.g. `slot` in `{% slot ... %}` tag_name_attr = attrs.pop(0) - tag_name = tag_name_attr.value + tag_name = tag_name_attr.formatted_value() # Sanity check if tag_name != tag_spec.tag: @@ -872,7 +872,7 @@ def _parse_tag_preprocess( # Otherwise, depending on the tag spec, the tag may be: # 2. Block tag - With corresponding end tag, e.g. `{% endslot %}` # 3. Inlined tag - Without the end tag. - last_token = attrs[-1].value if len(attrs) else None + last_token = attrs[-1].formatted_value() if len(attrs) else None if last_token == "/": attrs.pop() is_inline = True @@ -1168,7 +1168,8 @@ def _fix_nested_tags(parser: Parser, block_token: Token) -> None: return last_attr = attrs[-1] - last_token = last_attr.value + last_attr_part = last_attr.parts[-1] + last_token = last_attr_part.value # User probably forgot to wrap the nested tag in quotes, or this is the end of the input. # `{% component ... key={% nested %} %}` @@ -1204,7 +1205,7 @@ def _fix_nested_tags(parser: Parser, block_token: Token) -> None: # There is 3 double quotes, but if the contents get split at the first `%}` # then there will be a single unclosed double quote in the last bit. - has_unclosed_quote = not last_attr.quoted and last_token and last_token[0] in ('"', "'") + has_unclosed_quote = not last_attr_part.quoted and last_token and last_token[0] in ('"', "'") needs_fixing = has_unclosed_tag and has_unclosed_quote diff --git a/src/django_components/util/tag_parser.py b/src/django_components/util/tag_parser.py index ad409133..abdc6ac4 100644 --- a/src/django_components/util/tag_parser.py +++ b/src/django_components/util/tag_parser.py @@ -1,37 +1,71 @@ from dataclasses import dataclass from typing import List, Optional, Sequence, Tuple, Union -TAG_WHITESPACE = (" ", "\t", "\n", "\r", "\f") +TAG_WHITESPACES = (" ", "\t", "\n", "\r", "\f") +TAG_QUOTES = ("'", '"', '_("', "_('") +TAG_FILTER_JOINERS = ("|", ":") +TAG_SPREAD = "..." @dataclass -class TagAttr: - key: Optional[str] - value: str - start_index: int +class TagAttrPart: """ - Start index of the attribute (include both key and value), - relative to the start of the owner Tag. + Django tag attributes may consist of multiple parts, being separated by filter pipes (`|`) + or filter arguments (`:`). This class represents a single part of the attribute value. + + E.g. in the following tag: + ```django + {% component "my_comp" key="my val's" key2=val2|filter1:"one" %} + ``` + + The `key2` attribute has three parts: `val2`, `filter1` and `"one"`. + """ + + value: str + """The actual value of the part, e.g. `val2` in `key2=val2` or `my string` in `_("my string")`.""" + prefix: Optional[str] + """ + If this part is filter or filter arguent, `prefix` is the string that connects it to the previous part. + E.g. the `|` and `:` in `key2=val2|filter1:"one"`. """ quoted: Optional[str] """Whether the value is quoted, and the character that's used for the quotation""" - spread: bool - """Whether the value is a spread syntax, e.g. `...my_var`""" translation: bool """Whether the value is a translation string, e.g. `_("my string")`""" def __post_init__(self) -> None: if self.translation and not self.quoted: raise ValueError("Translation value must be quoted") - if self.translation and self.spread: - raise ValueError("Cannot combine translation and spread syntax") - def formatted_value(self) -> str: + def formatted(self) -> str: + """ + Format the part as a string that can be used in a Django template tag. + E.g. `val2`, `|filter1:"one"`, `_("my string")`. + """ value = f"{self.quoted}{self.value}{self.quoted}" if self.quoted else self.value if self.translation: value = f"_({value})" - elif self.spread: - value = f"...{value}" + if self.prefix: + value = f"{self.prefix}{value}" + return value + + +@dataclass +class TagAttr: + key: Optional[str] + parts: List[TagAttrPart] + start_index: int + """ + Start index of the attribute (include both key and value), + relative to the start of the owner Tag. + """ + spread: bool + """Whether the value is a spread syntax, e.g. `...my_var`""" + + def formatted_value(self) -> str: + value = "".join(part.formatted() for part in self.parts) + if self.spread: + value = f"{TAG_SPREAD}{value}" return value def formatted(self) -> str: @@ -138,90 +172,109 @@ def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]: return result - # Parse + def parse_attr_parts() -> List[TagAttrPart]: + parts: List[TagAttrPart] = [] + + while index < len(text) and not is_next_token("=", *TAG_WHITESPACES): + is_translation = False + value: str = "" + quoted: Optional[str] = None + prefix: Optional[str] = None + + if is_next_token(*TAG_FILTER_JOINERS): + prefix = taken_n(1) # | or : + + # E.g. `height="20"` or `height=_("my_text")` or `height="my_text"|fil1:"one"` + if is_next_token(*TAG_QUOTES): + # NOTE: Strings may be wrapped in `_()` to allow for translation. + # See https://docs.djangoproject.com/en/5.1/topics/i18n/translation/#string-literals-passed-to-tags-and-filters # noqa: E501 + if is_next_token("_("): + taken_n(2) # _( + is_translation = True + + # NOTE: We assume no space between the translation syntax and the quote. + quote_char = taken_n(1) # " or ' + + # NOTE: Handle escaped quotes like \" or \', and continue until we reach the closing quote. + value = take_until([quote_char], ignore=["\\" + quote_char]) + # Handle the case when there is a trailing quote, e.g. when a text value is not closed. + # `{% component 'my_comp' text="organis %}` + + if is_next_token(quote_char): + add_token(quote_char) + if is_translation: + taken_n(1) # ) + quoted = quote_char + else: + quoted = None + value = quote_char + value + # E.g. `height=20` or `height=my_var` or or `height=my_var|fil1:"one"` + else: + value = take_until(["=", *TAG_WHITESPACES, *TAG_FILTER_JOINERS]) + quoted = None + + parts.append( + TagAttrPart( + value=value, + prefix=prefix, + quoted=quoted, + translation=is_translation, + ) + ) + + return parts + + # Parse attributes attrs: List[TagAttr] = [] while index < len(text): # Skip whitespace - take_while(TAG_WHITESPACE) + take_while(TAG_WHITESPACES) start_index = len(normalized) - is_translation = False + key = None # If token starts with a quote, we assume it's a value without key part. # e.g. `component 'my_comp'` # Otherwise, parse the key. - if is_next_token("'", '"', '_("', "_('", "..."): + if is_next_token(*TAG_QUOTES, TAG_SPREAD): key = None else: - key = take_until(["=", *TAG_WHITESPACE]) + parts = parse_attr_parts() # We've reached the end of the text - if not key: + if not parts: break # Has value if is_next_token("="): add_token("=") + key = "".join(part.formatted() for part in parts) else: # Actually was a value without key part + key = None attrs.append( TagAttr( - key=None, - value=key, + key=key, + parts=parts, start_index=start_index, - quoted=None, - translation=False, spread=False, ) ) continue - # Move the spread synxtax out of the way, so that we properly handle what's next. - is_spread = is_next_token("...") + # Move the spread syntax out of the way, so that we properly handle what's next. + is_spread = is_next_token(TAG_SPREAD) if is_spread: - taken_n(3) # ... + taken_n(len(TAG_SPREAD)) # ... - # Parse the value - # - # E.g. `height="20"` - # NOTE: We don't need to parse the attributes fully. We just need to account - # for the quotes. - if is_next_token("'", '"', '_("', "_('"): - # NOTE: Strings may be wrapped in `_()` to allow for translation. - # See https://docs.djangoproject.com/en/5.1/topics/i18n/translation/#string-literals-passed-to-tags-and-filters # noqa: E501 - if is_next_token("_("): - taken_n(2) # _( - is_translation = True - - # NOTE: We assume no space between the translation syntax and the quote. - quote_char = taken_n(1) # " or ' - - # NOTE: Handle escaped quotes like \" or \', and continue until we reach the closing quote. - value = take_until([quote_char], ignore=["\\" + quote_char]) - # Handle the case when there is a trailing quote, e.g. when a text value is not closed. - # `{% component 'my_comp' text="organis %}` - - if is_next_token(quote_char): - add_token(quote_char) - if is_translation: - taken_n(1) # ) - quoted = quote_char - else: - quoted = None - value = quote_char + value - # E.g. `height=20` - else: - value = take_until(TAG_WHITESPACE) - quoted = None + parts = parse_attr_parts() attrs.append( TagAttr( key=key, - value=value, + parts=parts, start_index=start_index, - quoted=quoted, spread=is_spread, - translation=is_translation, ) ) diff --git a/tests/test_tag_parser.py b/tests/test_tag_parser.py index 483b988b..7797415a 100644 --- a/tests/test_tag_parser.py +++ b/tests/test_tag_parser.py @@ -1,4 +1,4 @@ -from django_components.util.tag_parser import TagAttr, parse_tag_attrs +from django_components.util.tag_parser import TagAttr, TagAttrPart, parse_tag_attrs from .django_test_setup import setup_test_config from .testutils import BaseTestCase @@ -7,14 +7,34 @@ setup_test_config({"autodiscover": False}) class TagParserTests(BaseTestCase): - def test_tag_parser(self): + def test_args_kwargs(self): _, attrs = parse_tag_attrs("component 'my_comp' key=val key2='val2 two' ") expected_attrs = [ - TagAttr(key=None, value="component", start_index=0, quoted=None, spread=False, translation=False), - TagAttr(key=None, value="my_comp", start_index=10, quoted="'", spread=False, translation=False), - TagAttr(key="key", value="val", start_index=20, quoted=None, spread=False, translation=False), - TagAttr(key="key2", value="val2 two", start_index=28, quoted="'", spread=False, translation=False), + TagAttr( + key=None, + start_index=0, + spread=False, + parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], + ), + TagAttr( + key=None, + start_index=10, + spread=False, + parts=[TagAttrPart(value="my_comp", prefix=None, quoted="'", translation=False)], + ), + TagAttr( + key="key", + start_index=20, + spread=False, + parts=[TagAttrPart(value="val", prefix=None, quoted=None, translation=False)], + ), + TagAttr( + key="key2", + start_index=28, + spread=False, + parts=[TagAttrPart(value="val2 two", prefix=None, quoted="'", translation=False)], + ), ] self.assertEqual(attrs, expected_attrs) @@ -28,15 +48,40 @@ class TagParserTests(BaseTestCase): ], ) - def test_tag_parser_nested_quotes(self): + def test_nested_quotes(self): _, attrs = parse_tag_attrs("component 'my_comp' key=val key2='val2 \"two\"' text=\"organisation's\" ") expected_attrs = [ - TagAttr(key=None, value="component", start_index=0, quoted=None, spread=False, translation=False), - TagAttr(key=None, value="my_comp", start_index=10, quoted="'", spread=False, translation=False), - TagAttr(key="key", value="val", start_index=20, quoted=None, spread=False, translation=False), - TagAttr(key="key2", value='val2 "two"', start_index=28, quoted="'", spread=False, translation=False), - TagAttr(key="text", value="organisation's", start_index=46, quoted='"', spread=False, translation=False), + TagAttr( + key=None, + start_index=0, + spread=False, + parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], + ), + TagAttr( + key=None, + start_index=10, + spread=False, + parts=[TagAttrPart(value="my_comp", prefix=None, quoted="'", translation=False)], + ), + TagAttr( + key="key", + start_index=20, + spread=False, + parts=[TagAttrPart(value="val", prefix=None, quoted=None, translation=False)], + ), + TagAttr( + key="key2", + start_index=28, + spread=False, + parts=[TagAttrPart(value='val2 "two"', prefix=None, quoted="'", translation=False)], + ), + TagAttr( + key="text", + start_index=46, + spread=False, + parts=[TagAttrPart(value="organisation's", prefix=None, quoted='"', translation=False)], + ), ] self.assertEqual(attrs, expected_attrs) @@ -51,16 +96,46 @@ class TagParserTests(BaseTestCase): ], ) - def test_tag_parser_trailing_quote_single(self): + def test_trailing_quote_single(self): _, attrs = parse_tag_attrs("component 'my_comp' key=val key2='val2 \"two\"' text=\"organisation's\" 'abc") expected_attrs = [ - TagAttr(key=None, value="component", start_index=0, quoted=None, spread=False, translation=False), - TagAttr(key=None, value="my_comp", start_index=10, quoted="'", spread=False, translation=False), - TagAttr(key="key", value="val", start_index=20, quoted=None, spread=False, translation=False), - TagAttr(key="key2", value='val2 "two"', start_index=28, quoted="'", spread=False, translation=False), - TagAttr(key="text", value="organisation's", start_index=46, quoted='"', spread=False, translation=False), - TagAttr(key=None, value="'abc", start_index=68, quoted=None, spread=False, translation=False), + TagAttr( + key=None, + start_index=0, + spread=False, + parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], + ), + TagAttr( + key=None, + start_index=10, + spread=False, + parts=[TagAttrPart(value="my_comp", prefix=None, quoted="'", translation=False)], + ), + TagAttr( + key="key", + start_index=20, + spread=False, + parts=[TagAttrPart(value="val", prefix=None, quoted=None, translation=False)], + ), + TagAttr( + key="key2", + start_index=28, + spread=False, + parts=[TagAttrPart(value='val2 "two"', prefix=None, quoted="'", translation=False)], + ), + TagAttr( + key="text", + start_index=46, + spread=False, + parts=[TagAttrPart(value="organisation's", prefix=None, quoted='"', translation=False)], + ), + TagAttr( + key=None, + start_index=68, + spread=False, + parts=[TagAttrPart(value="'abc", prefix=None, quoted=None, translation=False)], + ), ] self.assertEqual(attrs, expected_attrs) @@ -76,17 +151,46 @@ class TagParserTests(BaseTestCase): ], ) - def test_tag_parser_trailing_quote_double(self): + def test_trailing_quote_double(self): _, attrs = parse_tag_attrs('component "my_comp" key=val key2="val2 \'two\'" text=\'organisation"s\' "abc') + expected_attrs = [ - TagAttr(key=None, value="component", start_index=0, quoted=None, spread=False, translation=False), - TagAttr(key=None, value="my_comp", start_index=10, quoted='"', spread=False, translation=False), - TagAttr(key="key", value="val", start_index=20, quoted=None, spread=False, translation=False), - TagAttr(key="key2", value="val2 'two'", start_index=28, quoted='"', spread=False, translation=False), TagAttr( - key="text", value='organisation"s', start_index=46, quoted="'", spread=False, translation=False - ), # noqa: E501 - TagAttr(key=None, value='"abc', start_index=68, quoted=None, spread=False, translation=False), + key=None, + start_index=0, + spread=False, + parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], + ), + TagAttr( + key=None, + start_index=10, + spread=False, + parts=[TagAttrPart(value="my_comp", prefix=None, quoted='"', translation=False)], + ), + TagAttr( + key="key", + start_index=20, + spread=False, + parts=[TagAttrPart(value="val", prefix=None, quoted=None, translation=False)], + ), + TagAttr( + key="key2", + start_index=28, + spread=False, + parts=[TagAttrPart(value="val2 'two'", prefix=None, quoted='"', translation=False)], + ), + TagAttr( + key="text", + start_index=46, + spread=False, + parts=[TagAttrPart(value='organisation"s', prefix=None, quoted="'", translation=False)], + ), + TagAttr( + key=None, + start_index=68, + spread=False, + parts=[TagAttrPart(value='"abc', prefix=None, quoted=None, translation=False)], + ), ] self.assertEqual(attrs, expected_attrs) @@ -102,17 +206,48 @@ class TagParserTests(BaseTestCase): ], ) - def test_tag_parser_trailing_quote_as_value_single(self): + def test_trailing_quote_as_value_single(self): _, attrs = parse_tag_attrs( "component 'my_comp' key=val key2='val2 \"two\"' text=\"organisation's\" value='abc" ) + expected_attrs = [ - TagAttr(key=None, value="component", start_index=0, quoted=None, spread=False, translation=False), - TagAttr(key=None, value="my_comp", start_index=10, quoted="'", spread=False, translation=False), - TagAttr(key="key", value="val", start_index=20, quoted=None, spread=False, translation=False), - TagAttr(key="key2", value='val2 "two"', start_index=28, quoted="'", spread=False, translation=False), - TagAttr(key="text", value="organisation's", start_index=46, quoted='"', spread=False, translation=False), - TagAttr(key="value", value="'abc", start_index=68, quoted=None, spread=False, translation=False), + TagAttr( + key=None, + start_index=0, + spread=False, + parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], + ), + TagAttr( + key=None, + start_index=10, + spread=False, + parts=[TagAttrPart(value="my_comp", prefix=None, quoted="'", translation=False)], + ), + TagAttr( + key="key", + start_index=20, + spread=False, + parts=[TagAttrPart(value="val", prefix=None, quoted=None, translation=False)], + ), + TagAttr( + key="key2", + start_index=28, + spread=False, + parts=[TagAttrPart(value='val2 "two"', prefix=None, quoted="'", translation=False)], + ), + TagAttr( + key="text", + start_index=46, + spread=False, + parts=[TagAttrPart(value="organisation's", prefix=None, quoted='"', translation=False)], + ), + TagAttr( + key="value", + start_index=68, + spread=False, + parts=[TagAttrPart(value="'abc", prefix=None, quoted=None, translation=False)], + ), ] self.assertEqual(attrs, expected_attrs) @@ -128,17 +263,48 @@ class TagParserTests(BaseTestCase): ], ) - def test_tag_parser_trailing_quote_as_value_double(self): + def test_trailing_quote_as_value_double(self): _, attrs = parse_tag_attrs( 'component "my_comp" key=val key2="val2 \'two\'" text=\'organisation"s\' value="abc' ) + expected_attrs = [ - TagAttr(key=None, value="component", start_index=0, quoted=None, spread=False, translation=False), - TagAttr(key=None, value="my_comp", start_index=10, quoted='"', spread=False, translation=False), - TagAttr(key="key", value="val", start_index=20, quoted=None, spread=False, translation=False), - TagAttr(key="key2", value="val2 'two'", start_index=28, quoted='"', spread=False, translation=False), - TagAttr(key="text", value='organisation"s', start_index=46, quoted="'", spread=False, translation=False), - TagAttr(key="value", value='"abc', start_index=68, quoted=None, spread=False, translation=False), + TagAttr( + key=None, + start_index=0, + spread=False, + parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], + ), + TagAttr( + key=None, + start_index=10, + spread=False, + parts=[TagAttrPart(value="my_comp", prefix=None, quoted='"', translation=False)], + ), + TagAttr( + key="key", + start_index=20, + spread=False, + parts=[TagAttrPart(value="val", prefix=None, quoted=None, translation=False)], + ), + TagAttr( + key="key2", + start_index=28, + spread=False, + parts=[TagAttrPart(value="val2 'two'", prefix=None, quoted='"', translation=False)], + ), + TagAttr( + key="text", + start_index=46, + spread=False, + parts=[TagAttrPart(value='organisation"s', prefix=None, quoted="'", translation=False)], + ), + TagAttr( + key="value", + start_index=68, + spread=False, + parts=[TagAttrPart(value='"abc', prefix=None, quoted=None, translation=False)], + ), ] self.assertEqual(attrs, expected_attrs) @@ -154,14 +320,34 @@ class TagParserTests(BaseTestCase): ], ) - def test_tag_parser_translation(self): + def test_translation(self): _, attrs = parse_tag_attrs('component "my_comp" _("one") key=_("two")') expected_attrs = [ - TagAttr(key=None, value="component", start_index=0, quoted=None, spread=False, translation=False), - TagAttr(key=None, value="my_comp", start_index=10, quoted='"', spread=False, translation=False), - TagAttr(key=None, value="one", start_index=20, quoted='"', spread=False, translation=True), - TagAttr(key="key", value="two", start_index=29, quoted='"', spread=False, translation=True), + TagAttr( + key=None, + parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], + start_index=0, + spread=False, + ), + TagAttr( + key=None, + parts=[TagAttrPart(value="my_comp", prefix=None, quoted='"', translation=False)], + start_index=10, + spread=False, + ), + TagAttr( + key=None, + parts=[TagAttrPart(value="one", prefix=None, quoted='"', translation=True)], + start_index=20, + spread=False, + ), + TagAttr( + key="key", + parts=[TagAttrPart(value="two", prefix=None, quoted='"', translation=True)], + start_index=29, + spread=False, + ), ] self.assertEqual(attrs, expected_attrs) @@ -174,3 +360,75 @@ class TagParserTests(BaseTestCase): 'key=_("two")', ], ) + + def test_filter(self): + _, attrs = parse_tag_attrs( + 'component "my_comp" abc|fil1 key=val|fil2:"one two "|lower|safe "val2 two"|fil3 key2=\'val2 two\'|fil3' + ) + + expected_attrs = [ + TagAttr( + key=None, + start_index=0, + spread=False, + parts=[TagAttrPart(value="component", prefix=None, quoted=None, translation=False)], + ), + TagAttr( + key=None, + start_index=10, + spread=False, + parts=[TagAttrPart(value="my_comp", prefix=None, quoted='"', translation=False)], + ), + TagAttr( + key=None, + start_index=20, + spread=False, + parts=[ + TagAttrPart(value="abc", prefix=None, quoted=None, translation=False), + TagAttrPart(value="fil1", prefix="|", quoted=None, translation=False), + ], + ), + TagAttr( + key="key", + start_index=29, + spread=False, + parts=[ + TagAttrPart(value="val", prefix=None, quoted=None, translation=False), + TagAttrPart(value="fil2", prefix="|", quoted=None, translation=False), + TagAttrPart(value="one two ", prefix=":", quoted='"', translation=False), + TagAttrPart(value="lower", prefix="|", quoted=None, translation=False), + TagAttrPart(value="safe", prefix="|", quoted=None, translation=False), + ], + ), + TagAttr( + key=None, + start_index=64, + spread=False, + parts=[ + TagAttrPart(value="val2 two", prefix=None, quoted='"', translation=False), + TagAttrPart(value="fil3", prefix="|", quoted=None, translation=False), + ], + ), + TagAttr( + key="key2", + start_index=80, + spread=False, + parts=[ + TagAttrPart(value="val2 two", prefix=None, quoted="'", translation=False), + TagAttrPart(value="fil3", prefix="|", quoted=None, translation=False), + ], + ), + ] + + self.assertEqual(attrs, expected_attrs) + self.assertEqual( + [a.formatted() for a in attrs], + [ + "component", + '"my_comp"', + "abc|fil1", + 'key=val|fil2:"one two "|lower|safe', + '"val2 two"|fil3', + "key2='val2 two'|fil3", + ], + )