refactor: use the tag parser to streamline the tag handlers (#827)

This commit is contained in:
Juro Oravec 2024-12-13 09:00:03 +01:00 committed by GitHub
parent db4ca8b74f
commit 894dee3cad
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 351 additions and 429 deletions

View file

@ -169,10 +169,6 @@ def resolve_string(
return parser.compile_filter(s).resolve(context)
def is_kwarg(key: str) -> bool:
return "=" in key
def is_aggregate_key(key: str) -> bool:
# NOTE: If we get a key that starts with `:`, like `:class`, we do not split it.
# This syntax is used by Vue and AlpineJS.
@ -194,7 +190,7 @@ DYNAMIC_EXPR_RE = re.compile(
def is_dynamic_expression(value: Any) -> bool:
# NOTE: Currently dynamic expression need at least 6 characters
# for the opening and closing tags, and quotes
# for the opening and closing tags, and quotes, e.g. `"`, `{%`, `%}` in `" some text {% ... %}"`
MIN_EXPR_LEN = 6
if not isinstance(value, str) or not value or len(value) < MIN_EXPR_LEN:
@ -214,24 +210,6 @@ def is_spread_operator(value: Any) -> bool:
return value.startswith("...")
# A string that starts with `...1=`, `...29=`, etc.
# We convert the spread syntax to this, so Django parses
# it as a kwarg, so it remains in the original position.
#
# So from `...dict`, we make `...1=dict`
#
# That way it's trivial to merge the kwargs after the spread
# operator is replaced with actual values.
INTERNAL_SPREAD_OPERATOR_RE = re.compile(r"^\.\.\.\d+=")
def is_internal_spread_operator(value: Any) -> bool:
if not isinstance(value, str) or not value:
return False
return bool(INTERNAL_SPREAD_OPERATOR_RE.match(value))
def process_aggregate_kwargs(kwargs: Mapping[str, Any]) -> Dict[str, Any]:
"""
This function aggregates "prefixed" kwargs into dicts. "Prefixed" kwargs

View file

@ -6,16 +6,14 @@ from django.template import TemplateSyntaxError
from django.utils.module_loading import import_string
from django_components.expression import resolve_string
from django_components.template_parser import VAR_CHARS
from django_components.util.misc import is_str_wrapped_in_quotes
if TYPE_CHECKING:
from django_components.component_registry import ComponentRegistry
# Forward slash is added so it's possible to define components like
# `{% MyComp %}..{% /MyComp %}`
TAG_CHARS = VAR_CHARS + r"/"
# Require the start / end tags to contain NO spaces and only these characters
TAG_CHARS = r"\w\-\:\@\.\#/"
TAG_RE = re.compile(r"^[{chars}]+$".format(chars=TAG_CHARS))

View file

@ -1,207 +0,0 @@
"""
Overrides for the Django Template system to allow finer control over template parsing.
Based on Django Slippers v0.6.2 - https://github.com/mixxorz/slippers/blob/main/slippers/template.py
"""
import re
from typing import Any, Dict, List, Tuple
from django.template.base import (
FILTER_ARGUMENT_SEPARATOR,
FILTER_SEPARATOR,
FilterExpression,
Parser,
Variable,
VariableDoesNotExist,
constant_string,
)
from django.template.exceptions import TemplateSyntaxError
from django.utils.regex_helper import _lazy_re_compile
######################################################################################################################
# Custom FilterExpression
#
# This is a copy of the original FilterExpression. The only difference is to allow variable names to have extra special
# characters: - : . @ #
######################################################################################################################
VAR_CHARS = r"\w\-\:\@\.\#"
filter_raw_string = r"""
^(?P<constant>{constant})|
^(?P<var>[{var_chars}]+|{num})|
(?:\s*{filter_sep}\s*
(?P<filter_name>\w+)
(?:{arg_sep}
(?:
(?P<constant_arg>{constant})|
(?P<var_arg>[{var_chars}]+|{num})
)
)?
)""".format(
constant=constant_string,
num=r"[-+\.]?\d[\d\.e]*",
# The following is the only difference from the original FilterExpression. We allow variable names to have extra
# special characters: - : . @ #
var_chars=VAR_CHARS,
filter_sep=re.escape(FILTER_SEPARATOR),
arg_sep=re.escape(FILTER_ARGUMENT_SEPARATOR),
)
filter_re = _lazy_re_compile(filter_raw_string, re.VERBOSE)
class ComponentsFilterExpression(FilterExpression):
def __init__(self, token: str, parser: Parser) -> None:
# This method is exactly the same as the original FilterExpression.__init__ method, the only difference being
# the value of `filter_re`.
self.token = token
matches = filter_re.finditer(token)
var_obj = None
filters: List[Any] = []
upto = 0
for match in matches:
start = match.start()
if upto != start:
raise TemplateSyntaxError(
"Could not parse some characters: " "%s|%s|%s" % (token[:upto], token[upto:start], token[start:])
)
if var_obj is None:
var, constant = match["var"], match["constant"]
if constant:
try:
var_obj = Variable(constant).resolve({})
except VariableDoesNotExist:
var_obj = None
elif var is None:
raise TemplateSyntaxError("Could not find variable at " "start of %s." % token)
else:
var_obj = Variable(var)
else:
filter_name = match["filter_name"]
args = []
constant_arg, var_arg = match["constant_arg"], match["var_arg"]
if constant_arg:
args.append((False, Variable(constant_arg).resolve({})))
elif var_arg:
args.append((True, Variable(var_arg)))
filter_func = parser.find_filter(filter_name)
self.args_check(filter_name, filter_func, args)
filters.append((filter_func, args))
upto = match.end()
if upto != len(token):
raise TemplateSyntaxError("Could not parse the remainder: '%s' " "from '%s'" % (token[upto:], token))
self.filters = filters
self.var = var_obj
self.is_var = isinstance(var_obj, Variable)
######################################################################################################################
# Custom token_kwargs
#
# Same as the original token_kwargs, but uses the ComponentsFilterExpression instead of the original FilterExpression.
######################################################################################################################
# Regex for token keyword arguments
kwarg_re = _lazy_re_compile(r"(?:([{var_chars}]+)=)?(.+)".format(var_chars=VAR_CHARS))
def token_kwargs(bits: List[str], parser: Parser) -> Dict[str, FilterExpression]:
"""
Parse token keyword arguments and return a dictionary of the arguments
retrieved from the ``bits`` token list.
`bits` is a list containing the remainder of the token (split by spaces)
that is to be checked for arguments. Valid arguments are removed from this
list.
There is no requirement for all remaining token ``bits`` to be keyword
arguments, so return the dictionary as soon as an invalid argument format
is reached.
"""
if not bits:
return {}
match = kwarg_re.match(bits[0])
kwarg_format = match and match[1]
if not kwarg_format:
return {}
kwargs: Dict[str, FilterExpression] = {}
while bits:
if kwarg_format:
match = kwarg_re.match(bits[0])
if not match or not match[1]:
return kwargs
key, value = match.groups()
del bits[:1]
else:
if len(bits) < 3 or bits[1] != "as":
return kwargs
key, value = bits[2], bits[0]
del bits[:3]
# This is the only difference from the original token_kwargs. We use
# the ComponentsFilterExpression instead of the original FilterExpression.
kwargs[key] = ComponentsFilterExpression(value, parser)
if bits and not kwarg_format:
if bits[0] != "and":
return kwargs
del bits[:1]
return kwargs
def parse_bits(
parser: Parser,
bits: List[str],
params: List[str],
name: str,
) -> Tuple[List[FilterExpression], List[Tuple[str, FilterExpression]]]:
"""
Parse bits for template tag helpers simple_tag and inclusion_tag, in
particular by detecting syntax errors and by extracting positional and
keyword arguments.
This is a simplified version of `django.template.library.parse_bits`
where we use custom regex to handle special characters in keyword names.
Furthermore, our version allows duplicate keys, and instead of return kwargs
as a dict, we return it as a list of key-value pairs. So it is up to the
user of this function to decide whether they support duplicate keys or not.
"""
args: List[FilterExpression] = []
kwargs: List[Tuple[str, FilterExpression]] = []
unhandled_params = list(params)
for bit in bits:
# First we try to extract a potential kwarg from the bit
kwarg = token_kwargs([bit], parser)
if kwarg:
# The kwarg was successfully extracted
param, value = kwarg.popitem()
# All good, record the keyword argument
kwargs.append((str(param), value))
if param in unhandled_params:
# If using the keyword syntax for a positional arg, then
# consume it.
unhandled_params.remove(param)
else:
if kwargs:
raise TemplateSyntaxError(
"'%s' received some positional argument(s) after some " "keyword argument(s)" % name
)
else:
# Record the positional argument
args.append(parser.compile_filter(bit))
try:
# Consume from the list of expected positional arguments
unhandled_params.pop(0)
except IndexError:
pass
if unhandled_params:
# Some positional arguments were not supplied
raise TemplateSyntaxError(
"'%s' did not receive value(s) for the argument(s): %s"
% (name, ", ".join("'%s'" % p for p in unhandled_params))
)
return args, kwargs

View file

@ -12,10 +12,10 @@
# During documentation generation, we access the `fn._tag_spec`.
import functools
from typing import Any, Callable, Dict, List, Literal, NamedTuple, Optional, Set, Union
from typing import Any, Callable, Dict, List, Literal, NamedTuple, Optional, Set, Tuple, Union
import django.template
from django.template.base import NodeList, Parser, TextNode, Token, TokenType
from django.template.base import FilterExpression, NodeList, Parser, TextNode, Token, TokenType
from django.template.exceptions import TemplateSyntaxError
from django.utils.safestring import SafeString, mark_safe
@ -34,8 +34,6 @@ from django_components.expression import (
SpreadOperator,
is_aggregate_key,
is_dynamic_expression,
is_internal_spread_operator,
is_kwarg,
is_spread_operator,
)
from django_components.provide import PROVIDE_NAME_KWARG, ProvideNode
@ -49,10 +47,9 @@ from django_components.slots import (
SlotNode,
)
from django_components.tag_formatter import get_tag_formatter
from django_components.template_parser import parse_bits
from django_components.util.logger import trace_msg
from django_components.util.misc import gen_id
from django_components.util.tag_parser import parse_tag_attrs
from django_components.util.tag_parser import TagAttr, parse_tag_attrs
# NOTE: Variable name `register` is required by Django to recognize this as a template tag library
# See https://docs.djangoproject.com/en/dev/howto/custom-template-tags
@ -159,11 +156,12 @@ def component_css_dependencies(parser: Parser, token: Token, tag_spec: TagSpec)
If you insert this tag multiple times, ALL CSS links will be duplicately inserted into ALL these places.
"""
# Parse to check that the syntax is valid
_parse_tag(parser, token, tag_spec)
tag_id = gen_id()
_parse_tag(parser, token, tag_spec, tag_id)
return _component_dependencies("css")
@register.tag(name="component_js_dependencies")
@register.tag("component_js_dependencies")
@with_tag_spec(
TagSpec(
tag="component_js_dependencies",
@ -184,7 +182,8 @@ def component_js_dependencies(parser: Parser, token: Token, tag_spec: TagSpec) -
If you insert this tag multiple times, ALL JS scripts will be duplicately inserted into ALL these places.
"""
# Parse to check that the syntax is valid
_parse_tag(parser, token, tag_spec)
tag_id = gen_id()
_parse_tag(parser, token, tag_spec, tag_id)
return _component_dependencies("js")
@ -323,7 +322,8 @@ def slot(parser: Parser, token: Token, tag_spec: TagSpec) -> SlotNode:
\"\"\"
```
"""
tag = _parse_tag(parser, token, tag_spec)
tag_id = gen_id()
tag = _parse_tag(parser, token, tag_spec, tag_id=tag_id)
slot_name_kwarg = tag.kwargs.kwargs.get(SLOT_NAME_KWARG, None)
trace_id = f"slot-id-{tag.id} ({slot_name_kwarg})" if slot_name_kwarg else f"slot-id-{tag.id}"
@ -444,7 +444,8 @@ def fill(parser: Parser, token: Token, tag_spec: TagSpec) -> FillNode:
{% endcomponent %}
```
"""
tag = _parse_tag(parser, token, tag_spec)
tag_id = gen_id()
tag = _parse_tag(parser, token, tag_spec, tag_id=tag_id)
fill_name_kwarg = tag.kwargs.kwargs.get(SLOT_NAME_KWARG, None)
trace_id = f"fill-id-{tag.id} ({fill_name_kwarg})" if fill_name_kwarg else f"fill-id-{tag.id}"
@ -577,6 +578,8 @@ def component(
{% component "name" positional_arg keyword_arg=value ... only %}
```
"""
tag_id = gen_id()
_fix_nested_tags(parser, token)
bits = token.split_contents()
@ -599,6 +602,7 @@ def component(
"end_tag": end_tag,
}
),
tag_id=tag_id,
)
# Check for isolated context keyword
@ -703,8 +707,10 @@ def provide(parser: Parser, token: Token, tag_spec: TagSpec) -> ProvideNode:
user = self.inject("user_data")["user"]
```
"""
tag_id = gen_id()
# e.g. {% provide <name> key=val key2=val2 %}
tag = _parse_tag(parser, token, tag_spec)
tag = _parse_tag(parser, token, tag_spec, tag_id)
name_kwarg = tag.kwargs.kwargs.get(PROVIDE_NAME_KWARG, None)
trace_id = f"provide-id-{tag.id} ({name_kwarg})" if name_kwarg else f"fill-id-{tag.id}"
@ -788,7 +794,8 @@ def html_attrs(parser: Parser, token: Token, tag_spec: TagSpec) -> HtmlAttrsNode
**See more usage examples in
[HTML attributes](../../concepts/fundamentals/html_attributes#examples-for-html_attrs).**
"""
tag = _parse_tag(parser, token, tag_spec)
tag_id = gen_id()
tag = _parse_tag(parser, token, tag_spec, tag_id)
return HtmlAttrsNode(
kwargs=tag.kwargs,
@ -799,7 +806,6 @@ def html_attrs(parser: Parser, token: Token, tag_spec: TagSpec) -> HtmlAttrsNode
class ParsedTag(NamedTuple):
id: str
name: str
bits: List[str]
flags: Dict[str, bool]
args: List[Expression]
named_args: Dict[str, Expression]
@ -809,164 +815,279 @@ class ParsedTag(NamedTuple):
parse_body: Callable[[], NodeList]
class TagArg(NamedTuple):
name: str
positional_only: bool
def _parse_tag(
parser: Parser,
token: Token,
tag_spec: TagSpec,
tag_id: str,
) -> ParsedTag:
# Use a unique ID to be able to tie the fill nodes with components and slots
# NOTE: MUST be called BEFORE `parse_body()` to ensure predictable numbering
tag_id = gen_id()
tag_name, raw_args, raw_kwargs, raw_flags, is_inline = _parse_tag_preprocess(parser, token, tag_spec)
params = [*(tag_spec.positional_only_args or []), *(tag_spec.pos_or_keyword_args or [])]
parsed_tag = _parse_tag_process(
parser=parser,
tag_id=tag_id,
tag_name=tag_name,
tag_spec=tag_spec,
raw_args=raw_args,
raw_kwargs=raw_kwargs,
raw_flags=raw_flags,
is_inline=is_inline,
)
return parsed_tag
class TagKwarg(NamedTuple):
type: Literal["kwarg", "spread"]
key: str
# E.g. `class` in `attrs:class="my-class"`
inner_key: Optional[str]
value: str
def _parse_tag_preprocess(
parser: Parser,
token: Token,
tag_spec: TagSpec,
) -> Tuple[str, List[str], List[TagKwarg], Set[str], bool]:
_fix_nested_tags(parser, token)
# e.g. {% slot <name> ... %}
tag_name, *bits = token.split_contents()
_, attrs = parse_tag_attrs(token.contents)
# First token is tag name, e.g. `slot` in `{% slot <name> ... %}`
tag_name_attr = attrs.pop(0)
tag_name = tag_name_attr.value
# Sanity check
if tag_name != tag_spec.tag:
raise TemplateSyntaxError(f"Start tag parser received tag '{tag_name}', expected '{tag_spec.tag}'")
# Decide if the template tag is inline or block and strip the trailing slash
last_token = bits[-1] if len(bits) else None
# There's 3 ways how we tell when a tag ends:
# 1. If the tag contains `/` at the end, it's a self-closing tag (like `<div />`),
# and it doesn't have an end tag. In this case we strip the trailing slash.
# Otherwise, depending on the tag spec, the tag may be:
# 2. Block tag - With corresponding end tag, e.g. `{% endslot %}`
# 3. Inlined tag - Without the end tag.
last_token = attrs[-1].value if len(attrs) else None
if last_token == "/":
bits.pop()
attrs.pop()
is_inline = True
else:
# If no end tag was given, we assume that the tag is inline-only
is_inline = not tag_spec.end_tag
parsed_flags = {flag: False for flag in (tag_spec.flags or [])}
bits_without_flags: List[str] = []
seen_kwargs: Set[str] = set()
seen_agg_keys: Set[str] = set()
raw_args, raw_kwargs, raw_flags = _parse_tag_input(tag_name, attrs)
def mark_kwarg_key(key: str, is_agg_key: bool) -> None:
if (is_agg_key and key in seen_kwargs) or (not is_agg_key and key in seen_agg_keys):
raise TemplateSyntaxError(
f"Received argument '{key}' both as a regular input ({key}=...)"
f" and as an aggregate dict ('{key}:key=...'). Must be only one of the two"
)
if is_agg_key:
seen_agg_keys.add(key)
else:
seen_kwargs.add(key)
return tag_name, raw_args, raw_kwargs, raw_flags, is_inline
spread_count = 0
for bit in bits:
value = bit
bit_is_kwarg = is_kwarg(bit)
# Record which kwargs we've seen, to detect if kwargs were passed in
# as both aggregate and regular kwargs
if bit_is_kwarg:
key, value = bit.split("=", 1)
# Also pick up on aggregate keys like `attr:key=val`
if is_aggregate_key(key):
key = key.split(":")[0]
mark_kwarg_key(key, True)
else:
mark_kwarg_key(key, False)
else:
# Extract flags, which are like keywords but without the value part
if value in parsed_flags:
parsed_flags[value] = True
continue
# Extract spread operator (...dict)
elif is_spread_operator(value):
if value == "...":
raise TemplateSyntaxError("Syntax operator is missing a value")
# Replace the leading `...` with `...=`, so the parser
# interprets it as a kwargs, and keeps it in the correct
# position.
# Since there can be multiple spread operators, we suffix
# them with an index, e.g. `...0=`
internal_spread_bit = f"...{spread_count}={value[3:]}"
bits_without_flags.append(internal_spread_bit)
spread_count += 1
continue
bits_without_flags.append(bit)
bits = bits_without_flags
# To support optional args, we need to convert these to kwargs, so `parse_bits`
# can handle them. So we assign the keys to matched positional args,
# and then move the kwarg AFTER the pos args.
def _parse_tag_input(tag_name: str, attrs: List[TagAttr]) -> Tuple[List[str], List[TagKwarg], Set[str]]:
# Given a list of attributes passed to a tag, categorise them into args, kwargs, and flags.
# The result of this will be passed to plugins to allow them to modify the tag inputs.
# And only once we get back the modified inputs, we will parse the data into
# internal structures like `DynamicFilterExpression`, or `SpreadOperator`.
#
# TODO: This following section should live in `parse_bits`, but I don't want to
# modify it much to maintain some sort of compatibility with Django's version of
# `parse_bits`.
# Ideally, Django's parser would be expanded to support our use cases.
params_to_sort = [param for param in params if param not in seen_kwargs]
new_args = []
new_params = []
new_kwargs = []
for index, bit in enumerate(bits):
if is_kwarg(bit) or not len(params_to_sort):
# Pass all remaining bits (including current one) as kwargs
new_kwargs.extend(bits[index:])
break
# NOTES:
# - When args end, kwargs start. Positional args cannot follow kwargs
# - There can be multiple kwargs with same keys
# - Flags can be anywhere
# - Each flag can be present only once
is_args = True
args_or_flags: List[str] = []
kwarg_pairs: List[TagKwarg] = []
flags = set()
seen_spreads = 0
for attr in attrs:
value = attr.formatted_value()
param = params_to_sort.pop(0)
if tag_spec.pos_or_keyword_args and param in tag_spec.pos_or_keyword_args:
mark_kwarg_key(param, False)
new_kwargs.append(f"{param}={bit}")
# Spread
if is_spread_operator(value):
if value == "...":
raise TemplateSyntaxError("Syntax operator is missing a value")
kwarg = TagKwarg(type="spread", key=f"...{seen_spreads}", inner_key=None, value=value[3:])
kwarg_pairs.append(kwarg)
is_args = False
seen_spreads += 1
continue
new_args.append(bit)
new_params.append(param)
bits = [*new_args, *new_kwargs]
params = [*new_params, *params_to_sort]
# Positional or flag
elif is_args and not attr.key:
args_or_flags.append(value)
continue
# Remove any remaining optional positional args if they were not given
if tag_spec.pos_or_keyword_args:
params = [param for param in params_to_sort if param not in tag_spec.pos_or_keyword_args]
# Parse args/kwargs that will be passed to the fill
raw_args, raw_kwarg_pairs = parse_bits(
parser=parser,
bits=bits,
params=[] if tag_spec.positional_args_allow_extra else params,
name=tag_name,
)
# Post-process args/kwargs - Mark special cases like aggregate dicts
# or dynamic expressions
args: List[Expression] = []
for val in raw_args:
if is_dynamic_expression(val.token):
args.append(DynamicFilterExpression(parser, val.token))
else:
args.append(val)
kwarg_pairs: RuntimeKwargPairsInput = []
for key, val in raw_kwarg_pairs:
is_spread_op = is_internal_spread_operator(key + "=")
if is_spread_op:
# Allow to use dynamic expressions with spread operator, e.g.
# `..."{{ }}"`
if is_dynamic_expression(val.token):
expr = DynamicFilterExpression(parser, val.token)
# Keyword
elif attr.key:
if is_aggregate_key(attr.key):
key, inner_key = attr.key.split(":", 1)
else:
expr = parser.compile_filter(val.token)
kwarg_pairs.append((key, SpreadOperator(expr)))
elif is_dynamic_expression(val.token) and not is_spread_op:
kwarg_pairs.append((key, DynamicFilterExpression(parser, val.token)))
key, inner_key = attr.key, None
kwarg = TagKwarg(type="kwarg", key=key, inner_key=inner_key, value=value)
kwarg_pairs.append(kwarg)
is_args = False
continue
# Either flag or a misplaced positional arg
elif not is_args and not attr.key:
# NOTE: By definition, dynamic expressions CANNOT be identifiers, because
# they contain quotes. So we can catch those early.
if not value.isidentifier():
raise TemplateSyntaxError(
f"'{tag_name}' received positional argument '{value}' after keyword argument(s)"
)
# Otherwise, we assume that the token is a flag. It is up to the tag logic
# to decide whether this is a recognized flag or a misplaced positional arg.
if value in flags:
raise TemplateSyntaxError(f"'{tag_name}' received flag '{value}' multiple times")
flags.add(value)
continue
return args_or_flags, kwarg_pairs, flags
def _parse_tag_process(
parser: Parser,
tag_id: str,
tag_name: str,
tag_spec: TagSpec,
raw_args: List[str],
raw_kwargs: List[TagKwarg],
raw_flags: Set[str],
is_inline: bool,
) -> ParsedTag:
seen_kwargs = set([kwarg.key for kwarg in raw_kwargs if kwarg.key and kwarg.type == "kwarg"])
seen_regular_kwargs = set()
seen_agg_kwargs = set()
def check_kwarg_for_agg_conflict(kwarg: TagKwarg) -> None:
# Skip spread operators
if kwarg.type == "spread":
return
is_agg_kwarg = kwarg.inner_key
if (
(is_agg_kwarg and (kwarg.key in seen_regular_kwargs))
or (not is_agg_kwarg and (kwarg.key in seen_agg_kwargs))
): # fmt: skip
raise TemplateSyntaxError(
f"Received argument '{kwarg.key}' both as a regular input ({kwarg.key}=...)"
f" and as an aggregate dict ('{kwarg.key}:key=...'). Must be only one of the two"
)
if is_agg_kwarg:
seen_agg_kwargs.add(kwarg.key)
else:
kwarg_pairs.append((key, val))
seen_regular_kwargs.add(kwarg.key)
# Allow only as many positional args as given
if not tag_spec.positional_args_allow_extra and len(args) > len(params): # noqa F712
raise TemplateSyntaxError(f"Tag '{tag_name}' received unexpected positional arguments: {args[len(params):]}")
for raw_kwarg in raw_kwargs:
check_kwarg_for_agg_conflict(raw_kwarg)
# Params that may be passed as positional args
pos_params: List[TagArg] = [
*[TagArg(name=name, positional_only=True) for name in (tag_spec.positional_only_args or [])],
*[TagArg(name=name, positional_only=False) for name in (tag_spec.pos_or_keyword_args or [])],
]
args: List[Expression] = []
# For convenience, allow to access named args by their name instead of index
named_args = {param: args[index] for index, param in enumerate(params)}
named_args: Dict[str, Expression] = {}
kwarg_pairs: RuntimeKwargPairsInput = []
flags = set()
# When we come across a flag within positional args, we need to remember
# the offset, so we can correctly assign the args to the correct params
flag_offset = 0
for index, arg_input in enumerate(raw_args):
# Flags may be anywhere, so we need to check if the arg is a flag
if tag_spec.flags and arg_input in tag_spec.flags:
if arg_input in raw_flags or arg_input in flags:
raise TemplateSyntaxError(f"'{tag_name}' received flag '{arg_input}' multiple times")
flags.add(arg_input)
flag_offset += 1
continue
# Allow to use dynamic expressions as args, e.g. `"{{ }}"`
if is_dynamic_expression(arg_input):
arg = DynamicFilterExpression(parser, arg_input)
else:
arg = FilterExpression(arg_input, parser)
if (index - flag_offset) >= len(pos_params):
if tag_spec.positional_args_allow_extra:
args.append(arg)
continue
else:
# Allow only as many positional args as given
raise TemplateSyntaxError(
f"Tag '{tag_name}' received too many positional arguments: {raw_args[index:]}"
)
param = pos_params[index - flag_offset]
if param.positional_only:
args.append(arg)
named_args[param.name] = arg
else:
kwarg = TagKwarg(type="kwarg", key=param.name, inner_key=None, value=arg_input)
check_kwarg_for_agg_conflict(kwarg)
if param.name in seen_kwargs:
raise TemplateSyntaxError(
f"'{tag_name}' received argument '{param.name}' both as positional and keyword argument"
)
kwarg_pairs.append((param.name, arg))
if len(raw_args) - flag_offset < len(tag_spec.positional_only_args or []):
raise TemplateSyntaxError(
f"Tag '{tag_name}' received too few positional arguments. "
f"Expected {len(tag_spec.positional_only_args or [])}, got {len(raw_args) - flag_offset}"
)
for kwarg_input in raw_kwargs:
# Allow to use dynamic expressions with spread operator, e.g.
# `..."{{ }}"` or as kwargs values `key="{{ }}"`
if is_dynamic_expression(kwarg_input.value):
expr: Union[Expression, Operator] = DynamicFilterExpression(parser, kwarg_input.value)
else:
expr = FilterExpression(kwarg_input.value, parser)
if kwarg_input.type == "spread":
expr = SpreadOperator(expr)
if kwarg_input.inner_key:
full_key = f"{kwarg_input.key}:{kwarg_input.inner_key}"
else:
full_key = kwarg_input.key
kwarg_pairs.append((full_key, expr))
# Flags
flags_dict: Dict[str, bool] = {
# Base state, as defined in the tag spec
**{flag: False for flag in (tag_spec.flags or [])},
# Flags found among positional args
**{flag: True for flag in flags},
}
# Flags found among kwargs
for flag in raw_flags:
if flag in flags:
raise TemplateSyntaxError(f"'{tag_name}' received flag '{flag}' multiple times")
if flag not in (tag_spec.flags or []):
raise TemplateSyntaxError(f"'{tag_name}' received unknown flag '{flag}'")
flags.add(flag)
flags_dict[flag] = True
# Validate that there are no name conflicts between kwargs and flags
if flags.intersection(seen_kwargs):
raise TemplateSyntaxError(
f"'{tag_name}' received flags that conflict with keyword arguments: {flags.intersection(seen_kwargs)}"
)
# Validate kwargs
kwargs: RuntimeKwargsInput = {}
@ -1010,8 +1131,7 @@ def _parse_tag(
return ParsedTag(
id=tag_id,
name=tag_name,
bits=bits,
flags=parsed_flags,
flags=flags_dict,
args=args,
named_args=named_args,
kwargs=RuntimeKwargs(kwargs),

View file

@ -13,8 +13,26 @@ class TagAttr:
Start index of the attribute (include both key and value),
relative to the start of the owner Tag.
"""
quoted: bool
"""Whether the value is quoted (either with single or double quotes)"""
quoted: Optional[str]
"""Whether the value is quoted, and the character that's used for the quotation"""
spread: bool
"""Whether the value is a spread syntax, e.g. `...my_var`"""
translation: bool
"""Whether the value is a translation string, e.g. `_("my string")`"""
def __post_init__(self) -> None:
if self.translation and not self.quoted:
raise ValueError("Translation value must be quoted")
if self.translation and self.spread:
raise ValueError("Cannot combine translation and spread syntax")
def formatted_value(self) -> str:
value = f"{self.quoted}{self.value}{self.quoted}" if self.quoted else self.value
if self.translation:
value = f"_({value})"
elif self.spread:
value = f"...{value}"
return value
# Parse the content of a Django template tag like this:
@ -125,7 +143,7 @@ def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]:
# If token starts with a quote, we assume it's a value without key part.
# e.g. `component 'my_comp'`
# Otherwise, parse the key.
if is_next_token("'", '"', '_("', "_('"):
if is_next_token("'", '"', '_("', "_('", "..."):
key = None
else:
key = take_until(["=", *TAG_WHITESPACE])
@ -144,11 +162,18 @@ def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]:
key=None,
value=key,
start_index=start_index,
quoted=False,
quoted=None,
translation=False,
spread=False,
)
)
continue
# Move the spread synxtax out of the way, so that we properly handle what's next.
is_spread = is_next_token("...")
if is_spread:
taken_n(3) # ...
# Parse the value
#
# E.g. `height="20"`
@ -174,17 +199,15 @@ def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]:
if is_next_token(quote_char):
add_token(quote_char)
if is_translation:
value += taken_n(1) # )
quoted = True
taken_n(1) # )
quoted = quote_char
else:
quoted = False
quoted = None
value = quote_char + value
if is_translation:
value = "_(" + value
# E.g. `height=20`
else:
value = take_until(TAG_WHITESPACE)
quoted = False
quoted = None
attrs.append(
TagAttr(
@ -192,6 +215,8 @@ def parse_tag_attrs(text: str) -> Tuple[str, List[TagAttr]]:
value=value,
start_index=start_index,
quoted=quoted,
spread=is_spread,
translation=False,
)
)

View file

@ -128,7 +128,7 @@ class HtmlAttrsTests(BaseTestCase):
template = Template(self.template_str)
with self.assertRaisesMessage(
TemplateSyntaxError, "'html_attrs' received some positional argument(s) after some keyword"
TemplateSyntaxError, "'html_attrs' received too many positional arguments: ['class']"
):
template.render(Context({"class_var": "padding-top-8"}))

View file

@ -775,9 +775,7 @@ class SpreadOperatorTests(BaseTestCase):
)
)
with self.assertRaisesMessage(
TemplateSyntaxError, "'component' received some positional argument(s) after some keyword argument(s)"
):
with self.assertRaisesMessage(TemplateSyntaxError, "'component' received unknown flag 'var_a'"):
Template(template_str)
@parametrize_context_behavior(["django", "isolated"])

View file

@ -12,10 +12,10 @@ class TagParserTests(BaseTestCase):
self.assertEqual(
attrs,
[
TagAttr(key=None, value="component", start_index=0, quoted=False),
TagAttr(key=None, value="my_comp", start_index=10, quoted=True),
TagAttr(key="key", value="val", start_index=20, quoted=False),
TagAttr(key="key2", value="val2 two", start_index=28, quoted=True),
TagAttr(key=None, value="component", start_index=0, quoted=None, spread=False, translation=False),
TagAttr(key=None, value="my_comp", start_index=10, quoted="'", spread=False, translation=False),
TagAttr(key="key", value="val", start_index=20, quoted=None, spread=False, translation=False),
TagAttr(key="key2", value="val2 two", start_index=28, quoted="'", spread=False, translation=False),
],
)
@ -24,11 +24,13 @@ class TagParserTests(BaseTestCase):
self.assertEqual(
attrs,
[
TagAttr(key=None, value="component", start_index=0, quoted=False),
TagAttr(key=None, value="my_comp", start_index=10, quoted=True),
TagAttr(key="key", value="val", start_index=20, quoted=False),
TagAttr(key="key2", value='val2 "two"', start_index=28, quoted=True),
TagAttr(key="text", value="organisation's", start_index=46, quoted=True),
TagAttr(key=None, value="component", start_index=0, quoted=None, spread=False, translation=False),
TagAttr(key=None, value="my_comp", start_index=10, quoted="'", spread=False, translation=False),
TagAttr(key="key", value="val", start_index=20, quoted=None, spread=False, translation=False),
TagAttr(key="key2", value='val2 "two"', start_index=28, quoted="'", spread=False, translation=False),
TagAttr(
key="text", value="organisation's", start_index=46, quoted='"', spread=False, translation=False
),
],
)
@ -38,12 +40,14 @@ class TagParserTests(BaseTestCase):
self.assertEqual(
attrs,
[
TagAttr(key=None, value="component", start_index=0, quoted=False),
TagAttr(key=None, value="my_comp", start_index=10, quoted=True),
TagAttr(key="key", value="val", start_index=20, quoted=False),
TagAttr(key="key2", value='val2 "two"', start_index=28, quoted=True),
TagAttr(key="text", value="organisation's", start_index=46, quoted=True),
TagAttr(key=None, value="'abc", start_index=68, quoted=False),
TagAttr(key=None, value="component", start_index=0, quoted=None, spread=False, translation=False),
TagAttr(key=None, value="my_comp", start_index=10, quoted="'", spread=False, translation=False),
TagAttr(key="key", value="val", start_index=20, quoted=None, spread=False, translation=False),
TagAttr(key="key2", value='val2 "two"', start_index=28, quoted="'", spread=False, translation=False),
TagAttr(
key="text", value="organisation's", start_index=46, quoted='"', spread=False, translation=False
),
TagAttr(key=None, value="'abc", start_index=68, quoted=None, spread=False, translation=False),
],
)
@ -52,12 +56,14 @@ class TagParserTests(BaseTestCase):
self.assertEqual(
attrs,
[
TagAttr(key=None, value="component", start_index=0, quoted=False),
TagAttr(key=None, value="my_comp", start_index=10, quoted=True),
TagAttr(key="key", value="val", start_index=20, quoted=False),
TagAttr(key="key2", value="val2 'two'", start_index=28, quoted=True),
TagAttr(key="text", value='organisation"s', start_index=46, quoted=True),
TagAttr(key=None, value='"abc', start_index=68, quoted=False),
TagAttr(key=None, value="component", start_index=0, quoted=None, spread=False, translation=False),
TagAttr(key=None, value="my_comp", start_index=10, quoted='"', spread=False, translation=False),
TagAttr(key="key", value="val", start_index=20, quoted=None, spread=False, translation=False),
TagAttr(key="key2", value="val2 'two'", start_index=28, quoted='"', spread=False, translation=False),
TagAttr(
key="text", value='organisation"s', start_index=46, quoted="'", spread=False, translation=False
), # noqa: E501
TagAttr(key=None, value='"abc', start_index=68, quoted=None, spread=False, translation=False),
],
)
@ -68,12 +74,14 @@ class TagParserTests(BaseTestCase):
self.assertEqual(
attrs,
[
TagAttr(key=None, value="component", start_index=0, quoted=False),
TagAttr(key=None, value="my_comp", start_index=10, quoted=True),
TagAttr(key="key", value="val", start_index=20, quoted=False),
TagAttr(key="key2", value='val2 "two"', start_index=28, quoted=True),
TagAttr(key="text", value="organisation's", start_index=46, quoted=True),
TagAttr(key="value", value="'abc", start_index=68, quoted=False),
TagAttr(key=None, value="component", start_index=0, quoted=None, spread=False, translation=False),
TagAttr(key=None, value="my_comp", start_index=10, quoted="'", spread=False, translation=False),
TagAttr(key="key", value="val", start_index=20, quoted=None, spread=False, translation=False),
TagAttr(key="key2", value='val2 "two"', start_index=28, quoted="'", spread=False, translation=False),
TagAttr(
key="text", value="organisation's", start_index=46, quoted='"', spread=False, translation=False
),
TagAttr(key="value", value="'abc", start_index=68, quoted=None, spread=False, translation=False),
],
)
@ -84,11 +92,13 @@ class TagParserTests(BaseTestCase):
self.assertEqual(
attrs,
[
TagAttr(key=None, value="component", start_index=0, quoted=False),
TagAttr(key=None, value="my_comp", start_index=10, quoted=True),
TagAttr(key="key", value="val", start_index=20, quoted=False),
TagAttr(key="key2", value="val2 'two'", start_index=28, quoted=True),
TagAttr(key="text", value='organisation"s', start_index=46, quoted=True),
TagAttr(key="value", value='"abc', start_index=68, quoted=False),
TagAttr(key=None, value="component", start_index=0, quoted=None, spread=False, translation=False),
TagAttr(key=None, value="my_comp", start_index=10, quoted='"', spread=False, translation=False),
TagAttr(key="key", value="val", start_index=20, quoted=None, spread=False, translation=False),
TagAttr(key="key2", value="val2 'two'", start_index=28, quoted='"', spread=False, translation=False),
TagAttr(
key="text", value='organisation"s', start_index=46, quoted="'", spread=False, translation=False
),
TagAttr(key="value", value='"abc', start_index=68, quoted=None, spread=False, translation=False),
],
)

View file

@ -26,7 +26,7 @@ class ParserTest(BaseTestCase):
pos_or_keyword_args=["num", "var"],
keywordonly_args=True,
)
tag = _parse_tag(parser, parser.tokens[0], tag_spec=spec)
tag = _parse_tag(parser, parser.tokens[0], tag_spec=spec, tag_id="my-id")
ctx = {"myvar": {"a": "b"}, "val2": 1}
args = safe_resolve_list(ctx, tag.args)
@ -42,7 +42,7 @@ class ParserTest(BaseTestCase):
tokens = Lexer(template_str).tokenize()
parser = Parser(tokens=tokens)
spec = TagSpec(tag="component", keywordonly_args=True)
tag = _parse_tag(parser, parser.tokens[0], tag_spec=spec)
tag = _parse_tag(parser, parser.tokens[0], tag_spec=spec, tag_id="my-id")
ctx = Context({"date": 2024, "bzz": "fzz"})
args = safe_resolve_list(ctx, tag.args)

View file

@ -551,7 +551,7 @@ class NestedTagsTests(BaseTestCase):
"""
rendered = Template(template).render(Context())
expected = """
Variable: <strong>organisation's</strong>
Variable: <strong>organisation&#x27;s</strong>
"""
self.assertHTMLEqual(rendered, expected)
@ -565,7 +565,7 @@ class NestedTagsTests(BaseTestCase):
"""
rendered = Template(template).render(Context())
expected = """
Variable: <strong>organisation's</strong>
Variable: <strong>organisation&#x27;s</strong>
"""
self.assertHTMLEqual(rendered, expected)