From a293787f8cdc5fb27c3b80ce4ff91fd6c1946094 Mon Sep 17 00:00:00 2001 From: Benjamin Woodruff Date: Wed, 3 Jul 2019 17:29:44 -0700 Subject: [PATCH] Add node class and parser implementation for List This is based heavily on the implementation of Tuple, and was pretty straightforward as a result. --- libcst/nodes/__init__.py | 1 + libcst/nodes/_expression.py | 46 +++++++++- libcst/nodes/_statement.py | 5 +- libcst/nodes/tests/test_list.py | 109 +++++++++++++++++++++++ libcst/parser/_conversions/expression.py | 38 ++++++-- 5 files changed, 184 insertions(+), 15 deletions(-) create mode 100644 libcst/nodes/tests/test_list.py diff --git a/libcst/nodes/__init__.py b/libcst/nodes/__init__.py index d3d375f8..6e7f9988 100644 --- a/libcst/nodes/__init__.py +++ b/libcst/nodes/__init__.py @@ -46,6 +46,7 @@ from libcst.nodes._expression import ( Lambda, LeftParen, LeftSquareBracket, + List, Name, Number, Param, diff --git a/libcst/nodes/_expression.py b/libcst/nodes/_expression.py index d30492b5..dfcd82a0 100644 --- a/libcst/nodes/_expression.py +++ b/libcst/nodes/_expression.py @@ -15,7 +15,7 @@ from tokenize import ( Imagnumber as IMAGNUMBER_RE, Intnumber as INTNUMBER_RE, ) -from typing import Callable, Generator, List, Optional, Sequence, Union +from typing import Callable, Generator, Optional, Sequence, Union from typing_extensions import Literal @@ -1505,7 +1505,7 @@ class Lambda(BaseExpression): # Validate parents super(Lambda, self)._validate() # Sum up all parameters - all_params: List[Param] = [ + all_params = [ *self.params.params, *self.params.default_params, *self.params.kwonly_params, @@ -2214,3 +2214,45 @@ class Tuple(BaseAtom, BaseAssignTargetExpression, BaseDelTargetExpression): default_comma=(idx < len(elements) - 1), default_comma_whitespace=True, ) + + +@add_slots +@dataclass(frozen=True) +class List(BaseAtom, BaseAssignTargetExpression, BaseDelTargetExpression): + elements: Sequence[Union[Element, StarredElement]] + + # Open bracket surrounding the list + lbracket: LeftSquareBracket = LeftSquareBracket() + + # Close bracket surrounding the list + rbracket: RightSquareBracket = RightSquareBracket() + + # Sequence of open parenthesis for precedence dictation. + lpar: Sequence[LeftParen] = () + + # Sequence of close parenthesis for precedence dictation. + rpar: Sequence[RightParen] = () + + def _safe_to_use_with_word_operator(self, position: ExpressionPosition) -> bool: + return True + + def _visit_and_replace_children(self, visitor: CSTVisitor) -> "List": + return List( + lpar=visit_sequence("lpar", self.lpar, visitor), + lbracket=visit_required("lbracket", self.lbracket, visitor), + elements=visit_sequence("elements", self.elements, visitor), + rbracket=visit_required("rbracket", self.rbracket, visitor), + rpar=visit_sequence("rpar", self.rpar, visitor), + ) + + def _codegen_impl(self, state: CodegenState) -> None: + with self._parenthesize(state): + self.lbracket._codegen(state) + elements = self.elements + for idx, el in enumerate(elements): + el._codegen( + state, + default_comma=(idx < len(elements) - 1), + default_comma_whitespace=True, + ) + self.rbracket._codegen(state) diff --git a/libcst/nodes/_statement.py b/libcst/nodes/_statement.py index a9eb75ca..5c99689c 100644 --- a/libcst/nodes/_statement.py +++ b/libcst/nodes/_statement.py @@ -27,7 +27,6 @@ from libcst.nodes._expression import ( Name, Parameters, RightParen, - Tuple, ) from libcst.nodes._internal import ( CodegenState, @@ -1603,9 +1602,7 @@ class For(BaseCompoundStatement): """ # The target of the iterator in the for statement. - target: Union[ - Name, Tuple - ] # TODO: Should be a Union[Name, Tuple, List] once we support this. + target: BaseAssignTargetExpression # The iterable expression we will loop over. iter: BaseExpression diff --git a/libcst/nodes/tests/test_list.py b/libcst/nodes/tests/test_list.py new file mode 100644 index 00000000..f31fa9e4 --- /dev/null +++ b/libcst/nodes/tests/test_list.py @@ -0,0 +1,109 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +# pyre-strict +from typing import Any, Callable + +import libcst.nodes as cst +from libcst.nodes.tests.base import CSTNodeTest +from libcst.parser import parse_expression, parse_statement +from libcst.testing.utils import data_provider + + +class ListTest(CSTNodeTest): + + # A lot of Element/StarredElement tests are provided by the tests for Tuple, so we + # we don't need to duplicate them here. + @data_provider( + [ + # zero-element list + {"node": cst.List([]), "code": "[]", "parser": parse_expression}, + # one-element list, sentinel comma value + { + "node": cst.List([cst.Element(cst.Name("single_element"))]), + "code": "[single_element]", + "parser": parse_expression, + }, + # custom whitespace between brackets + { + "node": cst.List( + [cst.Element(cst.Name("single_element"))], + lbracket=cst.LeftSquareBracket( + whitespace_after=cst.SimpleWhitespace("\t") + ), + rbracket=cst.RightSquareBracket( + whitespace_before=cst.SimpleWhitespace(" ") + ), + ), + "code": "[\tsingle_element ]", + "parser": parse_expression, + }, + # two-element list, sentinel comma value + { + "node": cst.List( + [cst.Element(cst.Name("one")), cst.Element(cst.Name("two"))] + ), + "code": "[one, two]", + "parser": None, + }, + # with parenthesis + { + "node": cst.List( + [cst.Element(cst.Name("one"))], + lpar=[cst.LeftParen()], + rpar=[cst.RightParen()], + ), + "code": "([one])", + "parser": None, + }, + # starred element + { + "node": cst.List( + [ + cst.StarredElement(cst.Name("one")), + cst.StarredElement(cst.Name("two")), + ] + ), + "code": "[*one, *two]", + "parser": None, + }, + # missing spaces around list, always okay + { + "node": cst.For( + target=cst.List( + [ + cst.Element(cst.Name("k"), comma=cst.Comma()), + cst.Element(cst.Name("v")), + ] + ), + iter=cst.Name("abc"), + body=cst.SimpleStatementSuite([cst.Pass()]), + whitespace_after_for=cst.SimpleWhitespace(""), + whitespace_before_in=cst.SimpleWhitespace(""), + ), + "code": "for[k,v]in abc: pass\n", + "parser": parse_statement, + }, + ] + ) + def test_valid(self, **kwargs: Any) -> None: + self.validate_node(**kwargs) + + @data_provider( + ( + ( + lambda: cst.List( + [cst.Element(cst.Name("mismatched"))], + lpar=[cst.LeftParen(), cst.LeftParen()], + rpar=[cst.RightParen()], + ), + "unbalanced parens", + ), + ) + ) + def test_invalid( + self, get_node: Callable[[], cst.CSTNode], expected_re: str + ) -> None: + self.assert_invalid(get_node, expected_re) diff --git a/libcst/parser/_conversions/expression.py b/libcst/parser/_conversions/expression.py index f33462bd..749a7012 100644 --- a/libcst/parser/_conversions/expression.py +++ b/libcst/parser/_conversions/expression.py @@ -9,7 +9,7 @@ from tokenize import ( Imagnumber as IMAGNUMBER_RE, Intnumber as INTNUMBER_RE, ) -from typing import Any, Dict, List, Sequence, Type +from typing import Any, Dict, List, Sequence, Type, Union import libcst.nodes as cst from libcst._maybe_sentinel import MaybeSentinel @@ -736,7 +736,28 @@ def convert_atom_basic(config: ParserConfig, children: Sequence[Any]) -> Any: @with_production("atom_squarebrackets", "'[' [testlist_comp_list] ']'") def convert_atom_squarebrackets(config: ParserConfig, children: Sequence[Any]) -> Any: - return make_dummy_node(config, children) + lbracket_tok, *body, rbracket_tok = children + lbracket = cst.LeftSquareBracket( + whitespace_after=parse_parenthesizable_whitespace( + config, lbracket_tok.whitespace_after + ) + ) + + rbracket = cst.RightSquareBracket( + whitespace_before=parse_parenthesizable_whitespace( + config, rbracket_tok.whitespace_before + ) + ) + + if len(body) == 0: + list_node = cst.List((), lbracket=lbracket, rbracket=rbracket) + else: # len(body) == 1 + if isinstance(body[0].value, cst.List): # TODO: Remove this conditional + list_node = body[0].value.with_changes(lbracket=lbracket, rbracket=rbracket) + else: # TODO: Remove this branch; this handles for DummyNodes + list_node = cst.DummyNode([lbracket, body[0].value, rbracket]) + + return WithLeadingWhitespace(list_node, lbracket_tok.whitespace_before) @with_production("atom_curlybrackets", "'{' [dictorsetmaker] '}'") @@ -921,10 +942,7 @@ def convert_fstring_format_spec(config: ParserConfig, children: Sequence[Any]) - ) def convert_testlist_comp_tuple(config: ParserConfig, children: Sequence[Any]) -> Any: return _convert_testlist_comp( - config, - children, - single_child_is_sequence=False, # should be true for testlist_comp_list - sequence_type=cst.Tuple, + config, children, single_child_is_sequence=False, sequence_type=cst.Tuple ) @@ -933,14 +951,16 @@ def convert_testlist_comp_tuple(config: ParserConfig, children: Sequence[Any]) - "(test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )", ) def convert_testlist_comp_list(config: ParserConfig, children: Sequence[Any]) -> Any: - return make_dummy_node(config, children) + return _convert_testlist_comp( + config, children, single_child_is_sequence=True, sequence_type=cst.List + ) def _convert_testlist_comp( config: ParserConfig, children: Sequence[Any], single_child_is_sequence: bool, - sequence_type: Type[cst.Tuple], + sequence_type: Union[Type[cst.Tuple], Type[cst.List]], ) -> Any: # This is either a single-element list, or the second token is a comma, so we're not # in a generator. @@ -968,7 +988,7 @@ def _convert_sequencelike( config: ParserConfig, children: Sequence[Any], single_child_is_sequence: bool, - sequence_type: Type[cst.Tuple], # TODO: Type[Union[Tuple, List, Set]] + sequence_type: Union[Type[cst.Tuple], Type[cst.List]], # TODO: support cst.Set ) -> Any: if not single_child_is_sequence and len(children) == 1: return children[0]