PEP 448: additional unpacking generalizations (closes #2292)

Patch by Neil Girdhar.

Grammar/Grammar

@@ -111,17 +111,29 @@ subscript: test | [test] ':' [test] [sliceop]
 sliceop: ':' [test]
 exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
 testlist: test (',' test)* [',']
-dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
+dictorsetmaker: ( ((test ':' test | '**' expr)
-                  (test (comp_for | (',' test)* [','])) )
+                   (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+                  ((test | star_expr)
+                   (comp_for | (',' (test | star_expr))* [','])) )
 
 classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
 
-arglist: (argument ',')* (argument [',']
+arglist: argument (',' argument)*  [',']
-                         |'*' test (',' argument)* [',' '**' test]
+
-                         |'**' test)
 # The reason that keywords are test nodes instead of NAME is that using NAME
 # results in an ambiguity. ast.c makes sure it's a NAME.
-argument: test [comp_for] | test '=' test  # Really [keyword '='] test
+# "test '=' test" is really "keyword '=' test", but we have no such token.
+# These need to be in a single rule to avoid grammar that is ambiguous
+# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
+# we explicitly match '*' here, too, to give it proper precedence.
+# Illegal combinations and orderings are blocked in ast.c:
+# multiple (test comp_for) arguements are blocked; keyword unpackings
+# that precede iterable unpackings are blocked; etc.
+argument: ( test [comp_for] |
+            test '=' test |
+            '**' expr |
+            star_expr )
+
 comp_iter: comp_for | comp_if
 comp_for: 'for' exprlist 'in' or_test [comp_iter]
 comp_if: 'if' test_nocond [comp_iter]

Include/Python-ast.h

@@ -84,8 +84,6 @@ struct _stmt {
             identifier name;
             asdl_seq *bases;
             asdl_seq *keywords;
-            expr_ty starargs;
-            expr_ty kwargs;
             asdl_seq *body;
             asdl_seq *decorator_list;
         } ClassDef;
@@ -263,8 +261,6 @@ struct _expr {
             expr_ty func;
             asdl_seq *args;
             asdl_seq *keywords;
-            expr_ty starargs;
-            expr_ty kwargs;
         } Call;
         
         struct {
@@ -406,11 +402,10 @@ mod_ty _Py_Suite(asdl_seq * body, PyArena *arena);
 stmt_ty _Py_FunctionDef(identifier name, arguments_ty args, asdl_seq * body,
                         asdl_seq * decorator_list, expr_ty returns, int lineno,
                         int col_offset, PyArena *arena);
-#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9)
+#define ClassDef(a0, a1, a2, a3, a4, a5, a6, a7) _Py_ClassDef(a0, a1, a2, a3, a4, a5, a6, a7)
 stmt_ty _Py_ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords,
-                     expr_ty starargs, expr_ty kwargs, asdl_seq * body,
+                     asdl_seq * body, asdl_seq * decorator_list, int lineno,
-                     asdl_seq * decorator_list, int lineno, int col_offset,
+                     int col_offset, PyArena *arena);
-                     PyArena *arena);
 #define Return(a0, a1, a2, a3) _Py_Return(a0, a1, a2, a3)
 stmt_ty _Py_Return(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define Delete(a0, a1, a2, a3) _Py_Delete(a0, a1, a2, a3)
@@ -504,10 +499,9 @@ expr_ty _Py_YieldFrom(expr_ty value, int lineno, int col_offset, PyArena
 #define Compare(a0, a1, a2, a3, a4, a5) _Py_Compare(a0, a1, a2, a3, a4, a5)
 expr_ty _Py_Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators,
                     int lineno, int col_offset, PyArena *arena);
-#define Call(a0, a1, a2, a3, a4, a5, a6, a7) _Py_Call(a0, a1, a2, a3, a4, a5, a6, a7)
+#define Call(a0, a1, a2, a3, a4, a5) _Py_Call(a0, a1, a2, a3, a4, a5)
-expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty
+expr_ty _Py_Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int
-                 starargs, expr_ty kwargs, int lineno, int col_offset, PyArena
+                 lineno, int col_offset, PyArena *arena);
-                 *arena);
 #define Num(a0, a1, a2, a3) _Py_Num(a0, a1, a2, a3)
 expr_ty _Py_Num(object n, int lineno, int col_offset, PyArena *arena);
 #define Str(a0, a1, a2, a3) _Py_Str(a0, a1, a2, a3)

Include/dictobject.h

@@ -105,6 +105,10 @@ PyAPI_FUNC(int) PyDict_Merge(PyObject *mp,
                                    PyObject *other,
                                    int override);
 
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject *) _PyDictView_Intersect(PyObject* self, PyObject *other);
+#endif
+
 /* PyDict_MergeFromSeq2 updates/merges from an iterable object producing
    iterable objects of length 2.  If override is true, the last occurrence
    of a key wins, else the first.  The Python dict constructor dict(seq2)

Include/opcode.h

@@ -111,6 +111,11 @@ extern "C" {
 #define SET_ADD             	146
 #define MAP_ADD             	147
 #define LOAD_CLASSDEREF     	148
+#define BUILD_LIST_UNPACK   	149
+#define BUILD_MAP_UNPACK    	150
+#define BUILD_MAP_UNPACK_WITH_CALL	151
+#define BUILD_TUPLE_UNPACK  	152
+#define BUILD_SET_UNPACK    	153
 
 /* EXCEPT_HANDLER is a special, implicit block type which is created when
    entering an except handler. It is not an opcode but we define it here

Lib/importlib/_bootstrap_external.py

@@ -220,12 +220,13 @@ _code_type = type(_write_atomic.__code__)
 #     Python 3.4a4  3300 (more changes to __qualname__ computation)
 #     Python 3.4rc2 3310 (alter __qualname__ computation)
 #     Python 3.5a0  3320 (matrix multiplication operator)
+#     Python 3.5b1  3330 (PEP 448: Additional Unpacking Generalizations)
 #
 # MAGIC must change whenever the bytecode emitted by the compiler may no
 # longer be understood by older implementations of the eval loop (usually
 # due to the addition of new opcodes).
 
-MAGIC_NUMBER = (3320).to_bytes(2, 'little') + b'\r\n'
+MAGIC_NUMBER = (3330).to_bytes(2, 'little') + b'\r\n'
 _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little')  # For import.c
 
 _PYCACHE = '__pycache__'

Lib/opcode.py

@@ -200,4 +200,10 @@ hasfree.append(148)
 def_op('EXTENDED_ARG', 144)
 EXTENDED_ARG = 144
 
+def_op('BUILD_LIST_UNPACK', 149)
+def_op('BUILD_MAP_UNPACK', 150)
+def_op('BUILD_MAP_UNPACK_WITH_CALL', 151)
+def_op('BUILD_TUPLE_UNPACK', 152)
+def_op('BUILD_SET_UNPACK', 153)
+
 del def_op, name_op, jrel_op, jabs_op

Lib/test/test_ast.py

@@ -427,17 +427,17 @@ class ASTHelpers_Test(unittest.TestCase):
         self.assertEqual(ast.dump(node),
             "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), "
             "args=[Name(id='eggs', ctx=Load()), Str(s='and cheese')], "
-            "keywords=[], starargs=None, kwargs=None))])"
+            "keywords=[]))])"
         )
         self.assertEqual(ast.dump(node, annotate_fields=False),
             "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), "
-            "Str('and cheese')], [], None, None))])"
+            "Str('and cheese')], []))])"
         )
         self.assertEqual(ast.dump(node, include_attributes=True),
             "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), "
             "lineno=1, col_offset=0), args=[Name(id='eggs', ctx=Load(), "
             "lineno=1, col_offset=5), Str(s='and cheese', lineno=1, "
-            "col_offset=11)], keywords=[], starargs=None, kwargs=None, "
+            "col_offset=11)], keywords=[], "
             "lineno=1, col_offset=0), lineno=1, col_offset=0)])"
         )
 
@@ -453,16 +453,16 @@ class ASTHelpers_Test(unittest.TestCase):
     def test_fix_missing_locations(self):
         src = ast.parse('write("spam")')
         src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()),
-                                          [ast.Str('eggs')], [], None, None)))
+                                          [ast.Str('eggs')], [])))
         self.assertEqual(src, ast.fix_missing_locations(src))
         self.assertEqual(ast.dump(src, include_attributes=True),
             "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), "
             "lineno=1, col_offset=0), args=[Str(s='spam', lineno=1, "
-            "col_offset=6)], keywords=[], starargs=None, kwargs=None, "
+            "col_offset=6)], keywords=[], "
             "lineno=1, col_offset=0), lineno=1, col_offset=0), "
             "Expr(value=Call(func=Name(id='spam', ctx=Load(), lineno=1, "
             "col_offset=0), args=[Str(s='eggs', lineno=1, col_offset=0)], "
-            "keywords=[], starargs=None, kwargs=None, lineno=1, "
+            "keywords=[], lineno=1, "
             "col_offset=0), lineno=1, col_offset=0)])"
         )
 
@@ -487,8 +487,7 @@ class ASTHelpers_Test(unittest.TestCase):
         node = ast.parse('foo()', mode='eval')
         d = dict(ast.iter_fields(node.body))
         self.assertEqual(d.pop('func').id, 'foo')
-        self.assertEqual(d, {'keywords': [], 'kwargs': None,
+        self.assertEqual(d, {'keywords': [], 'args': []})
-                             'args': [], 'starargs': None})
 
     def test_iter_child_nodes(self):
         node = ast.parse("spam(23, 42, eggs='leek')", mode='eval')
@@ -604,8 +603,7 @@ class ASTValidatorTests(unittest.TestCase):
         self._check_arguments(fac, self.stmt)
 
     def test_classdef(self):
-        def cls(bases=None, keywords=None, starargs=None, kwargs=None,
+        def cls(bases=None, keywords=None, body=None, decorator_list=None):
-                body=None, decorator_list=None):
             if bases is None:
                 bases = []
             if keywords is None:
@@ -614,16 +612,12 @@ class ASTValidatorTests(unittest.TestCase):
                 body = [ast.Pass()]
             if decorator_list is None:
                 decorator_list = []
-            return ast.ClassDef("myclass", bases, keywords, starargs,
+            return ast.ClassDef("myclass", bases, keywords,
-                                kwargs, body, decorator_list)
+                                body, decorator_list)
         self.stmt(cls(bases=[ast.Name("x", ast.Store())]),
                   "must have Load context")
         self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]),
                   "must have Load context")
-        self.stmt(cls(starargs=ast.Name("x", ast.Store())),
-                  "must have Load context")
-        self.stmt(cls(kwargs=ast.Name("x", ast.Store())),
-                  "must have Load context")
         self.stmt(cls(body=[]), "empty body on ClassDef")
         self.stmt(cls(body=[None]), "None disallowed")
         self.stmt(cls(decorator_list=[ast.Name("x", ast.Store())]),
@@ -854,20 +848,12 @@ class ASTValidatorTests(unittest.TestCase):
         func = ast.Name("x", ast.Load())
         args = [ast.Name("y", ast.Load())]
         keywords = [ast.keyword("w", ast.Name("z", ast.Load()))]
-        stararg = ast.Name("p", ast.Load())
+        call = ast.Call(ast.Name("x", ast.Store()), args, keywords)
-        kwarg = ast.Name("q", ast.Load())
-        call = ast.Call(ast.Name("x", ast.Store()), args, keywords, stararg,
-                        kwarg)
         self.expr(call, "must have Load context")
-        call = ast.Call(func, [None], keywords, stararg, kwarg)
+        call = ast.Call(func, [None], keywords)
         self.expr(call, "None disallowed")
         bad_keywords = [ast.keyword("w", ast.Name("z", ast.Store()))]
-        call = ast.Call(func, args, bad_keywords, stararg, kwarg)
+        call = ast.Call(func, args, bad_keywords)
-        self.expr(call, "must have Load context")
-        call = ast.Call(func, args, keywords, ast.Name("z", ast.Store()), kwarg)
-        self.expr(call, "must have Load context")
-        call = ast.Call(func, args, keywords, stararg,
-                        ast.Name("w", ast.Store()))
         self.expr(call, "must have Load context")
 
     def test_num(self):
@@ -957,8 +943,8 @@ exec_results = [
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None), [], [], None, []), [('Pass', (1, 14))], [], None)]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None), []), [('Pass', (1, 17))], [], None)]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None), ('arg', (1, 9), 'b', None), ('arg', (1, 14), 'c', None), ('arg', (1, 22), 'd', None), ('arg', (1, 28), 'e', None)], ('arg', (1, 35), 'args', None), [('arg', (1, 41), 'f', None)], [('Num', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None), [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Pass', (1, 58))], [], None)]),
-('Module', [('ClassDef', (1, 0), 'C', [], [], None, None, [('Pass', (1, 8))], [])]),
+('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (1, 8))], [])]),
-('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], None, None, [('Pass', (1, 17))], [])]),
+('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], [('Pass', (1, 17))], [])]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
 ('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
 ('Module', [('Assign', (1, 0), [('Name', (1, 0), 'v', ('Store',))], ('Num', (1, 4), 1))]),
@@ -968,7 +954,7 @@ exec_results = [
 ('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]),
 ('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]),
 ('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]),
-('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]),
+('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], []), None)]),
 ('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])]),
 ('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])]),
 ('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]),
@@ -998,14 +984,14 @@ eval_results = [
 ('Expression', ('BinOp', (1, 0), ('Name', (1, 0), 'a', ('Load',)), ('Add',), ('Name', (1, 4), 'b', ('Load',)))),
 ('Expression', ('UnaryOp', (1, 0), ('Not',), ('Name', (1, 4), 'v', ('Load',)))),
 ('Expression', ('Lambda', (1, 0), ('arguments', [], None, [], [], None, []), ('NameConstant', (1, 7), None))),
-('Expression', ('Dict', (1, 0), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
+('Expression', ('Dict', (1, 2), [('Num', (1, 2), 1)], [('Num', (1, 4), 2)])),
 ('Expression', ('Dict', (1, 0), [], [])),
-('Expression', ('Set', (1, 0), [('NameConstant', (1, 1), None)])),
+('Expression', ('Set', (1, 1), [('NameConstant', (1, 1), None)])),
-('Expression', ('Dict', (1, 0), [('Num', (2, 6), 1)], [('Num', (4, 10), 2)])),
+('Expression', ('Dict', (2, 6), [('Num', (2, 6), 1)], [('Num', (4, 10), 2)])),
 ('Expression', ('ListComp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
 ('Expression', ('GeneratorExp', (1, 1), ('Name', (1, 1), 'a', ('Load',)), [('comprehension', ('Name', (1, 7), 'b', ('Store',)), ('Name', (1, 12), 'c', ('Load',)), [('Name', (1, 17), 'd', ('Load',))])])),
 ('Expression', ('Compare', (1, 0), ('Num', (1, 0), 1), [('Lt',), ('Lt',)], [('Num', (1, 4), 2), ('Num', (1, 8), 3)])),
-('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2)], [('keyword', 'c', ('Num', (1, 8), 3))], ('Name', (1, 11), 'd', ('Load',)), ('Name', (1, 15), 'e', ('Load',)))),
+('Expression', ('Call', (1, 0), ('Name', (1, 0), 'f', ('Load',)), [('Num', (1, 2), 1), ('Num', (1, 4), 2), ('Starred', (1, 10), ('Name', (1, 11), 'd', ('Load',)), ('Load',))], [('keyword', 'c', ('Num', (1, 8), 3)), ('keyword', None, ('Name', (1, 15), 'e', ('Load',)))])),
 ('Expression', ('Num', (1, 0), 10)),
 ('Expression', ('Str', (1, 0), 'string')),
 ('Expression', ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',))),
@@ -1016,6 +1002,6 @@ eval_results = [
 ('Expression', ('Tuple', (1, 0), [('Num', (1, 0), 1), ('Num', (1, 2), 2), ('Num', (1, 4), 3)], ('Load',))),
 ('Expression', ('Tuple', (1, 1), [('Num', (1, 1), 1), ('Num', (1, 3), 2), ('Num', (1, 5), 3)], ('Load',))),
 ('Expression', ('Tuple', (1, 0), [], ('Load',))),
-('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [], None, None)),
+('Expression', ('Call', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Attribute', (1, 0), ('Name', (1, 0), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8), ('Attribute', (1, 8), ('Name', (1, 8), 'a', ('Load',)), 'b', ('Load',)), ('Slice', ('Num', (1, 12), 1), ('Num', (1, 14), 2), None), ('Load',))], [])),
 ]
 main()

Lib/test/test_extcall.py

@@ -34,17 +34,37 @@ Argument list examples
     (1, 2, 3, 4, 5) {}
     >>> f(1, 2, 3, *[4, 5])
     (1, 2, 3, 4, 5) {}
+    >>> f(*[1, 2, 3], 4, 5)
+    (1, 2, 3, 4, 5) {}
     >>> f(1, 2, 3, *UserList([4, 5]))
     (1, 2, 3, 4, 5) {}
+    >>> f(1, 2, 3, *[4, 5], *[6, 7])
+    (1, 2, 3, 4, 5, 6, 7) {}
+    >>> f(1, *[2, 3], 4, *[5, 6], 7)
+    (1, 2, 3, 4, 5, 6, 7) {}
+    >>> f(*UserList([1, 2]), *UserList([3, 4]), 5, *UserList([6, 7]))
+    (1, 2, 3, 4, 5, 6, 7) {}
 
 Here we add keyword arguments
 
     >>> f(1, 2, 3, **{'a':4, 'b':5})
     (1, 2, 3) {'a': 4, 'b': 5}
+    >>> f(1, 2, **{'a': -1, 'b': 5}, **{'a': 4, 'c': 6})
+    Traceback (most recent call last):
+        ...
+    TypeError: f() got multiple values for keyword argument 'a'
+    >>> f(1, 2, **{'a': -1, 'b': 5}, a=4, c=6)
+    Traceback (most recent call last):
+        ...
+    TypeError: f() got multiple values for keyword argument 'a'
     >>> f(1, 2, 3, *[4, 5], **{'a':6, 'b':7})
     (1, 2, 3, 4, 5) {'a': 6, 'b': 7}
     >>> f(1, 2, 3, x=4, y=5, *(6, 7), **{'a':8, 'b': 9})
     (1, 2, 3, 6, 7) {'a': 8, 'b': 9, 'x': 4, 'y': 5}
+    >>> f(1, 2, 3, *[4, 5], **{'c': 8}, **{'a':6, 'b':7})
+    (1, 2, 3, 4, 5) {'a': 6, 'b': 7, 'c': 8}
+    >>> f(1, 2, 3, *(4, 5), x=6, y=7, **{'a':8, 'b': 9})
+    (1, 2, 3, 4, 5) {'a': 8, 'b': 9, 'x': 6, 'y': 7}
 
     >>> f(1, 2, 3, **UserDict(a=4, b=5))
     (1, 2, 3) {'a': 4, 'b': 5}
@@ -52,6 +72,8 @@ Here we add keyword arguments
     (1, 2, 3, 4, 5) {'a': 6, 'b': 7}
     >>> f(1, 2, 3, x=4, y=5, *(6, 7), **UserDict(a=8, b=9))
     (1, 2, 3, 6, 7) {'a': 8, 'b': 9, 'x': 4, 'y': 5}
+    >>> f(1, 2, 3, *(4, 5), x=6, y=7, **UserDict(a=8, b=9))
+    (1, 2, 3, 4, 5) {'a': 8, 'b': 9, 'x': 6, 'y': 7}
 
 Examples with invalid arguments (TypeErrors). We're also testing the function
 names in the exception messages.

Lib/test/test_grammar.py

@@ -296,8 +296,12 @@ class GrammarTests(unittest.TestCase):
             return args, kwargs
         self.assertEqual(f(1, x=2, *[3, 4], y=5), ((1, 3, 4),
                                                     {'x':2, 'y':5}))
-        self.assertRaises(SyntaxError, eval, "f(1, *(2,3), 4)")
+        self.assertEqual(f(1, *(2,3), 4), ((1, 2, 3, 4), {}))
         self.assertRaises(SyntaxError, eval, "f(1, x=2, *(3,4), x=5)")
+        self.assertEqual(f(**{'eggs':'scrambled', 'spam':'fried'}),
+                         ((), {'eggs':'scrambled', 'spam':'fried'}))
+        self.assertEqual(f(spam='fried', **{'eggs':'scrambled'}),
+                         ((), {'eggs':'scrambled', 'spam':'fried'}))
 
         # argument annotation tests
         def f(x) -> list: pass

Lib/test/test_parser.py

@@ -313,7 +313,12 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase):
                          "except Exception as e:\n"
                          "    raise ValueError from e\n")
 
+    def test_list_displays(self):
+        self.check_expr('[]')
+        self.check_expr('[*{2}, 3, *[4]]')
+
     def test_set_displays(self):
+        self.check_expr('{*{2}, 3, *[4]}')
         self.check_expr('{2}')
         self.check_expr('{2,}')
         self.check_expr('{2, 3}')
@@ -325,6 +330,13 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase):
         self.check_expr('{a:b,}')
         self.check_expr('{a:b, c:d}')
         self.check_expr('{a:b, c:d,}')
+        self.check_expr('{**{}}')
+        self.check_expr('{**{}, 3:4, **{5:6, 7:8}}')
+
+    def test_argument_unpacking(self):
+        self.check_expr('f(a, *b, *c, *d)')
+        self.check_expr('f(**a, **b)')
+        self.check_expr('f(2, *a, *b, **b, **c, **d)')
 
     def test_set_comprehensions(self):
         self.check_expr('{x for x in seq}')

Lib/test/test_syntax.py

@@ -141,6 +141,9 @@ From ast_for_call():
 >>> f(x for x in L, 1)
 Traceback (most recent call last):
 SyntaxError: Generator expression must be parenthesized if not sole argument
+>>> f(x for x in L, y for y in L)
+Traceback (most recent call last):
+SyntaxError: Generator expression must be parenthesized if not sole argument
 >>> f((x for x in L), 1)
 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
 
@@ -582,7 +585,18 @@ class SyntaxTestCase(unittest.TestCase):
                           subclass=IndentationError)
 
     def test_kwargs_last(self):
-        self._check_error("int(base=10, '2')", "non-keyword arg")
+        self._check_error("int(base=10, '2')",
+                          "positional argument follows keyword argument")
+
+    def test_kwargs_last2(self):
+        self._check_error("int(**{base: 10}, '2')",
+                          "positional argument follows "
+                          "keyword argument unpacking")
+
+    def test_kwargs_last3(self):
+        self._check_error("int(**{base: 10}, *['2'])",
+                          "iterable argument unpacking follows "
+                          "keyword argument unpacking")
 
 def test_main():
     support.run_unittest(SyntaxTestCase)

Lib/test/test_unpack_ex.py

@@ -71,8 +71,185 @@ Multiple targets
     >>> a == 0 and b == [1, 2, 3] and c == 4 and d == [0, 1, 2, 3] and e == 4
     True
 
+Assignment unpacking
+
+    >>> a, b, *c = range(5)
+    >>> a, b, c
+    (0, 1, [2, 3, 4])
+    >>> *a, b, c = a, b, *c
+    >>> a, b, c
+    ([0, 1, 2], 3, 4)
+
+Set display element unpacking
+
+    >>> a = [1, 2, 3]
+    >>> sorted({1, *a, 0, 4})
+    [0, 1, 2, 3, 4]
+
+    >>> {1, *1, 0, 4}
+    Traceback (most recent call last):
+      ...
+    TypeError: 'int' object is not iterable
+
+Dict display element unpacking
+
+    >>> kwds = {'z': 0, 'w': 12}
+    >>> sorted({'x': 1, 'y': 2, **kwds}.items())
+    [('w', 12), ('x', 1), ('y', 2), ('z', 0)]
+
+    >>> sorted({**{'x': 1}, 'y': 2, **{'z': 3}}.items())
+    [('x', 1), ('y', 2), ('z', 3)]
+
+    >>> sorted({**{'x': 1}, 'y': 2, **{'x': 3}}.items())
+    [('x', 3), ('y', 2)]
+
+    >>> sorted({**{'x': 1}, **{'x': 3}, 'x': 4}.items())
+    [('x', 4)]
+
+    >>> {**{}}
+    {}
+
+    >>> a = {}
+    >>> {**a}[0] = 1
+    >>> a
+    {}
+
+    >>> {**1}
+    Traceback (most recent call last):
+    ...
+    TypeError: 'int' object is not a mapping
+
+    >>> {**[]}
+    Traceback (most recent call last):
+    ...
+    TypeError: 'list' object is not a mapping
+
+    >>> len(eval("{" + ", ".join("**{{{}: {}}}".format(i, i)
+    ...                          for i in range(1000)) + "}"))
+    1000
+
+List comprehension element unpacking
+
+    >>> a, b, c = [0, 1, 2], 3, 4
+    >>> [*a, b, c]
+    [0, 1, 2, 3, 4]
+
+    >>> l = [a, (3, 4), {5}, {6: None}, (i for i in range(7, 10))]
+    >>> [*item for item in l]
+    Traceback (most recent call last):
+    ...
+    SyntaxError: iterable unpacking cannot be used in comprehension
+
+    >>> [*[0, 1] for i in range(10)]
+    Traceback (most recent call last):
+    ...
+    SyntaxError: iterable unpacking cannot be used in comprehension
+
+    >>> [*'a' for i in range(10)]
+    Traceback (most recent call last):
+    ...
+    SyntaxError: iterable unpacking cannot be used in comprehension
+
+    >>> [*[] for i in range(10)]
+    Traceback (most recent call last):
+    ...
+    SyntaxError: iterable unpacking cannot be used in comprehension
+
+Generator expression in function arguments
+
+    >>> list(*x for x in (range(5) for i in range(3)))
+    Traceback (most recent call last):
+    ...
+        list(*x for x in (range(5) for i in range(3)))
+                  ^
+    SyntaxError: invalid syntax
+
+    >>> dict(**x for x in [{1:2}])
+    Traceback (most recent call last):
+    ...
+        dict(**x for x in [{1:2}])
+                   ^
+    SyntaxError: invalid syntax
+
+Iterable argument unpacking
+
+    >>> print(*[1], *[2], 3)
+    1 2 3
+
+Make sure that they don't corrupt the passed-in dicts.
+
+    >>> def f(x, y):
+    ...     print(x, y)
+    ...
+    >>> original_dict = {'x': 1}
+    >>> f(**original_dict, y=2)
+    1 2
+    >>> original_dict
+    {'x': 1}
+
 Now for some failures
 
+Make sure the raised errors are right for keyword argument unpackings
+
+    >>> from collections.abc import MutableMapping
+    >>> class CrazyDict(MutableMapping):
+    ...     def __init__(self):
+    ...         self.d = {}
+    ...
+    ...     def __iter__(self):
+    ...         for x in self.d.__iter__():
+    ...             if x == 'c':
+    ...                 self.d['z'] = 10
+    ...             yield x
+    ...
+    ...     def __getitem__(self, k):
+    ...         return self.d[k]
+    ...
+    ...     def __len__(self):
+    ...         return len(self.d)
+    ...
+    ...     def __setitem__(self, k, v):
+    ...         self.d[k] = v
+    ...
+    ...     def __delitem__(self, k):
+    ...         del self.d[k]
+    ...
+    >>> d = CrazyDict()
+    >>> d.d = {chr(ord('a') + x): x for x in range(5)}
+    >>> e = {**d}
+    Traceback (most recent call last):
+    ...
+    RuntimeError: dictionary changed size during iteration
+
+    >>> d.d = {chr(ord('a') + x): x for x in range(5)}
+    >>> def f(**kwargs): print(kwargs)
+    >>> f(**d)
+    Traceback (most recent call last):
+    ...
+    RuntimeError: dictionary changed size during iteration
+
+Overridden parameters
+
+    >>> f(x=5, **{'x': 3}, y=2)
+    Traceback (most recent call last):
+      ...
+    TypeError: f() got multiple values for keyword argument 'x'
+
+    >>> f(**{'x': 3}, x=5, y=2)
+    Traceback (most recent call last):
+      ...
+    TypeError: f() got multiple values for keyword argument 'x'
+
+    >>> f(**{'x': 3}, **{'x': 5}, y=2)
+    Traceback (most recent call last):
+      ...
+    TypeError: f() got multiple values for keyword argument 'x'
+
+    >>> f(**{1: 3}, **{1: 5})
+    Traceback (most recent call last):
+      ...
+    TypeError: f() keywords must be strings
+
 Unpacking non-sequence
 
     >>> a, *b = 7
@@ -138,17 +315,17 @@ Now some general starred expressions (all fail).
     >>> *a # doctest:+ELLIPSIS
     Traceback (most recent call last):
       ...
-    SyntaxError: can use starred expression only as assignment target
+    SyntaxError: can't use starred expression here
 
     >>> *1 # doctest:+ELLIPSIS
     Traceback (most recent call last):
       ...
-    SyntaxError: can use starred expression only as assignment target
+    SyntaxError: can't use starred expression here
 
     >>> x = *a # doctest:+ELLIPSIS
     Traceback (most recent call last):
       ...
-    SyntaxError: can use starred expression only as assignment target
+    SyntaxError: can't use starred expression here
 
 Some size constraints (all fail.)
 

Misc/ACKS

@@ -493,6 +493,7 @@ Johannes Gijsbers
 Michael Gilfix
 Julian Gindi
 Yannick Gingras
+Neil Girdhar
 Matt Giuca
 Wim Glenn
 Michael Goderbauer

Misc/NEWS

@@ -10,6 +10,8 @@ Release date: 2015-04-24
 Core and Builtins
 -----------------
 
+- Issue #2292: PEP 448: Additional Unpacking Generalizations.
+
 - Issue #24096: Make warnings.warn_explicit more robust against mutation of the
   warnings.filters list.
 

Modules/parsermodule.c

@@ -23,6 +23,11 @@
  *  that lint detects are gone, but there are still warnings with
  *  Py_[X]DECREF() and Py_[X]INCREF() macros.  The lint annotations
  *  look like "NOTE(...)".
+ *
+ *  To debug parser errors like
+ *      "parser.ParserError: Expected node type 12, got 333."
+ *  decode symbol numbers using the automatically-generated files
+ *  Lib/symbol.h and Include/token.h.
  */
 
 #include "Python.h"                     /* general Python API             */
@@ -1087,30 +1092,56 @@ validate_terminal(node *terminal, int type, char *string)
     return (res);
 }
 
-
+/*  X (',' X) [','] */
-/*  X (',' X) [',']
- */
 static int
-validate_repeating_list(node *tree, int ntype, int (*vfunc)(node *),
+validate_repeating_list_variable(node *tree,
-                        const char *const name)
+                                 int list_node_type,
+                                 int (*validate_child_func_inc)(node *, int *),
+                                 int *pos,
+                                 const char *const list_node_type_name)
 {
     int nch = NCH(tree);
-    int res = (nch && validate_ntype(tree, ntype)
+    int res = (nch && validate_ntype(tree, list_node_type));
-               && vfunc(CHILD(tree, 0)));
 
-    if (!res && !PyErr_Occurred())
+    if (!res && !PyErr_Occurred()) {
-        (void) validate_numnodes(tree, 1, name);
+        /* Unconditionally raise. */
+        (void) validate_numnodes(tree, 1, list_node_type_name);
+    }
     else {
-        if (is_even(nch))
+        for ( ; res && *pos < nch; ) {
-            res = validate_comma(CHILD(tree, --nch));
+            res = validate_child_func_inc(tree, pos);
-        if (res && nch > 1) {
+            if (!res || *pos >= nch)
-            int pos = 1;
+                break;
-            for ( ; res && pos < nch; pos += 2)
+            res = validate_comma(CHILD(tree, (*pos)++));
-                res = (validate_comma(CHILD(tree, pos))
-                       && vfunc(CHILD(tree, pos + 1)));
         }
     }
-    return (res);
+    return res;
+}
+
+/*  X (',' X) [','] */
+static int
+validate_repeating_list(node *tree,
+                        int list_node_type,
+                        int (*validate_child_func)(node *),
+                        const char *const list_node_type_name)
+{
+    int nch = NCH(tree);
+    int res = (nch && validate_ntype(tree, list_node_type));
+    int pos = 0;
+
+    if (!res && !PyErr_Occurred()) {
+        /* Unconditionally raise. */
+        (void) validate_numnodes(tree, 1, list_node_type_name);
+    }
+    else {
+        for ( ; res && pos < nch; ) {
+            res = validate_child_func(CHILD(tree, pos++));
+            if (!res || pos >= nch)
+                break;
+            res = validate_comma(CHILD(tree, pos++));
+        }
+    }
+    return res;
 }
 
 
@@ -2451,9 +2482,9 @@ validate_atom(node *tree)
 
             if (res && (nch == 3)) {
                 if (TYPE(CHILD(tree, 1))==yield_expr)
-                        res = validate_yield_expr(CHILD(tree, 1));
+                    res = validate_yield_expr(CHILD(tree, 1));
                 else
-                        res = validate_testlist_comp(CHILD(tree, 1));
+                    res = validate_testlist_comp(CHILD(tree, 1));
             }
             break;
           case LSQB:
@@ -2493,39 +2524,28 @@ validate_atom(node *tree)
 
 
 /*  testlist_comp:
- *    test ( comp_for | (',' test)* [','] )
+ *    (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
  */
 static int
 validate_testlist_comp(node *tree)
 {
     int nch = NCH(tree);
-    int ok = nch;
+    int ok;
 
-    if (nch == 0)
+    if (nch == 0) {
         err_string("missing child nodes of testlist_comp");
-    else {
+        return 0;
-        ok = validate_test_or_star_expr(CHILD(tree, 0));
     }
 
-    /*
+    if (nch == 2 && TYPE(CHILD(tree, 1)) == comp_for) {
-     *  comp_for | (',' test)* [',']
+        ok = (validate_test(CHILD(tree, 0))
-     */
+                && validate_comp_for(CHILD(tree, 1)));
-    if (nch == 2 && TYPE(CHILD(tree, 1)) == comp_for)
+    }
-        ok = validate_comp_for(CHILD(tree, 1));
     else {
-        /*  (',' test)* [',']  */
+        ok = validate_repeating_list(tree,
-        int i = 1;
+                testlist_comp,
-        while (ok && nch - i >= 2) {
+                validate_test_or_star_expr,
-            ok = (validate_comma(CHILD(tree, i))
+                "testlist_comp");
-                  && validate_test_or_star_expr(CHILD(tree, i+1)));
-            i += 2;
-        }
-        if (ok && i == nch-1)
-            ok = validate_comma(CHILD(tree, i));
-        else if (i != nch) {
-            ok = 0;
-            err_string("illegal trailing nodes for testlist_comp");
-        }
     }
     return ok;
 }
@@ -2732,9 +2752,6 @@ validate_arglist(node *tree)
     }
     ok = 1;
     if (nch-i > 0) {
-        /*
-         * argument | '*' test [',' '**' test] | '**' test
-         */
         int sym = TYPE(CHILD(tree, i));
 
         if (sym == argument) {
@@ -2745,30 +2762,7 @@ validate_arglist(node *tree)
                 ok = 0;
             }
         }
-        else if (sym == STAR) {
+       else {
-            ok = validate_star(CHILD(tree, i));
-            if (ok && (nch-i == 2))
-                ok = validate_test(CHILD(tree, i+1));
-            else if (ok && (nch-i == 5))
-                ok = (validate_test(CHILD(tree, i+1))
-                      && validate_comma(CHILD(tree, i+2))
-                      && validate_doublestar(CHILD(tree, i+3))
-                      && validate_test(CHILD(tree, i+4)));
-            else {
-                err_string("illegal use of '*' in arglist");
-                ok = 0;
-            }
-        }
-        else if (sym == DOUBLESTAR) {
-            if (nch-i == 2)
-                ok = (validate_doublestar(CHILD(tree, i))
-                      && validate_test(CHILD(tree, i+1)));
-            else {
-                err_string("illegal use of '**' in arglist");
-                ok = 0;
-            }
-        }
-        else {
             err_string("illegal arglist specification");
             ok = 0;
         }
@@ -2778,9 +2772,10 @@ validate_arglist(node *tree)
 
 
 
-/*  argument:
+/*  argument: ( test [comp_for] |
- *
+ *              test '=' test |
- *  [test '='] test [comp_for]
+ *              '**' expr |
+ *              star_expr )
  */
 static int
 validate_argument(node *tree)
@@ -2788,14 +2783,24 @@ validate_argument(node *tree)
     int nch = NCH(tree);
     int res = (validate_ntype(tree, argument)
                && ((nch == 1) || (nch == 2) || (nch == 3)));
-    if (res)
-        res = validate_test(CHILD(tree, 0));
-    if (res && (nch == 2))
-        res = validate_comp_for(CHILD(tree, 1));
-    else if (res && (nch == 3))
-        res = (validate_equal(CHILD(tree, 1))
-               && validate_test(CHILD(tree, 2)));
 
+    if (res) {
+        if (TYPE(CHILD(tree, 0)) == DOUBLESTAR) {
+            res = validate_expr(CHILD(tree, 1));
+        }
+        else if (nch == 1) {
+            res = validate_test_or_star_expr(CHILD(tree, 0));
+        }
+        else if (nch == 2) {
+            res = (validate_test(CHILD(tree, 0))
+                    && validate_comp_for(CHILD(tree, 1)));
+        }
+        else if (res && (nch == 3)) {
+            res = (validate_test(CHILD(tree, 0))
+                    && validate_equal(CHILD(tree, 1))
+                    && validate_test(CHILD(tree, 2)));
+        }
+    }
     return (res);
 }
 
@@ -2948,11 +2953,44 @@ validate_exprlist(node *tree)
                                     validate_expr_or_star_expr, "exprlist"));
 }
 
+/* Incrementing validate functions returns nonzero iff success (like other
+ * validate functions, and advance *i by the length of the matched pattern. */
+
+/* test ':' test */
+static int
+validate_test_colon_test_inc(node *tree, int *i)
+{
+    return (validate_test(CHILD(tree, (*i)++))
+            && validate_colon(CHILD(tree, (*i)++))
+            && validate_test(CHILD(tree, (*i)++)));
+}
+
+/* test ':' test | '**' expr */
+static int
+validate_dict_element_inc(node *tree, int *i)
+{
+    int nch = NCH(tree);
+    int res = 0;
+    if (nch - *i >= 2) {
+        if (TYPE(CHILD(tree, *i+1)) == COLON) {
+            /* test ':' test */
+            res = validate_test_colon_test_inc(tree, i);
+        } else {
+            /* '**' expr */
+            res = (validate_doublestar(CHILD(tree, (*i)++))
+                    && validate_expr(CHILD(tree, (*i)++)));
+        }
+    }
+    return res;
+}
+
 /*
  *  dictorsetmaker:
  *
- *  (test ':' test (comp_for | (',' test ':' test)* [','])) |
+ *   ( ((test ':' test | '**' expr)
- *  (test (comp_for | (',' test)* [',']))
+ *      (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+ *     ((test | '*' test)
+ *      (comp_for | (',' (test | '*' test))* [','])) )
  */
 static int
 validate_dictorsetmaker(node *tree)
@@ -2966,65 +3004,44 @@ validate_dictorsetmaker(node *tree)
         return 0;
 
     if (nch - i < 1) {
+        /* Unconditionally raise. */
         (void) validate_numnodes(tree, 1, "dictorsetmaker");
         return 0;
     }
 
-    res = validate_test(CHILD(tree, i++));
+    if (nch - i >= 2
-    if (!res)
+        && ((TYPE(CHILD(tree, i+1)) == COLON) ||
-        return 0;
+            (TYPE(CHILD(tree, i)) == DOUBLESTAR))) {
-
-    if (nch - i >= 2 && TYPE(CHILD(tree, i)) == COLON) {
         /* Dictionary display or dictionary comprehension. */
-        res = (validate_colon(CHILD(tree, i++))
+        if (nch - i >= 4 && TYPE(CHILD(tree, i+3)) == comp_for) {
-               && validate_test(CHILD(tree, i++)));
-        if (!res)
-            return 0;
-
-        if (nch - i >= 1 && TYPE(CHILD(tree, i)) == comp_for) {
             /* Dictionary comprehension. */
-            res = validate_comp_for(CHILD(tree, i++));
+            res = (validate_test_colon_test_inc(tree, &i)
+                    && validate_comp_for(CHILD(tree, i++)));
             if (!res)
                 return 0;
-        }
+        } else {
-        else {
             /* Dictionary display. */
-            while (nch - i >= 4) {
+            return validate_repeating_list_variable(
-                res = (validate_comma(CHILD(tree, i++))
+                    tree,
-                       && validate_test(CHILD(tree, i++))
+                    dictorsetmaker,
-                       && validate_colon(CHILD(tree, i++))
+                    validate_dict_element_inc,
-                       && validate_test(CHILD(tree, i++)));
+                    &i,
-                if (!res)
+                    "dictorsetmaker");
-                    return 0;
-            }
-            if (nch - i == 1) {
-                res = validate_comma(CHILD(tree, i++));
-                if (!res)
-                    return 0;
-            }
         }
-    }
+    } else {
-    else {
         /* Set display or set comprehension. */
-        if (nch - i >= 1 && TYPE(CHILD(tree, i)) == comp_for) {
+        if (nch - i >= 2 && TYPE(CHILD(tree, i + 1)) == comp_for) {
             /* Set comprehension. */
-            res = validate_comp_for(CHILD(tree, i++));
+            res = (validate_test(CHILD(tree, i++))
+                   && validate_comp_for(CHILD(tree, i++)));
             if (!res)
                 return 0;
-        }
+        } else {
-        else {
             /* Set display. */
-            while (nch - i >= 2) {
+           return validate_repeating_list(tree,
-                res = (validate_comma(CHILD(tree, i++))
+                                          dictorsetmaker,
-                       && validate_test(CHILD(tree, i++)));
+                                          validate_test_or_star_expr,
-                if (!res)
+                                          "dictorsetmaker");
-                    return 0;
-            }
-            if (nch - i == 1) {
-                res = validate_comma(CHILD(tree, i++));
-                if (!res)
-                    return 0;
-            }
         }
     }
 

Objects/dictobject.c

@@ -3412,8 +3412,8 @@ dictviews_sub(PyObject* self, PyObject *other)
     return result;
 }
 
-static PyObject*
+PyObject*
-dictviews_and(PyObject* self, PyObject *other)
+_PyDictView_Intersect(PyObject* self, PyObject *other)
 {
     PyObject *result = PySet_New(self);
     PyObject *tmp;
@@ -3487,7 +3487,7 @@ static PyNumberMethods dictviews_as_number = {
     0,                                  /*nb_invert*/
     0,                                  /*nb_lshift*/
     0,                                  /*nb_rshift*/
-    (binaryfunc)dictviews_and,          /*nb_and*/
+    (binaryfunc)_PyDictView_Intersect,  /*nb_and*/
     (binaryfunc)dictviews_xor,          /*nb_xor*/
     (binaryfunc)dictviews_or,           /*nb_or*/
 };

Parser/Python.asdl

@@ -14,8 +14,6 @@ module Python
           | ClassDef(identifier name,
              expr* bases,
              keyword* keywords,
-             expr? starargs,
-             expr? kwargs,
              stmt* body,
              expr* decorator_list)
           | Return(expr? value)
@@ -64,8 +62,7 @@ module Python
          -- need sequences for compare to distinguish between
          -- x < 4 < 3 and (x < 4) < 3
          | Compare(expr left, cmpop* ops, expr* comparators)
-         | Call(expr func, expr* args, keyword* keywords,
+         | Call(expr func, expr* args, keyword* keywords)
-             expr? starargs, expr? kwargs)
          | Num(object n) -- a number as a PyObject.
          | Str(string s) -- need to specify raw, unicode, etc?
          | Bytes(bytes s)
@@ -109,8 +106,8 @@ module Python
     arg = (identifier arg, expr? annotation)
            attributes (int lineno, int col_offset)
 
-    -- keyword arguments supplied to call
+    -- keyword arguments supplied to call (NULL identifier for **kwargs)
-    keyword = (identifier arg, expr value)
+    keyword = (identifier? arg, expr value)
 
     -- import name with optional 'as' alias.
     alias = (identifier name, identifier? asname)

Python/Python-ast.c

@@ -48,14 +48,10 @@ static char *FunctionDef_fields[]={
 static PyTypeObject *ClassDef_type;
 _Py_IDENTIFIER(bases);
 _Py_IDENTIFIER(keywords);
-_Py_IDENTIFIER(starargs);
-_Py_IDENTIFIER(kwargs);
 static char *ClassDef_fields[]={
     "name",
     "bases",
     "keywords",
-    "starargs",
-    "kwargs",
     "body",
     "decorator_list",
 };
@@ -254,8 +250,6 @@ static char *Call_fields[]={
     "func",
     "args",
     "keywords",
-    "starargs",
-    "kwargs",
 };
 static PyTypeObject *Num_type;
 _Py_IDENTIFIER(n);
@@ -812,7 +806,7 @@ static int init_types(void)
     FunctionDef_type = make_type("FunctionDef", stmt_type, FunctionDef_fields,
                                  5);
     if (!FunctionDef_type) return 0;
-    ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 7);
+    ClassDef_type = make_type("ClassDef", stmt_type, ClassDef_fields, 5);
     if (!ClassDef_type) return 0;
     Return_type = make_type("Return", stmt_type, Return_fields, 1);
     if (!Return_type) return 0;
@@ -884,7 +878,7 @@ static int init_types(void)
     if (!YieldFrom_type) return 0;
     Compare_type = make_type("Compare", expr_type, Compare_fields, 3);
     if (!Compare_type) return 0;
-    Call_type = make_type("Call", expr_type, Call_fields, 5);
+    Call_type = make_type("Call", expr_type, Call_fields, 3);
     if (!Call_type) return 0;
     Num_type = make_type("Num", expr_type, Num_fields, 1);
     if (!Num_type) return 0;
@@ -1207,9 +1201,9 @@ FunctionDef(identifier name, arguments_ty args, asdl_seq * body, asdl_seq *
 }
 
 stmt_ty
-ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, expr_ty
+ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, asdl_seq *
-         starargs, expr_ty kwargs, asdl_seq * body, asdl_seq * decorator_list,
+         body, asdl_seq * decorator_list, int lineno, int col_offset, PyArena
-         int lineno, int col_offset, PyArena *arena)
+         *arena)
 {
     stmt_ty p;
     if (!name) {
@@ -1224,8 +1218,6 @@ ClassDef(identifier name, asdl_seq * bases, asdl_seq * keywords, expr_ty
     p->v.ClassDef.name = name;
     p->v.ClassDef.bases = bases;
     p->v.ClassDef.keywords = keywords;
-    p->v.ClassDef.starargs = starargs;
-    p->v.ClassDef.kwargs = kwargs;
     p->v.ClassDef.body = body;
     p->v.ClassDef.decorator_list = decorator_list;
     p->lineno = lineno;
@@ -1885,8 +1877,8 @@ Compare(expr_ty left, asdl_int_seq * ops, asdl_seq * comparators, int lineno,
 }
 
 expr_ty
-Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty starargs,
+Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, int lineno, int
-     expr_ty kwargs, int lineno, int col_offset, PyArena *arena)
+     col_offset, PyArena *arena)
 {
     expr_ty p;
     if (!func) {
@@ -1901,8 +1893,6 @@ Call(expr_ty func, asdl_seq * args, asdl_seq * keywords, expr_ty starargs,
     p->v.Call.func = func;
     p->v.Call.args = args;
     p->v.Call.keywords = keywords;
-    p->v.Call.starargs = starargs;
-    p->v.Call.kwargs = kwargs;
     p->lineno = lineno;
     p->col_offset = col_offset;
     return p;
@@ -2276,11 +2266,6 @@ keyword_ty
 keyword(identifier arg, expr_ty value, PyArena *arena)
 {
     keyword_ty p;
-    if (!arg) {
-        PyErr_SetString(PyExc_ValueError,
-                        "field arg is required for keyword");
-        return NULL;
-    }
     if (!value) {
         PyErr_SetString(PyExc_ValueError,
                         "field value is required for keyword");
@@ -2442,16 +2427,6 @@ ast2obj_stmt(void* _o)
         if (_PyObject_SetAttrId(result, &PyId_keywords, value) == -1)
             goto failed;
         Py_DECREF(value);
-        value = ast2obj_expr(o->v.ClassDef.starargs);
-        if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_starargs, value) == -1)
-            goto failed;
-        Py_DECREF(value);
-        value = ast2obj_expr(o->v.ClassDef.kwargs);
-        if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_kwargs, value) == -1)
-            goto failed;
-        Py_DECREF(value);
         value = ast2obj_list(o->v.ClassDef.body, ast2obj_stmt);
         if (!value) goto failed;
         if (_PyObject_SetAttrId(result, &PyId_body, value) == -1)
@@ -2964,16 +2939,6 @@ ast2obj_expr(void* _o)
         if (_PyObject_SetAttrId(result, &PyId_keywords, value) == -1)
             goto failed;
         Py_DECREF(value);
-        value = ast2obj_expr(o->v.Call.starargs);
-        if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_starargs, value) == -1)
-            goto failed;
-        Py_DECREF(value);
-        value = ast2obj_expr(o->v.Call.kwargs);
-        if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_kwargs, value) == -1)
-            goto failed;
-        Py_DECREF(value);
         break;
     case Num_kind:
         result = PyType_GenericNew(Num_type, NULL, NULL);
@@ -3875,8 +3840,6 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         identifier name;
         asdl_seq* bases;
         asdl_seq* keywords;
-        expr_ty starargs;
-        expr_ty kwargs;
         asdl_seq* body;
         asdl_seq* decorator_list;
 
@@ -3939,26 +3902,6 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             PyErr_SetString(PyExc_TypeError, "required field \"keywords\" missing from ClassDef");
             return 1;
         }
-        if (exists_not_none(obj, &PyId_starargs)) {
-            int res;
-            tmp = _PyObject_GetAttrId(obj, &PyId_starargs);
-            if (tmp == NULL) goto failed;
-            res = obj2ast_expr(tmp, &starargs, arena);
-            if (res != 0) goto failed;
-            Py_CLEAR(tmp);
-        } else {
-            starargs = NULL;
-        }
-        if (exists_not_none(obj, &PyId_kwargs)) {
-            int res;
-            tmp = _PyObject_GetAttrId(obj, &PyId_kwargs);
-            if (tmp == NULL) goto failed;
-            res = obj2ast_expr(tmp, &kwargs, arena);
-            if (res != 0) goto failed;
-            Py_CLEAR(tmp);
-        } else {
-            kwargs = NULL;
-        }
         if (_PyObject_HasAttrId(obj, &PyId_body)) {
             int res;
             Py_ssize_t len;
@@ -4007,8 +3950,8 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
             PyErr_SetString(PyExc_TypeError, "required field \"decorator_list\" missing from ClassDef");
             return 1;
         }
-        *out = ClassDef(name, bases, keywords, starargs, kwargs, body,
+        *out = ClassDef(name, bases, keywords, body, decorator_list, lineno,
-                        decorator_list, lineno, col_offset, arena);
+                        col_offset, arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -5506,8 +5449,6 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
         expr_ty func;
         asdl_seq* args;
         asdl_seq* keywords;
-        expr_ty starargs;
-        expr_ty kwargs;
 
         if (_PyObject_HasAttrId(obj, &PyId_func)) {
             int res;
@@ -5568,28 +5509,7 @@ obj2ast_expr(PyObject* obj, expr_ty* out, PyArena* arena)
             PyErr_SetString(PyExc_TypeError, "required field \"keywords\" missing from Call");
             return 1;
         }
-        if (exists_not_none(obj, &PyId_starargs)) {
+        *out = Call(func, args, keywords, lineno, col_offset, arena);
-            int res;
-            tmp = _PyObject_GetAttrId(obj, &PyId_starargs);
-            if (tmp == NULL) goto failed;
-            res = obj2ast_expr(tmp, &starargs, arena);
-            if (res != 0) goto failed;
-            Py_CLEAR(tmp);
-        } else {
-            starargs = NULL;
-        }
-        if (exists_not_none(obj, &PyId_kwargs)) {
-            int res;
-            tmp = _PyObject_GetAttrId(obj, &PyId_kwargs);
-            if (tmp == NULL) goto failed;
-            res = obj2ast_expr(tmp, &kwargs, arena);
-            if (res != 0) goto failed;
-            Py_CLEAR(tmp);
-        } else {
-            kwargs = NULL;
-        }
-        *out = Call(func, args, keywords, starargs, kwargs, lineno, col_offset,
-                    arena);
         if (*out == NULL) goto failed;
         return 0;
     }
@@ -6737,7 +6657,7 @@ obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena)
     identifier arg;
     expr_ty value;
 
-    if (_PyObject_HasAttrId(obj, &PyId_arg)) {
+    if (exists_not_none(obj, &PyId_arg)) {
         int res;
         tmp = _PyObject_GetAttrId(obj, &PyId_arg);
         if (tmp == NULL) goto failed;
@@ -6745,8 +6665,7 @@ obj2ast_keyword(PyObject* obj, keyword_ty* out, PyArena* arena)
         if (res != 0) goto failed;
         Py_CLEAR(tmp);
     } else {
-        PyErr_SetString(PyExc_TypeError, "required field \"arg\" missing from keyword");
+        arg = NULL;
-        return 1;
     }
     if (_PyObject_HasAttrId(obj, &PyId_value)) {
         int res;

Python/ast.c

@@ -235,9 +235,7 @@ validate_expr(expr_ty exp, expr_context_ty ctx)
     case Call_kind:
         return validate_expr(exp->v.Call.func, Load) &&
             validate_exprs(exp->v.Call.args, Load, 0) &&
-            validate_keywords(exp->v.Call.keywords) &&
+            validate_keywords(exp->v.Call.keywords);
-            (!exp->v.Call.starargs || validate_expr(exp->v.Call.starargs, Load)) &&
-            (!exp->v.Call.kwargs || validate_expr(exp->v.Call.kwargs, Load));
     case Num_kind: {
         PyObject *n = exp->v.Num.n;
         if (!PyLong_CheckExact(n) && !PyFloat_CheckExact(n) &&
@@ -322,9 +320,7 @@ validate_stmt(stmt_ty stmt)
         return validate_body(stmt->v.ClassDef.body, "ClassDef") &&
             validate_exprs(stmt->v.ClassDef.bases, Load, 0) &&
             validate_keywords(stmt->v.ClassDef.keywords) &&
-            validate_exprs(stmt->v.ClassDef.decorator_list, Load, 0) &&
+            validate_exprs(stmt->v.ClassDef.decorator_list, Load, 0);
-            (!stmt->v.ClassDef.starargs || validate_expr(stmt->v.ClassDef.starargs, Load)) &&
-            (!stmt->v.ClassDef.kwargs || validate_expr(stmt->v.ClassDef.kwargs, Load));
     case Return_kind:
         return !stmt->v.Return.value || validate_expr(stmt->v.Return.value, Load);
     case Delete_kind:
@@ -848,7 +844,8 @@ static const char* FORBIDDEN[] = {
 };
 
 static int
-forbidden_name(struct compiling *c, identifier name, const node *n, int full_checks)
+forbidden_name(struct compiling *c, identifier name, const node *n,
+               int full_checks)
 {
     assert(PyUnicode_Check(name));
     if (PyUnicode_CompareWithASCIIString(name, "__debug__") == 0) {
@@ -1445,7 +1442,7 @@ ast_for_decorator(struct compiling *c, const node *n)
         name_expr = NULL;
     }
     else if (NCH(n) == 5) { /* Call with no arguments */
-        d = Call(name_expr, NULL, NULL, NULL, NULL, LINENO(n),
+        d = Call(name_expr, NULL, NULL, LINENO(n),
                  n->n_col_offset, c->c_arena);
         if (!d)
             return NULL;
@@ -1735,16 +1732,22 @@ ast_for_comprehension(struct compiling *c, const node *n)
 static expr_ty
 ast_for_itercomp(struct compiling *c, const node *n, int type)
 {
-    /* testlist_comp: test ( comp_for | (',' test)* [','] )
+    /* testlist_comp: (test|star_expr)
-       argument: [test '='] test [comp_for]       # Really [keyword '='] test */
+     *                ( comp_for | (',' (test|star_expr))* [','] ) */
     expr_ty elt;
     asdl_seq *comps;
+    node *ch;
 
     assert(NCH(n) > 1);
 
-    elt = ast_for_expr(c, CHILD(n, 0));
+    ch = CHILD(n, 0);
+    elt = ast_for_expr(c, ch);
     if (!elt)
         return NULL;
+    if (elt->kind == Starred_kind) {
+        ast_error(c, ch, "iterable unpacking cannot be used in comprehension");
+        return NULL;
+    }
 
     comps = ast_for_comprehension(c, CHILD(n, 1));
     if (!comps)
@@ -1761,29 +1764,98 @@ ast_for_itercomp(struct compiling *c, const node *n, int type)
         return NULL;
 }
 
+/* Fills in the key, value pair corresponding to the dict element.  In case
+ * of an unpacking, key is NULL.  *i is advanced by the number of ast
+ * elements.  Iff successful, nonzero is returned.
+ */
+static int
+ast_for_dictelement(struct compiling *c, const node *n, int *i,
+                    expr_ty *key, expr_ty *value)
+{
+    expr_ty expression;
+    if (TYPE(CHILD(n, *i)) == DOUBLESTAR) {
+        assert(NCH(n) - *i >= 2);
+
+        expression = ast_for_expr(c, CHILD(n, *i + 1));
+        if (!expression)
+            return 0;
+        *key = NULL;
+        *value = expression;
+
+        *i += 2;
+    }
+    else {
+        assert(NCH(n) - *i >= 3);
+
+        expression = ast_for_expr(c, CHILD(n, *i));
+        if (!expression)
+            return 0;
+        *key = expression;
+
+        REQ(CHILD(n, *i + 1), COLON);
+
+        expression = ast_for_expr(c, CHILD(n, *i + 2));
+        if (!expression)
+            return 0;
+        *value = expression;
+
+        *i += 3;
+    }
+    return 1;
+}
+
 static expr_ty
 ast_for_dictcomp(struct compiling *c, const node *n)
 {
     expr_ty key, value;
     asdl_seq *comps;
+    int i = 0;
 
-    assert(NCH(n) > 3);
+    if (!ast_for_dictelement(c, n, &i, &key, &value))
-    REQ(CHILD(n, 1), COLON);
-
-    key = ast_for_expr(c, CHILD(n, 0));
-    if (!key)
-        return NULL;
-    value = ast_for_expr(c, CHILD(n, 2));
-    if (!value)
         return NULL;
+    assert(key);
+    assert(NCH(n) - i >= 1);
 
-    comps = ast_for_comprehension(c, CHILD(n, 3));
+    comps = ast_for_comprehension(c, CHILD(n, i));
     if (!comps)
         return NULL;
 
     return DictComp(key, value, comps, LINENO(n), n->n_col_offset, c->c_arena);
 }
 
+static expr_ty
+ast_for_dictdisplay(struct compiling *c, const node *n)
+{
+    int i;
+    int j;
+    int size;
+    asdl_seq *keys, *values;
+
+    size = (NCH(n) + 1) / 3; /* +1 in case no trailing comma */
+    keys = _Py_asdl_seq_new(size, c->c_arena);
+    if (!keys)
+        return NULL;
+
+    values = _Py_asdl_seq_new(size, c->c_arena);
+    if (!values)
+        return NULL;
+
+    j = 0;
+    for (i = 0; i < NCH(n); i++) {
+        expr_ty key, value;
+
+        if (!ast_for_dictelement(c, n, &i, &key, &value))
+            return NULL;
+        asdl_seq_SET(keys, j, key);
+        asdl_seq_SET(values, j, value);
+
+        j++;
+    }
+    keys->size = j;
+    values->size = j;
+    return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena);
+}
+
 static expr_ty
 ast_for_genexp(struct compiling *c, const node *n)
 {
@@ -1805,6 +1877,27 @@ ast_for_setcomp(struct compiling *c, const node *n)
     return ast_for_itercomp(c, n, COMP_SETCOMP);
 }
 
+static expr_ty
+ast_for_setdisplay(struct compiling *c, const node *n)
+{
+    int i;
+    int size;
+    asdl_seq *elts;
+
+    assert(TYPE(n) == (dictorsetmaker));
+    size = (NCH(n) + 1) / 2; /* +1 in case no trailing comma */
+    elts = _Py_asdl_seq_new(size, c->c_arena);
+    if (!elts)
+        return NULL;
+    for (i = 0; i < NCH(n); i += 2) {
+        expr_ty expression;
+        expression = ast_for_expr(c, CHILD(n, i));
+        if (!expression)
+            return NULL;
+        asdl_seq_SET(elts, i / 2, expression);
+    }
+    return Set(elts, LINENO(n), n->n_col_offset, c->c_arena);
+}
 
 static expr_ty
 ast_for_atom(struct compiling *c, const node *n)
@@ -1915,62 +2008,42 @@ ast_for_atom(struct compiling *c, const node *n)
         else
             return ast_for_listcomp(c, ch);
     case LBRACE: {
-        /* dictorsetmaker: test ':' test (',' test ':' test)* [','] |
+        /* dictorsetmaker: ( ((test ':' test | '**' test)
-         *                 test (gen_for | (',' test)* [','])  */
+         *                    (comp_for | (',' (test ':' test | '**' test))* [','])) |
-        int i, size;
+         *                   ((test | '*' test)
-        asdl_seq *keys, *values;
+         *                    (comp_for | (',' (test | '*' test))* [','])) ) */
-
         ch = CHILD(n, 1);
         if (TYPE(ch) == RBRACE) {
-            /* it's an empty dict */
+            /* It's an empty dict. */
             return Dict(NULL, NULL, LINENO(n), n->n_col_offset, c->c_arena);
-        } else if (NCH(ch) == 1 || TYPE(CHILD(ch, 1)) == COMMA) {
+        }
-            /* it's a simple set */
+        else {
-            asdl_seq *elts;
+            int is_dict = (TYPE(CHILD(ch, 0)) == DOUBLESTAR);
-            size = (NCH(ch) + 1) / 2; /* +1 in case no trailing comma */
+            if (NCH(ch) == 1 ||
-            elts = _Py_asdl_seq_new(size, c->c_arena);
+                    (NCH(ch) > 1 &&
-            if (!elts)
+                     TYPE(CHILD(ch, 1)) == COMMA)) {
-                return NULL;
+                /* It's a set display. */
-            for (i = 0; i < NCH(ch); i += 2) {
+                return ast_for_setdisplay(c, ch);
-                expr_ty expression;
-                expression = ast_for_expr(c, CHILD(ch, i));
-                if (!expression)
-                    return NULL;
-                asdl_seq_SET(elts, i / 2, expression);
             }
-            return Set(elts, LINENO(n), n->n_col_offset, c->c_arena);
+            else if (NCH(ch) > 1 &&
-        } else if (TYPE(CHILD(ch, 1)) == comp_for) {
+                    TYPE(CHILD(ch, 1)) == comp_for) {
-            /* it's a set comprehension */
+                /* It's a set comprehension. */
-            return ast_for_setcomp(c, ch);
+                return ast_for_setcomp(c, ch);
-        } else if (NCH(ch) > 3 && TYPE(CHILD(ch, 3)) == comp_for) {
+            }
-            return ast_for_dictcomp(c, ch);
+            else if (NCH(ch) > 3 - is_dict &&
-        } else {
+                    TYPE(CHILD(ch, 3 - is_dict)) == comp_for) {
-            /* it's a dict */
+                /* It's a dictionary comprehension. */
-            size = (NCH(ch) + 1) / 4; /* +1 in case no trailing comma */
+                if (is_dict) {
-            keys = _Py_asdl_seq_new(size, c->c_arena);
+                    ast_error(c, n, "dict unpacking cannot be used in "
-            if (!keys)
+                            "dict comprehension");
-                return NULL;
+                    return NULL;
-
+                }
-            values = _Py_asdl_seq_new(size, c->c_arena);
+                return ast_for_dictcomp(c, ch);
-            if (!values)
+            }
-                return NULL;
+            else {
-
+                /* It's a dictionary display. */
-            for (i = 0; i < NCH(ch); i += 4) {
+                return ast_for_dictdisplay(c, ch);
-                expr_ty expression;
-
-                expression = ast_for_expr(c, CHILD(ch, i));
-                if (!expression)
-                    return NULL;
-
-                asdl_seq_SET(keys, i / 4, expression);
-
-                expression = ast_for_expr(c, CHILD(ch, i + 2));
-                if (!expression)
-                    return NULL;
-
-                asdl_seq_SET(values, i / 4, expression);
             }
-            return Dict(keys, values, LINENO(n), n->n_col_offset, c->c_arena);
         }
     }
     default:
@@ -2106,7 +2179,7 @@ ast_for_trailer(struct compiling *c, const node *n, expr_ty left_expr)
     REQ(n, trailer);
     if (TYPE(CHILD(n, 0)) == LPAR) {
         if (NCH(n) == 2)
-            return Call(left_expr, NULL, NULL, NULL, NULL, LINENO(n),
+            return Call(left_expr, NULL, NULL, LINENO(n),
                         n->n_col_offset, c->c_arena);
         else
             return ast_for_call(c, CHILD(n, 1), left_expr);
@@ -2415,15 +2488,14 @@ static expr_ty
 ast_for_call(struct compiling *c, const node *n, expr_ty func)
 {
     /*
-      arglist: (argument ',')* (argument [',']| '*' test [',' '**' test]
+      arglist: argument (',' argument)*  [',']
-               | '**' test)
+      argument: ( test [comp_for] | '*' test | test '=' test | '**' test )
-      argument: [test '='] (test) [comp_for]        # Really [keyword '='] test
     */
 
     int i, nargs, nkeywords, ngens;
+    int ndoublestars;
     asdl_seq *args;
     asdl_seq *keywords;
-    expr_ty vararg = NULL, kwarg = NULL;
 
     REQ(n, arglist);
 
@@ -2437,7 +2509,10 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
                 nargs++;
             else if (TYPE(CHILD(ch, 1)) == comp_for)
                 ngens++;
+            else if (TYPE(CHILD(ch, 0)) == STAR)
+                nargs++;
             else
+                /* TYPE(CHILD(ch, 0)) == DOUBLESTAR or keyword argument */
                 nkeywords++;
         }
     }
@@ -2458,41 +2533,82 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
     keywords = _Py_asdl_seq_new(nkeywords, c->c_arena);
     if (!keywords)
         return NULL;
-    nargs = 0;
+
-    nkeywords = 0;
+    nargs = 0;  /* positional arguments + iterable argument unpackings */
+    nkeywords = 0;  /* keyword arguments + keyword argument unpackings */
+    ndoublestars = 0;  /* just keyword argument unpackings */
     for (i = 0; i < NCH(n); i++) {
         node *ch = CHILD(n, i);
         if (TYPE(ch) == argument) {
             expr_ty e;
+            node *chch = CHILD(ch, 0);
             if (NCH(ch) == 1) {
-                if (nkeywords) {
+                if (TYPE(chch) == star_expr) {
-                    ast_error(c, CHILD(ch, 0),
+                    /* an iterable argument unpacking */
-                              "non-keyword arg after keyword arg");
+                    expr_ty starred;
-                    return NULL;
+                    if (ndoublestars) {
+                        ast_error(c, chch,
+                                "iterable argument unpacking follows "
+                                "keyword argument unpacking");
+                        return NULL;
+                    }
+                    e = ast_for_expr(c, CHILD(chch, 1));
+                    if (!e)
+                        return NULL;
+                    starred = Starred(e, Load, LINENO(chch),
+                            chch->n_col_offset,
+                            c->c_arena);
+                    if (!starred)
+                        return NULL;
+                    asdl_seq_SET(args, nargs++, starred);
                 }
-                if (vararg) {
+                else {
-                    ast_error(c, CHILD(ch, 0),
+                    /* a positional argument */
-                              "only named arguments may follow *expression");
+                    if (nkeywords) {
-                    return NULL;
+                        if (ndoublestars) {
+                            ast_error(c, chch,
+                                    "positional argument follows "
+                                    "keyword argument unpacking");
+                        }
+                        else {
+                            ast_error(c, chch,
+                                    "positional argument follows "
+                                    "keyword argument");
+                        }
+                        return NULL;
+                    }
+                    e = ast_for_expr(c, chch);
+                    if (!e)
+                        return NULL;
+                    asdl_seq_SET(args, nargs++, e);
                 }
-                e = ast_for_expr(c, CHILD(ch, 0));
+            }
+            else if (TYPE(chch) == DOUBLESTAR) {
+                /* a keyword argument unpacking */
+                keyword_ty kw;
+                i++;
+                e = ast_for_expr(c, CHILD(ch, 1));
                 if (!e)
                     return NULL;
-                asdl_seq_SET(args, nargs++, e);
+                kw = keyword(NULL, e, c->c_arena);
+                asdl_seq_SET(keywords, nkeywords++, kw);
+                ndoublestars++;
             }
             else if (TYPE(CHILD(ch, 1)) == comp_for) {
+                /* the lone generator expression */
                 e = ast_for_genexp(c, ch);
                 if (!e)
                     return NULL;
                 asdl_seq_SET(args, nargs++, e);
             }
             else {
+                /* a keyword argument */
                 keyword_ty kw;
                 identifier key, tmp;
                 int k;
 
-                /* CHILD(ch, 0) is test, but must be an identifier? */
+                /* chch is test, but must be an identifier? */
-                e = ast_for_expr(c, CHILD(ch, 0));
+                e = ast_for_expr(c, chch);
                 if (!e)
                     return NULL;
                 /* f(lambda x: x[0] = 3) ends up getting parsed with
@@ -2501,19 +2617,24 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
                  * then is very confusing.
                  */
                 if (e->kind == Lambda_kind) {
-                    ast_error(c, CHILD(ch, 0), "lambda cannot contain assignment");
+                    ast_error(c, chch,
+                            "lambda cannot contain assignment");
                     return NULL;
-                } else if (e->kind != Name_kind) {
+                }
-                    ast_error(c, CHILD(ch, 0), "keyword can't be an expression");
+                else if (e->kind != Name_kind) {
+                    ast_error(c, chch,
+                            "keyword can't be an expression");
                     return NULL;
-                } else if (forbidden_name(c, e->v.Name.id, ch, 1)) {
+                }
+                else if (forbidden_name(c, e->v.Name.id, ch, 1)) {
                     return NULL;
                 }
                 key = e->v.Name.id;
                 for (k = 0; k < nkeywords; k++) {
                     tmp = ((keyword_ty)asdl_seq_GET(keywords, k))->arg;
-                    if (!PyUnicode_Compare(tmp, key)) {
+                    if (tmp && !PyUnicode_Compare(tmp, key)) {
-                        ast_error(c, CHILD(ch, 0), "keyword argument repeated");
+                        ast_error(c, chch,
+                                "keyword argument repeated");
                         return NULL;
                     }
                 }
@@ -2526,21 +2647,9 @@ ast_for_call(struct compiling *c, const node *n, expr_ty func)
                 asdl_seq_SET(keywords, nkeywords++, kw);
             }
         }
-        else if (TYPE(ch) == STAR) {
-            vararg = ast_for_expr(c, CHILD(n, i+1));
-            if (!vararg)
-                return NULL;
-            i++;
-        }
-        else if (TYPE(ch) == DOUBLESTAR) {
-            kwarg = ast_for_expr(c, CHILD(n, i+1));
-            if (!kwarg)
-                return NULL;
-            i++;
-        }
     }
 
-    return Call(func, args, keywords, vararg, kwarg, func->lineno, func->col_offset, c->c_arena);
+    return Call(func, args, keywords, func->lineno, func->col_offset, c->c_arena);
 }
 
 static expr_ty
@@ -3520,8 +3629,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
             return NULL;
         if (forbidden_name(c, classname, CHILD(n, 3), 0))
             return NULL;
-        return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq,
+        return ClassDef(classname, NULL, NULL, s, decorator_seq, LINENO(n),
-                        LINENO(n), n->n_col_offset, c->c_arena);
+                        n->n_col_offset, c->c_arena);
     }
 
     if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */
@@ -3533,8 +3642,8 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
             return NULL;
         if (forbidden_name(c, classname, CHILD(n, 3), 0))
             return NULL;
-        return ClassDef(classname, NULL, NULL, NULL, NULL, s, decorator_seq,
+        return ClassDef(classname, NULL, NULL, s, decorator_seq, LINENO(n),
-                        LINENO(n), n->n_col_offset, c->c_arena);
+                        n->n_col_offset, c->c_arena);
     }
 
     /* class NAME '(' arglist ')' ':' suite */
@@ -3559,8 +3668,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
     if (forbidden_name(c, classname, CHILD(n, 1), 0))
         return NULL;
 
-    return ClassDef(classname, call->v.Call.args, call->v.Call.keywords,
+    return ClassDef(classname, call->v.Call.args, call->v.Call.keywords, s,
-                    call->v.Call.starargs, call->v.Call.kwargs, s,
                     decorator_seq, LINENO(n), n->n_col_offset, c->c_arena);
 }
 

Python/ceval.c

@@ -12,8 +12,10 @@
 #include "Python.h"
 
 #include "code.h"
+#include "dictobject.h"
 #include "frameobject.h"
 #include "opcode.h"
+#include "setobject.h"
 #include "structmember.h"
 
 #include <ctype.h>
@@ -2379,6 +2381,43 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
+        TARGET_WITH_IMPL(BUILD_TUPLE_UNPACK, _build_list_unpack)
+        TARGET(BUILD_LIST_UNPACK)
+        _build_list_unpack: {
+            int convert_to_tuple = opcode == BUILD_TUPLE_UNPACK;
+            int i;
+            PyObject *sum = PyList_New(0);
+            PyObject *return_value;
+            if (sum == NULL)
+                goto error;
+
+            for (i = oparg; i > 0; i--) {
+                PyObject *none_val;
+
+                none_val = _PyList_Extend((PyListObject *)sum, PEEK(i));
+                if (none_val == NULL) {
+                    Py_DECREF(sum);
+                    goto error;
+                }
+                Py_DECREF(none_val);
+            }
+
+            if (convert_to_tuple) {
+                return_value = PyList_AsTuple(sum);
+                Py_DECREF(sum);
+                if (return_value == NULL)
+                    goto error;
+            }
+            else {
+                return_value = sum;
+            }
+
+            while (oparg--)
+                Py_DECREF(POP());
+            PUSH(return_value);
+            DISPATCH();
+        }
+
         TARGET(BUILD_SET) {
             PyObject *set = PySet_New(NULL);
             int err = 0;
@@ -2398,14 +2437,127 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             DISPATCH();
         }
 
+        TARGET(BUILD_SET_UNPACK) {
+            int i;
+            PyObject *sum = PySet_New(NULL);
+            if (sum == NULL)
+                goto error;
+
+            for (i = oparg; i > 0; i--) {
+                if (_PySet_Update(sum, PEEK(i)) < 0) {
+                    Py_DECREF(sum);
+                    goto error;
+                }
+            }
+
+            while (oparg--)
+                Py_DECREF(POP());
+            PUSH(sum);
+            DISPATCH();
+        }
+
         TARGET(BUILD_MAP) {
             PyObject *map = _PyDict_NewPresized((Py_ssize_t)oparg);
             if (map == NULL)
                 goto error;
+            while (--oparg >= 0) {
+                int err;
+                PyObject *key = TOP();
+                PyObject *value = SECOND();
+                STACKADJ(-2);
+                err = PyDict_SetItem(map, key, value);
+                Py_DECREF(value);
+                Py_DECREF(key);
+                if (err != 0) {
+                    Py_DECREF(map);
+                    goto error;
+                }
+            }
             PUSH(map);
             DISPATCH();
         }
 
+        TARGET_WITH_IMPL(BUILD_MAP_UNPACK_WITH_CALL, _build_map_unpack)
+        TARGET(BUILD_MAP_UNPACK)
+        _build_map_unpack: {
+            int with_call = opcode == BUILD_MAP_UNPACK_WITH_CALL;
+            int num_maps;
+            int function_location;
+            int i;
+            PyObject *sum = PyDict_New();
+            if (sum == NULL)
+                goto error;
+            if (with_call) {
+                num_maps = oparg & 0xff;
+                function_location = (oparg>>8) & 0xff;
+            }
+            else {
+                num_maps = oparg;
+            }
+
+            for (i = num_maps; i > 0; i--) {
+                PyObject *arg = PEEK(i);
+                if (with_call) {
+                    PyObject *intersection = _PyDictView_Intersect(sum, arg);
+
+                    if (intersection == NULL) {
+                        if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
+                            PyObject *func = (
+                                    PEEK(function_location + num_maps));
+                            PyErr_Format(PyExc_TypeError,
+                                    "%.200s%.200s argument after ** "
+                                    "must be a mapping, not %.200s",
+                                    PyEval_GetFuncName(func),
+                                    PyEval_GetFuncDesc(func),
+                                    arg->ob_type->tp_name);
+                        }
+                        Py_DECREF(sum);
+                        goto error;
+                    }
+
+                    if (PySet_GET_SIZE(intersection)) {
+                        Py_ssize_t idx = 0;
+                        PyObject *key;
+                        PyObject *func = PEEK(function_location + num_maps);
+                        Py_hash_t hash;
+                        _PySet_NextEntry(intersection, &idx, &key, &hash);
+                        if (!PyUnicode_Check(key)) {
+                            PyErr_Format(PyExc_TypeError,
+                                    "%.200s%.200s keywords must be strings",
+                                    PyEval_GetFuncName(func),
+                                    PyEval_GetFuncDesc(func));
+                        } else {
+                            PyErr_Format(PyExc_TypeError,
+                                    "%.200s%.200s got multiple "
+                                    "values for keyword argument '%U'",
+                                    PyEval_GetFuncName(func),
+                                    PyEval_GetFuncDesc(func),
+                                    key);
+                        }
+                        Py_DECREF(intersection);
+                        Py_DECREF(sum);
+                        goto error;
+                    }
+                    Py_DECREF(intersection);
+                }
+
+                if (PyDict_Update(sum, arg) < 0) {
+                    if (PyErr_ExceptionMatches(PyExc_AttributeError)) {
+                        PyErr_Format(PyExc_TypeError,
+                                "'%.200s' object is not a mapping",
+                                arg->ob_type->tp_name);
+                    }
+                    Py_DECREF(sum);
+                    goto error;
+                }
+            }
+
+            while (num_maps--)
+                Py_DECREF(POP());
+            PUSH(sum);
+            DISPATCH();
+        }
+
         TARGET(STORE_MAP) {
             PyObject *key = TOP();
             PyObject *value = SECOND();
@@ -3050,6 +3202,7 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
             goto dispatch_opcode;
         }
 
+
 #if USE_COMPUTED_GOTOS
         _unknown_opcode:
 #endif
@@ -4557,6 +4710,12 @@ ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk)
             kwdict = d;
         }
     }
+    if (nk > 0) {
+        kwdict = update_keyword_args(kwdict, nk, pp_stack, func);
+        if (kwdict == NULL)
+            goto ext_call_fail;
+    }
+
     if (flags & CALL_FLAG_VAR) {
         stararg = EXT_POP(*pp_stack);
         if (!PyTuple_Check(stararg)) {
@@ -4578,11 +4737,6 @@ ext_do_call(PyObject *func, PyObject ***pp_stack, int flags, int na, int nk)
         }
         nstar = PyTuple_GET_SIZE(stararg);
     }
-    if (nk > 0) {
-        kwdict = update_keyword_args(kwdict, nk, pp_stack, func);
-        if (kwdict == NULL)
-            goto ext_call_fail;
-    }
     callargs = update_star_args(na, nstar, stararg, pp_stack);
     if (callargs == NULL)
         goto ext_call_fail;

Python/compile.c

@@ -195,9 +195,7 @@ static int expr_constant(struct compiler *, expr_ty);
 static int compiler_with(struct compiler *, stmt_ty, int);
 static int compiler_call_helper(struct compiler *c, Py_ssize_t n,
                                 asdl_seq *args,
-                                asdl_seq *keywords,
+                                asdl_seq *keywords);
-                                expr_ty starargs,
-                                expr_ty kwargs);
 static int compiler_try_except(struct compiler *, stmt_ty);
 static int compiler_set_qualname(struct compiler *);
 
@@ -672,7 +670,8 @@ compiler_set_qualname(struct compiler *c)
         PyObject *mangled, *capsule;
 
         capsule = PyList_GET_ITEM(c->c_stack, stack_size - 1);
-        parent = (struct compiler_unit *)PyCapsule_GetPointer(capsule, COMPILER_CAPSULE_NAME_COMPILER_UNIT);
+        parent = (struct compiler_unit *)PyCapsule_GetPointer(
+                capsule, COMPILER_CAPSULE_NAME_COMPILER_UNIT);
         assert(parent);
 
         if (u->u_scope_type == COMPILER_SCOPE_FUNCTION || u->u_scope_type == COMPILER_SCOPE_CLASS) {
@@ -973,6 +972,13 @@ PyCompile_OpcodeStackEffect(int opcode, int oparg)
         case BUILD_LIST:
         case BUILD_SET:
             return 1-oparg;
+        case BUILD_LIST_UNPACK:
+        case BUILD_TUPLE_UNPACK:
+        case BUILD_SET_UNPACK:
+        case BUILD_MAP_UNPACK:
+            return 1 - oparg;
+        case BUILD_MAP_UNPACK_WITH_CALL:
+            return 1 - (oparg & 0xFF);
         case BUILD_MAP:
             return 1;
         case LOAD_ATTR:
@@ -1821,9 +1827,7 @@ compiler_class(struct compiler *c, stmt_ty s)
     /* 5. generate the rest of the code for the call */
     if (!compiler_call_helper(c, 2,
                               s->v.ClassDef.bases,
-                              s->v.ClassDef.keywords,
+                              s->v.ClassDef.keywords))
-                              s->v.ClassDef.starargs,
-                              s->v.ClassDef.kwargs))
         return 0;
 
     /* 6. apply decorators */
@@ -2870,68 +2874,146 @@ compiler_boolop(struct compiler *c, expr_ty e)
     return 1;
 }
 
+static int
+starunpack_helper(struct compiler *c, asdl_seq *elts,
+                  int single_op, int inner_op, int outer_op)
+{
+    Py_ssize_t n = asdl_seq_LEN(elts);
+    Py_ssize_t i, nsubitems = 0, nseen = 0;
+    for (i = 0; i < n; i++) {
+        expr_ty elt = asdl_seq_GET(elts, i);
+        if (elt->kind == Starred_kind) {
+            if (nseen) {
+                ADDOP_I(c, inner_op, nseen);
+                nseen = 0;
+                nsubitems++;
+            }
+            VISIT(c, expr, elt->v.Starred.value);
+            nsubitems++;
+        }
+        else {
+            VISIT(c, expr, elt);
+            nseen++;
+        }
+    }
+    if (nsubitems) {
+        if (nseen) {
+            ADDOP_I(c, inner_op, nseen);
+            nsubitems++;
+        }
+        ADDOP_I(c, outer_op, nsubitems);
+    }
+    else
+        ADDOP_I(c, single_op, nseen);
+    return 1;
+}
+
+static int
+assignment_helper(struct compiler *c, asdl_seq *elts)
+{
+    Py_ssize_t n = asdl_seq_LEN(elts);
+    Py_ssize_t i;
+    int seen_star = 0;
+    for (i = 0; i < n; i++) {
+        expr_ty elt = asdl_seq_GET(elts, i);
+        if (elt->kind == Starred_kind && !seen_star) {
+            if ((i >= (1 << 8)) ||
+                (n-i-1 >= (INT_MAX >> 8)))
+                return compiler_error(c,
+                    "too many expressions in "
+                    "star-unpacking assignment");
+            ADDOP_I(c, UNPACK_EX, (i + ((n-i-1) << 8)));
+            seen_star = 1;
+            asdl_seq_SET(elts, i, elt->v.Starred.value);
+        }
+        else if (elt->kind == Starred_kind) {
+            return compiler_error(c,
+                "two starred expressions in assignment");
+        }
+    }
+    if (!seen_star) {
+        ADDOP_I(c, UNPACK_SEQUENCE, n);
+    }
+    VISIT_SEQ(c, expr, elts);
+    return 1;
+}
+
 static int
 compiler_list(struct compiler *c, expr_ty e)
 {
-    Py_ssize_t n = asdl_seq_LEN(e->v.List.elts);
+    asdl_seq *elts = e->v.List.elts;
     if (e->v.List.ctx == Store) {
-        int i, seen_star = 0;
+        return assignment_helper(c, elts);
-        for (i = 0; i < n; i++) {
-            expr_ty elt = asdl_seq_GET(e->v.List.elts, i);
-            if (elt->kind == Starred_kind && !seen_star) {
-                if ((i >= (1 << 8)) ||
-                    (n-i-1 >= (INT_MAX >> 8)))
-                    return compiler_error(c,
-                        "too many expressions in "
-                        "star-unpacking assignment");
-                ADDOP_I(c, UNPACK_EX, (i + ((n-i-1) << 8)));
-                seen_star = 1;
-                asdl_seq_SET(e->v.List.elts, i, elt->v.Starred.value);
-            } else if (elt->kind == Starred_kind) {
-                return compiler_error(c,
-                    "two starred expressions in assignment");
-            }
-        }
-        if (!seen_star) {
-            ADDOP_I(c, UNPACK_SEQUENCE, n);
-        }
     }
-    VISIT_SEQ(c, expr, e->v.List.elts);
+    else if (e->v.List.ctx == Load) {
-    if (e->v.List.ctx == Load) {
+        return starunpack_helper(c, elts,
-        ADDOP_I(c, BUILD_LIST, n);
+                                 BUILD_LIST, BUILD_TUPLE, BUILD_LIST_UNPACK);
     }
+    else
+        VISIT_SEQ(c, expr, elts);
     return 1;
 }
 
 static int
 compiler_tuple(struct compiler *c, expr_ty e)
 {
-    Py_ssize_t n = asdl_seq_LEN(e->v.Tuple.elts);
+    asdl_seq *elts = e->v.Tuple.elts;
     if (e->v.Tuple.ctx == Store) {
-        int i, seen_star = 0;
+        return assignment_helper(c, elts);
-        for (i = 0; i < n; i++) {
+    }
-            expr_ty elt = asdl_seq_GET(e->v.Tuple.elts, i);
+    else if (e->v.Tuple.ctx == Load) {
-            if (elt->kind == Starred_kind && !seen_star) {
+        return starunpack_helper(c, elts,
-                if ((i >= (1 << 8)) ||
+                                 BUILD_TUPLE, BUILD_TUPLE, BUILD_TUPLE_UNPACK);
-                    (n-i-1 >= (INT_MAX >> 8)))
+    }
-                    return compiler_error(c,
+    else
-                        "too many expressions in "
+        VISIT_SEQ(c, expr, elts);
-                        "star-unpacking assignment");
+    return 1;
-                ADDOP_I(c, UNPACK_EX, (i + ((n-i-1) << 8)));
+}
-                seen_star = 1;
+
-                asdl_seq_SET(e->v.Tuple.elts, i, elt->v.Starred.value);
+static int
-            } else if (elt->kind == Starred_kind) {
+compiler_set(struct compiler *c, expr_ty e)
-                return compiler_error(c,
+{
-                    "two starred expressions in assignment");
+    return starunpack_helper(c, e->v.Set.elts, BUILD_SET,
-            }
+                             BUILD_SET, BUILD_SET_UNPACK);
+}
+
+static int
+compiler_dict(struct compiler *c, expr_ty e)
+{
+    Py_ssize_t i, n, containers, elements;
+    int is_unpacking = 0;
+    n = asdl_seq_LEN(e->v.Dict.values);
+    containers = 0;
+    elements = 0;
+    for (i = 0; i < n; i++) {
+        is_unpacking = (expr_ty)asdl_seq_GET(e->v.Dict.keys, i) == NULL;
+        if (elements == 0xFFFF || (elements && is_unpacking)) {
+            ADDOP_I(c, BUILD_MAP, elements);
+            containers++;
+            elements = 0;
         }
-        if (!seen_star) {
+        if (is_unpacking) {
-            ADDOP_I(c, UNPACK_SEQUENCE, n);
+            VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
+            containers++;
+        }
+        else {
+            VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
+            VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Dict.keys, i));
+            elements++;
         }
     }
-    VISIT_SEQ(c, expr, e->v.Tuple.elts);
+    if (elements || containers == 0) {
-    if (e->v.Tuple.ctx == Load) {
+        ADDOP_I(c, BUILD_MAP, elements);
-        ADDOP_I(c, BUILD_TUPLE, n);
+        containers++;
+    }
+    /* If there is more than one dict, they need to be merged into a new
+     * dict.  If there is one dict and it's an unpacking, then it needs
+     * to be copied into a new dict." */
+    while (containers > 1 || is_unpacking) {
+        int oparg = containers < 255 ? containers : 255;
+        ADDOP_I(c, BUILD_MAP_UNPACK, oparg);
+        containers -= (oparg - 1);
+        is_unpacking = 0;
     }
     return 1;
 }
@@ -2987,9 +3069,7 @@ compiler_call(struct compiler *c, expr_ty e)
     VISIT(c, expr, e->v.Call.func);
     return compiler_call_helper(c, 0,
                                 e->v.Call.args,
-                                e->v.Call.keywords,
+                                e->v.Call.keywords);
-                                e->v.Call.starargs,
-                                e->v.Call.kwargs);
 }
 
 /* shared code between compiler_call and compiler_class */
@@ -2997,26 +3077,102 @@ static int
 compiler_call_helper(struct compiler *c,
                      Py_ssize_t n, /* Args already pushed */
                      asdl_seq *args,
-                     asdl_seq *keywords,
+                     asdl_seq *keywords)
-                     expr_ty starargs,
-                     expr_ty kwargs)
 {
     int code = 0;
+    Py_ssize_t nelts, i, nseen, nkw;
 
-    n += asdl_seq_LEN(args);
+    /* the number of tuples and dictionaries on the stack */
-    VISIT_SEQ(c, expr, args);
+    Py_ssize_t nsubargs = 0, nsubkwargs = 0;
-    if (keywords) {
+
-        VISIT_SEQ(c, keyword, keywords);
+    nkw = 0;
-        n |= asdl_seq_LEN(keywords) << 8;
+    nseen = 0;  /* the number of positional arguments on the stack */
+    nelts = asdl_seq_LEN(args);
+    for (i = 0; i < nelts; i++) {
+        expr_ty elt = asdl_seq_GET(args, i);
+        if (elt->kind == Starred_kind) {
+            /* A star-arg. If we've seen positional arguments,
+               pack the positional arguments into a
+               tuple. */
+            if (nseen) {
+                ADDOP_I(c, BUILD_TUPLE, nseen);
+                nseen = 0;
+                nsubargs++;
+            }
+            VISIT(c, expr, elt->v.Starred.value);
+            nsubargs++;
+        }
+        else if (nsubargs) {
+            /* We've seen star-args already, so we
+               count towards items-to-pack-into-tuple. */
+            VISIT(c, expr, elt);
+            nseen++;
+        }
+        else {
+            /* Positional arguments before star-arguments
+               are left on the stack. */
+            VISIT(c, expr, elt);
+            n++;
+        }
     }
-    if (starargs) {
+    if (nseen) {
-        VISIT(c, expr, starargs);
+        /* Pack up any trailing positional arguments. */
+        ADDOP_I(c, BUILD_TUPLE, nseen);
+        nsubargs++;
+    }
+    if (nsubargs) {
         code |= 1;
+        if (nsubargs > 1) {
+            /* If we ended up with more than one stararg, we need
+               to concatenate them into a single sequence. */
+            ADDOP_I(c, BUILD_LIST_UNPACK, nsubargs);
+        }
     }
-    if (kwargs) {
+
-        VISIT(c, expr, kwargs);
+    /* Same dance again for keyword arguments */
+    nseen = 0;  /* the number of keyword arguments on the stack following */
+    nelts = asdl_seq_LEN(keywords);
+    for (i = 0; i < nelts; i++) {
+        keyword_ty kw = asdl_seq_GET(keywords, i);
+        if (kw->arg == NULL) {
+            /* A keyword argument unpacking. */
+            if (nseen) {
+                ADDOP_I(c, BUILD_MAP, nseen);
+                nseen = 0;
+                nsubkwargs++;
+            }
+            VISIT(c, expr, kw->value);
+            nsubkwargs++;
+        }
+        else if (nsubkwargs) {
+            /* A keyword argument and we already have a dict. */
+            VISIT(c, expr, kw->value);
+            ADDOP_O(c, LOAD_CONST, kw->arg, consts);
+            nseen++;
+        }
+        else {
+            /* keyword argument */
+            VISIT(c, keyword, kw)
+            nkw++;
+        }
+    }
+    if (nseen) {
+        /* Pack up any trailing keyword arguments. */
+        ADDOP_I(c, BUILD_MAP, nseen);
+        nsubkwargs++;
+    }
+    if (nsubkwargs) {
         code |= 2;
+        if (nsubkwargs > 1) {
+            /* Pack it all up */
+            int function_pos = n + (code & 1) + nkw + 1;
+            ADDOP_I(c, BUILD_MAP_UNPACK_WITH_CALL, nsubkwargs | (function_pos << 8));
+        }
     }
+    assert(n < 1<<8);
+    assert(nkw < 1<<24);
+    n |= nkw << 8;
+
     switch (code) {
     case 0:
         ADDOP_I(c, CALL_FUNCTION, n);
@@ -3141,8 +3297,9 @@ compiler_comprehension_generator(struct compiler *c,
 }
 
 static int
-compiler_comprehension(struct compiler *c, expr_ty e, int type, identifier name,
+compiler_comprehension(struct compiler *c, expr_ty e, int type,
-                       asdl_seq *generators, expr_ty elt, expr_ty val)
+                       identifier name, asdl_seq *generators, expr_ty elt,
+                       expr_ty val)
 {
     PyCodeObject *co = NULL;
     expr_ty outermost_iter;
@@ -3399,8 +3556,6 @@ compiler_with(struct compiler *c, stmt_ty s, int pos)
 static int
 compiler_visit_expr(struct compiler *c, expr_ty e)
 {
-    Py_ssize_t i, n;
-
     /* If expr e has a different line number than the last expr/stmt,
        set a new line number for the next instruction.
     */
@@ -3427,23 +3582,9 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
     case IfExp_kind:
         return compiler_ifexp(c, e);
     case Dict_kind:
-        n = asdl_seq_LEN(e->v.Dict.values);
+        return compiler_dict(c, e);
-        /* BUILD_MAP parameter is only used to preallocate the dictionary,
-           it doesn't need to be exact */
-        ADDOP_I(c, BUILD_MAP, (n>0xFFFF ? 0xFFFF : n));
-        for (i = 0; i < n; i++) {
-            VISIT(c, expr,
-                (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
-            VISIT(c, expr,
-                (expr_ty)asdl_seq_GET(e->v.Dict.keys, i));
-            ADDOP(c, STORE_MAP);
-        }
-        break;
     case Set_kind:
-        n = asdl_seq_LEN(e->v.Set.elts);
+        return compiler_set(c, e);
-        VISIT_SEQ(c, expr, e->v.Set.elts);
-        ADDOP_I(c, BUILD_SET, n);
-        break;
     case GeneratorExp_kind:
         return compiler_genexp(c, e);
     case ListComp_kind:
@@ -3554,7 +3695,7 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
                 "starred assignment target must be in a list or tuple");
         default:
             return compiler_error(c,
-                "can use starred expression only as assignment target");
+                "can't use starred expression here");
         }
         break;
     case Name_kind:

Python/graminit.c

@@ -1569,60 +1569,78 @@ static state states_71[3] = {
     {2, arcs_71_1},
     {2, arcs_71_2},
 };
-static arc arcs_72_0[1] = {
+static arc arcs_72_0[3] = {
     {24, 1},
+    {32, 2},
+    {48, 3},
 };
 static arc arcs_72_1[4] = {
-    {25, 2},
+    {25, 4},
-    {159, 3},
+    {159, 5},
-    {30, 4},
+    {30, 6},
     {0, 1},
 };
 static arc arcs_72_2[1] = {
-    {24, 5},
+    {104, 7},
 };
-static arc arcs_72_3[1] = {
+static arc arcs_72_3[3] = {
+    {159, 5},
+    {30, 6},
     {0, 3},
 };
-static arc arcs_72_4[2] = {
+static arc arcs_72_4[1] = {
-    {24, 6},
+    {24, 7},
-    {0, 4},
 };
-static arc arcs_72_5[3] = {
+static arc arcs_72_5[1] = {
-    {159, 3},
-    {30, 7},
     {0, 5},
 };
-static arc arcs_72_6[2] = {
+static arc arcs_72_6[3] = {
-    {30, 4},
+    {24, 8},
+    {48, 8},
     {0, 6},
 };
-static arc arcs_72_7[2] = {
+static arc arcs_72_7[3] = {
-    {24, 8},
+    {159, 5},
+    {30, 9},
     {0, 7},
 };
-static arc arcs_72_8[1] = {
+static arc arcs_72_8[2] = {
-    {25, 9},
+    {30, 6},
+    {0, 8},
 };
-static arc arcs_72_9[1] = {
+static arc arcs_72_9[3] = {
     {24, 10},
+    {32, 11},
+    {0, 9},
 };
-static arc arcs_72_10[2] = {
+static arc arcs_72_10[1] = {
-    {30, 7},
+    {25, 12},
-    {0, 10},
 };
-static state states_72[11] = {
+static arc arcs_72_11[1] = {
-    {1, arcs_72_0},
+    {104, 13},
+};
+static arc arcs_72_12[1] = {
+    {24, 13},
+};
+static arc arcs_72_13[2] = {
+    {30, 9},
+    {0, 13},
+};
+static state states_72[14] = {
+    {3, arcs_72_0},
     {4, arcs_72_1},
     {1, arcs_72_2},
-    {1, arcs_72_3},
+    {3, arcs_72_3},
-    {2, arcs_72_4},
+    {1, arcs_72_4},
-    {3, arcs_72_5},
+    {1, arcs_72_5},
-    {2, arcs_72_6},
+    {3, arcs_72_6},
-    {2, arcs_72_7},
+    {3, arcs_72_7},
-    {1, arcs_72_8},
+    {2, arcs_72_8},
-    {1, arcs_72_9},
+    {3, arcs_72_9},
-    {2, arcs_72_10},
+    {1, arcs_72_10},
+    {1, arcs_72_11},
+    {1, arcs_72_12},
+    {2, arcs_72_13},
 };
 static arc arcs_73_0[1] = {
     {163, 1},
@@ -1660,67 +1678,47 @@ static state states_73[8] = {
     {1, arcs_73_6},
     {1, arcs_73_7},
 };
-static arc arcs_74_0[3] = {
+static arc arcs_74_0[1] = {
     {164, 1},
-    {31, 2},
-    {32, 3},
 };
 static arc arcs_74_1[2] = {
-    {30, 4},
+    {30, 2},
     {0, 1},
 };
-static arc arcs_74_2[1] = {
+static arc arcs_74_2[2] = {
-    {24, 5},
-};
-static arc arcs_74_3[1] = {
-    {24, 6},
-};
-static arc arcs_74_4[4] = {
     {164, 1},
-    {31, 2},
+    {0, 2},
-    {32, 3},
-    {0, 4},
 };
-static arc arcs_74_5[2] = {
+static state states_74[3] = {
-    {30, 7},
+    {1, arcs_74_0},
-    {0, 5},
-};
-static arc arcs_74_6[1] = {
-    {0, 6},
-};
-static arc arcs_74_7[2] = {
-    {164, 5},
-    {32, 3},
-};
-static state states_74[8] = {
-    {3, arcs_74_0},
     {2, arcs_74_1},
-    {1, arcs_74_2},
+    {2, arcs_74_2},
-    {1, arcs_74_3},
-    {4, arcs_74_4},
-    {2, arcs_74_5},
-    {1, arcs_74_6},
-    {2, arcs_74_7},
 };
-static arc arcs_75_0[1] = {
+static arc arcs_75_0[3] = {
     {24, 1},
+    {32, 2},
+    {48, 3},
 };
 static arc arcs_75_1[3] = {
-    {159, 2},
+    {159, 3},
-    {29, 3},
+    {29, 4},
     {0, 1},
 };
 static arc arcs_75_2[1] = {
-    {0, 2},
+    {104, 3},
 };
 static arc arcs_75_3[1] = {
-    {24, 2},
+    {0, 3},
 };
-static state states_75[4] = {
+static arc arcs_75_4[1] = {
-    {1, arcs_75_0},
+    {24, 3},
+};
+static state states_75[5] = {
+    {3, arcs_75_0},
     {3, arcs_75_1},
     {1, arcs_75_2},
     {1, arcs_75_3},
+    {1, arcs_75_4},
 };
 static arc arcs_76_0[2] = {
     {159, 1},
@@ -1964,14 +1962,14 @@ static dfa dfas[82] = {
      "\000\040\040\200\000\000\000\000\000\000\001\000\000\000\000\000\000\014\241\174\000\000"},
     {327, "testlist", 0, 3, states_71,
      "\000\040\040\000\000\000\000\000\000\000\001\000\000\000\041\000\000\014\241\174\000\000"},
-    {328, "dictorsetmaker", 0, 11, states_72,
+    {328, "dictorsetmaker", 0, 14, states_72,
-     "\000\040\040\000\000\000\000\000\000\000\001\000\000\000\041\000\000\014\241\174\000\000"},
+     "\000\040\040\200\001\000\000\000\000\000\001\000\000\000\041\000\000\014\241\174\000\000"},
     {329, "classdef", 0, 8, states_73,
      "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\010\000"},
-    {330, "arglist", 0, 8, states_74,
+    {330, "arglist", 0, 3, states_74,
+     "\000\040\040\200\001\000\000\000\000\000\001\000\000\000\041\000\000\014\241\174\000\000"},
+    {331, "argument", 0, 5, states_75,
      "\000\040\040\200\001\000\000\000\000\000\001\000\000\000\041\000\000\014\241\174\000\000"},
-    {331, "argument", 0, 4, states_75,
-     "\000\040\040\000\000\000\000\000\000\000\001\000\000\000\041\000\000\014\241\174\000\000"},
     {332, "comp_iter", 0, 2, states_76,
      "\000\000\000\000\000\000\000\000\000\000\000\040\002\000\000\000\000\000\000\000\000\000"},
     {333, "comp_for", 0, 6, states_77,

Python/opcode_targets.h

@@ -148,11 +148,11 @@ static void *opcode_targets[256] = {
     &&TARGET_SET_ADD,
     &&TARGET_MAP_ADD,
     &&TARGET_LOAD_CLASSDEREF,
-    &&_unknown_opcode,
+    &&TARGET_BUILD_LIST_UNPACK,
-    &&_unknown_opcode,
+    &&TARGET_BUILD_MAP_UNPACK,
-    &&_unknown_opcode,
+    &&TARGET_BUILD_MAP_UNPACK_WITH_CALL,
-    &&_unknown_opcode,
+    &&TARGET_BUILD_TUPLE_UNPACK,
-    &&_unknown_opcode,
+    &&TARGET_BUILD_SET_UNPACK,
     &&_unknown_opcode,
     &&_unknown_opcode,
     &&_unknown_opcode,

Python/symtable.c

@@ -1083,13 +1083,13 @@ error:
     } \
 }
 
-#define VISIT_KWONLYDEFAULTS(ST, KW_DEFAULTS) { \
+#define VISIT_SEQ_WITH_NULL(ST, TYPE, SEQ) {     \
     int i = 0; \
-    asdl_seq *seq = (KW_DEFAULTS); /* avoid variable capture */ \
+    asdl_seq *seq = (SEQ); /* avoid variable capture */ \
     for (i = 0; i < asdl_seq_LEN(seq); i++) { \
-        expr_ty elt = (expr_ty)asdl_seq_GET(seq, i); \
+        TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, i); \
         if (!elt) continue; /* can be NULL */ \
-        if (!symtable_visit_expr((ST), elt)) \
+        if (!symtable_visit_ ## TYPE((ST), elt)) \
             VISIT_QUIT((ST), 0);             \
     } \
 }
@@ -1146,8 +1146,7 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
         if (s->v.FunctionDef.args->defaults)
             VISIT_SEQ(st, expr, s->v.FunctionDef.args->defaults);
         if (s->v.FunctionDef.args->kw_defaults)
-            VISIT_KWONLYDEFAULTS(st,
+            VISIT_SEQ_WITH_NULL(st, expr, s->v.FunctionDef.args->kw_defaults);
-                               s->v.FunctionDef.args->kw_defaults);
         if (!symtable_visit_annotations(st, s))
             VISIT_QUIT(st, 0);
         if (s->v.FunctionDef.decorator_list)
@@ -1167,10 +1166,6 @@ symtable_visit_stmt(struct symtable *st, stmt_ty s)
             VISIT_QUIT(st, 0);
         VISIT_SEQ(st, expr, s->v.ClassDef.bases);
         VISIT_SEQ(st, keyword, s->v.ClassDef.keywords);
-        if (s->v.ClassDef.starargs)
-            VISIT(st, expr, s->v.ClassDef.starargs);
-        if (s->v.ClassDef.kwargs)
-            VISIT(st, expr, s->v.ClassDef.kwargs);
         if (s->v.ClassDef.decorator_list)
             VISIT_SEQ(st, expr, s->v.ClassDef.decorator_list);
         if (!symtable_enter_block(st, s->v.ClassDef.name, ClassBlock,
@@ -1349,8 +1344,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
         if (e->v.Lambda.args->defaults)
             VISIT_SEQ(st, expr, e->v.Lambda.args->defaults);
         if (e->v.Lambda.args->kw_defaults)
-            VISIT_KWONLYDEFAULTS(st,
+            VISIT_SEQ_WITH_NULL(st, expr, e->v.Lambda.args->kw_defaults);
-                                 e->v.Lambda.args->kw_defaults);
         if (!symtable_enter_block(st, lambda,
                                   FunctionBlock, (void *)e, e->lineno,
                                   e->col_offset))
@@ -1367,7 +1361,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
         VISIT(st, expr, e->v.IfExp.orelse);
         break;
     case Dict_kind:
-        VISIT_SEQ(st, expr, e->v.Dict.keys);
+        VISIT_SEQ_WITH_NULL(st, expr, e->v.Dict.keys);
         VISIT_SEQ(st, expr, e->v.Dict.values);
         break;
     case Set_kind:
@@ -1405,11 +1399,7 @@ symtable_visit_expr(struct symtable *st, expr_ty e)
     case Call_kind:
         VISIT(st, expr, e->v.Call.func);
         VISIT_SEQ(st, expr, e->v.Call.args);
-        VISIT_SEQ(st, keyword, e->v.Call.keywords);
+        VISIT_SEQ_WITH_NULL(st, keyword, e->v.Call.keywords);
-        if (e->v.Call.starargs)
-            VISIT(st, expr, e->v.Call.starargs);
-        if (e->v.Call.kwargs)
-            VISIT(st, expr, e->v.Call.kwargs);
         break;
     case Num_kind:
     case Str_kind:

Tools/parser/unparse.py

@@ -211,16 +211,6 @@ class Unparser:
             if comma: self.write(", ")
             else: comma = True
             self.dispatch(e)
-        if t.starargs:
-            if comma: self.write(", ")
-            else: comma = True
-            self.write("*")
-            self.dispatch(t.starargs)
-        if t.kwargs:
-            if comma: self.write(", ")
-            else: comma = True
-            self.write("**")
-            self.dispatch(t.kwargs)
         self.write(")")
 
         self.enter()
@@ -450,16 +440,6 @@ class Unparser:
             if comma: self.write(", ")
             else: comma = True
             self.dispatch(e)
-        if t.starargs:
-            if comma: self.write(", ")
-            else: comma = True
-            self.write("*")
-            self.dispatch(t.starargs)
-        if t.kwargs:
-            if comma: self.write(", ")
-            else: comma = True
-            self.write("**")
-            self.dispatch(t.kwargs)
         self.write(")")
 
     def _Subscript(self, t):
@@ -543,8 +523,11 @@ class Unparser:
                 self.dispatch(t.kwarg.annotation)
 
     def _keyword(self, t):
-        self.write(t.arg)
+        if t.arg is None:
-        self.write("=")
+            self.write("**")
+        else:
+            self.write(t.arg)
+            self.write("=")
         self.dispatch(t.value)
 
     def _Lambda(self, t):