Replace LALRPOP parser with hand-written parser (#10036)

(Supersedes #9152, authored by @LaBatata101)

## Summary

This PR replaces the current parser generated from LALRPOP to a
hand-written recursive descent parser.

It also updates the grammar for [PEP
646](https://peps.python.org/pep-0646/) so that the parser outputs the
correct AST. For example, in `data[*x]`, the index expression is now a
tuple with a single starred expression instead of just a starred
expression.

Beyond the performance improvements, the parser is also error resilient
and can provide better error messages. The behavior as seen by any
downstream tools isn't changed. That is, the linter and formatter can
still assume that the parser will _stop_ at the first syntax error. This
will be updated in the following months.

For more details about the change here, refer to the PR corresponding to
the individual commits and the release blog post.

## Test Plan

Write _lots_ and _lots_ of tests for both valid and invalid syntax and
verify the output.

## Acknowledgements

- @MichaReiser for reviewing 100+ parser PRs and continuously providing
guidance throughout the project
- @LaBatata101 for initiating the transition to a hand-written parser in
#9152
- @addisoncrump for implementing the fuzzer which helped
[catch](https://github.com/astral-sh/ruff/pull/10903)
[a](https://github.com/astral-sh/ruff/pull/10910)
[lot](https://github.com/astral-sh/ruff/pull/10966)
[of](https://github.com/astral-sh/ruff/pull/10896)
[bugs](https://github.com/astral-sh/ruff/pull/10877)

---------

Co-authored-by: Victor Hugo Gomes <labatata101@linuxmail.org>
Co-authored-by: Micha Reiser <micha@reiser.io>

crates/ruff_python_parser/resources/.editorconfig

@@ -0,0 +1,6 @@
+# Check http://editorconfig.org for more information
+# This is the main config file for this project:
+root = true
+
+[*.py]
+insert_final_newline = false

crates/ruff_python_parser/resources/inline/err/ann_assign_stmt_invalid_annotation.py

@@ -0,0 +1,4 @@
+x: *int = 1
+x: yield a = 1
+x: yield from b = 1
+x: y := int = 1

crates/ruff_python_parser/resources/inline/err/ann_assign_stmt_invalid_target.py

@@ -0,0 +1,10 @@
+"abc": str = "def"
+call(): str = "no"
+*x: int = 1, 2
+# Tuple assignment
+x,: int = 1
+x, y: int = 1, 2
+(x, y): int = 1, 2
+# List assignment
+[x]: int = 1
+[x, y]: int = 1, 2

crates/ruff_python_parser/resources/inline/err/ann_assign_stmt_invalid_value.py

@@ -0,0 +1,3 @@
+x: Any = *a and b
+x: Any = x := 1
+x: list = [x, *a | b, *a or b]

crates/ruff_python_parser/resources/inline/err/ann_assign_stmt_missing_rhs.py

@@ -0,0 +1 @@
+x: int =

crates/ruff_python_parser/resources/inline/err/ann_assign_stmt_type_alias_annotation.py

@@ -0,0 +1,2 @@
+a: type X = int
+lambda: type X = int

crates/ruff_python_parser/resources/inline/err/assert_empty_msg.py

@@ -0,0 +1 @@
+assert x,

crates/ruff_python_parser/resources/inline/err/assert_empty_test.py

@@ -0,0 +1 @@
+assert

crates/ruff_python_parser/resources/inline/err/assert_invalid_msg_expr.py

@@ -0,0 +1,4 @@
+assert False, *x
+assert False, assert x
+assert False, yield x
+assert False, x := 1

crates/ruff_python_parser/resources/inline/err/assert_invalid_test_expr.py

@@ -0,0 +1,4 @@
+assert *x
+assert assert x
+assert yield x
+assert x := 1

crates/ruff_python_parser/resources/inline/err/assign_stmt_invalid_target.py

@@ -0,0 +1,4 @@
+1 = 1
+x = 1 = 2
+x = 1 = y = 2 = z
+["a", "b"] = ["a", "b"]

crates/ruff_python_parser/resources/inline/err/assign_stmt_invalid_value_expr.py

@@ -0,0 +1,5 @@
+x = *a and b
+x = *yield x
+x = *yield from x
+x = *lambda x: x
+x = x := 1

crates/ruff_python_parser/resources/inline/err/assign_stmt_keyword_target.py

@@ -0,0 +1,4 @@
+a = pass = c
+a + b
+a = b = pass = c
+a + b

crates/ruff_python_parser/resources/inline/err/assign_stmt_missing_rhs.py

@@ -0,0 +1,6 @@
+x =
+1 + 1
+x = y =
+2 + 2
+x = = y
+3 + 3

crates/ruff_python_parser/resources/inline/err/async_unexpected_token.py

@@ -0,0 +1,8 @@
+async class Foo: ...
+async while test: ...
+async x = 1
+async async def foo(): ...
+# TODO(dhruvmanila): Here, `match` is actually a Name token because
+# of the soft keyword # transformer
+async match test:
+    case _: ...

crates/ruff_python_parser/resources/inline/err/aug_assign_stmt_invalid_target.py

@@ -0,0 +1,6 @@
+1 += 1
+"a" += "b"
+*x += 1
+pass += 1
+x += pass
+(x + y) += 1

crates/ruff_python_parser/resources/inline/err/aug_assign_stmt_invalid_value.py

@@ -0,0 +1,5 @@
+x += *a and b
+x += *yield x
+x += *yield from x
+x += *lambda x: x
+x += y := 1

crates/ruff_python_parser/resources/inline/err/aug_assign_stmt_missing_rhs.py

@@ -0,0 +1,4 @@
+x +=
+1 + 1
+x += y +=
+2 + 2

crates/ruff_python_parser/resources/inline/err/class_def_empty_body.py

@@ -0,0 +1,3 @@
+class Foo:
+class Foo():
+x = 42

crates/ruff_python_parser/resources/inline/err/class_def_missing_name.py

@@ -0,0 +1,3 @@
+class : ...
+class (): ...
+class (metaclass=ABC): ...

crates/ruff_python_parser/resources/inline/err/class_def_unclosed_type_param_list.py

@@ -0,0 +1,3 @@
+class Foo[T1, *T2(a, b):
+    pass
+x = 10

crates/ruff_python_parser/resources/inline/err/clause_expect_indented_block.py

@@ -0,0 +1,6 @@
+# Here, the error is highlighted at the `pass` token
+if True:
+pass
+# The parser is at the end of the program, so let's highlight
+# at the newline token after `:`
+if True:

crates/ruff_python_parser/resources/inline/err/clause_expect_single_statement.py

@@ -0,0 +1 @@
+if True: if True: pass

crates/ruff_python_parser/resources/inline/err/comprehension_missing_for_after_async.py

@@ -0,0 +1,2 @@
+(async)
+(x async x in iter)

crates/ruff_python_parser/resources/inline/err/decorator_invalid_expression.py

@@ -0,0 +1,6 @@
+@*x
+@(*x)
+@((*x))
+@yield x
+@yield from x
+def foo(): ...

crates/ruff_python_parser/resources/inline/err/decorator_missing_expression.py

@@ -0,0 +1,5 @@
+@def foo(): ...
+@
+def foo(): ...
+@@
+def foo(): ...

crates/ruff_python_parser/resources/inline/err/decorator_missing_newline.py

@@ -0,0 +1,3 @@
+@x def foo(): ...
+@x async def foo(): ...
+@x class Foo: ...

crates/ruff_python_parser/resources/inline/err/decorator_unexpected_token.py

@@ -0,0 +1,4 @@
+@foo
+async with x: ...
+@foo
+x = 1

crates/ruff_python_parser/resources/inline/err/del_incomplete_target.py

@@ -0,0 +1,4 @@
+del x, y.
+z
+del x, y[
+z

crates/ruff_python_parser/resources/inline/err/del_stmt_empty.py

@@ -0,0 +1 @@
+del

crates/ruff_python_parser/resources/inline/err/dotted_name_multiple_dots.py

@@ -0,0 +1,2 @@
+import a..b
+import a...b

crates/ruff_python_parser/resources/inline/err/except_stmt_invalid_expression.py

@@ -0,0 +1,8 @@
+try:
+    pass
+except yield x:
+    pass
+try:
+    pass
+except* *x:
+    pass

crates/ruff_python_parser/resources/inline/err/except_stmt_missing_as_name.py

@@ -0,0 +1,6 @@
+try:
+    pass
+except Exception as:
+    pass
+except Exception as
+    pass

crates/ruff_python_parser/resources/inline/err/except_stmt_missing_exception.py

@@ -0,0 +1,13 @@
+try:
+    pass
+except as exc:
+    pass
+# If a '*' is present then exception type is required
+try:
+    pass
+except*:
+    pass
+except*
+    pass
+except* as exc:
+    pass

crates/ruff_python_parser/resources/inline/err/except_stmt_missing_exception_and_as_name.py

@@ -0,0 +1,4 @@
+try:
+    pass
+except as:
+    pass

crates/ruff_python_parser/resources/inline/err/except_stmt_unparenthesized_tuple.py

@@ -0,0 +1,12 @@
+try:
+    pass
+except x, y:
+    pass
+except x, y as exc:
+    pass
+try:
+    pass
+except* x, y:
+    pass
+except* x, y as eg:
+    pass

crates/ruff_python_parser/resources/inline/err/f_string_empty_expression.py

@@ -0,0 +1,2 @@
+f"{}"
+f"{  }"

crates/ruff_python_parser/resources/inline/err/f_string_invalid_conversion_flag_name_tok.py

@@ -0,0 +1 @@
+f"{x!z}"

crates/ruff_python_parser/resources/inline/err/f_string_invalid_conversion_flag_other_tok.py

@@ -0,0 +1,2 @@
+f"{x!123}"
+f"{x!'a'}"

crates/ruff_python_parser/resources/inline/err/f_string_invalid_starred_expr.py

@@ -0,0 +1,4 @@
+# Starred expression inside f-string has a minimum precedence of bitwise or.
+f"{*}"
+f"{*x and y}"
+f"{*yield x}"

crates/ruff_python_parser/resources/inline/err/f_string_lambda_without_parentheses.py

@@ -0,0 +1 @@
+f"{lambda x: x}"

crates/ruff_python_parser/resources/inline/err/f_string_unclosed_lbrace.py

@@ -0,0 +1,5 @@
+f"{"
+f"{foo!r"
+f"{foo="
+f"{"
+f"""{"""

crates/ruff_python_parser/resources/inline/err/f_string_unclosed_lbrace_in_format_spec.py

@@ -0,0 +1,2 @@
+f"hello {x:"
+f"hello {x:.3f"

crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_iter_expr.py

@@ -0,0 +1,3 @@
+for x in *a and b: ...
+for x in yield a: ...
+for target in x := 1: ...

crates/ruff_python_parser/resources/inline/err/for_stmt_invalid_target.py

@@ -0,0 +1,6 @@
+for 1 in x: ...
+for "a" in x: ...
+for *x and y in z: ...
+for *x | y in z: ...
+for await x in z: ...
+for [x, 1, y, *["a"]] in z: ...

crates/ruff_python_parser/resources/inline/err/for_stmt_missing_in_keyword.py

@@ -0,0 +1,2 @@
+for a b: ...
+for a: ...

crates/ruff_python_parser/resources/inline/err/for_stmt_missing_iter.py

@@ -0,0 +1,2 @@
+for x in:
+    a = 1

crates/ruff_python_parser/resources/inline/err/for_stmt_missing_target.py

@@ -0,0 +1 @@
+for in x: ...

crates/ruff_python_parser/resources/inline/err/from_import_dotted_names.py

@@ -0,0 +1,3 @@
+from x import a.
+from x import a.b
+from x import a, b.c, d, e.f, g

crates/ruff_python_parser/resources/inline/err/from_import_empty_names.py

@@ -0,0 +1,3 @@
+from x import
+from x import ()
+from x import ,,

crates/ruff_python_parser/resources/inline/err/from_import_missing_module.py

@@ -0,0 +1,2 @@
+from
+from import x

crates/ruff_python_parser/resources/inline/err/from_import_missing_rpar.py

@@ -0,0 +1,4 @@
+from x import (a, b
+1 + 1
+from x import (a, b,
+2 + 2

crates/ruff_python_parser/resources/inline/err/from_import_star_with_other_names.py

@@ -0,0 +1,4 @@
+from x import *, a
+from x import a, *, b
+from x import *, a as b
+from x import *, *, a

crates/ruff_python_parser/resources/inline/err/from_import_unparenthesized_trailing_comma.py

@@ -0,0 +1,3 @@
+from a import b,
+from a import b as c,
+from a import b, c,

crates/ruff_python_parser/resources/inline/err/function_def_empty_body.py

@@ -0,0 +1,3 @@
+def foo():
+def foo() -> int:
+x = 42

crates/ruff_python_parser/resources/inline/err/function_def_invalid_return_expr.py

@@ -0,0 +1,3 @@
+def foo() -> *int: ...
+def foo() -> (*int): ...
+def foo() -> yield x: ...

crates/ruff_python_parser/resources/inline/err/function_def_missing_identifier.py

@@ -0,0 +1,2 @@
+def (): ...
+def () -> int: ...

crates/ruff_python_parser/resources/inline/err/function_def_missing_return_type.py

@@ -0,0 +1 @@
+def foo() -> : ...

crates/ruff_python_parser/resources/inline/err/function_def_unclosed_parameter_list.py

@@ -0,0 +1,5 @@
+def foo(a: int, b:
+def foo():
+    return 42
+def foo(a: int, b: str
+x = 10

crates/ruff_python_parser/resources/inline/err/function_def_unclosed_type_param_list.py

@@ -0,0 +1,3 @@
+def foo[T1, *T2(a, b):
+    return a + b
+x = 10

crates/ruff_python_parser/resources/inline/err/function_def_unparenthesized_return_types.py

@@ -0,0 +1,2 @@
+def foo() -> int,: ...
+def foo() -> int, str: ...

crates/ruff_python_parser/resources/inline/err/global_stmt_empty.py

@@ -0,0 +1 @@
+global

crates/ruff_python_parser/resources/inline/err/global_stmt_expression.py

@@ -0,0 +1 @@
+global x + 1

crates/ruff_python_parser/resources/inline/err/global_stmt_trailing_comma.py

@@ -0,0 +1,3 @@
+global ,
+global x,
+global x, y,

crates/ruff_python_parser/resources/inline/err/if_stmt_elif_missing_colon.py

@@ -0,0 +1,6 @@
+if x:
+    pass
+elif y
+    pass
+else:
+    pass

crates/ruff_python_parser/resources/inline/err/if_stmt_empty_body.py

@@ -0,0 +1,2 @@
+if True:
+1 + 1

crates/ruff_python_parser/resources/inline/err/if_stmt_invalid_elif_test_expr.py

@@ -0,0 +1,6 @@
+if x:
+    pass
+elif *x:
+    pass
+elif yield x:
+    pass

crates/ruff_python_parser/resources/inline/err/if_stmt_invalid_test_expr.py

@@ -0,0 +1,3 @@
+if *x: ...
+if yield x: ...
+if yield from x: ...

crates/ruff_python_parser/resources/inline/err/if_stmt_missing_colon.py

@@ -0,0 +1,4 @@
+if x
+if x
+    pass
+a = 1

crates/ruff_python_parser/resources/inline/err/if_stmt_missing_test.py

@@ -0,0 +1 @@
+if : ...

crates/ruff_python_parser/resources/inline/err/if_stmt_misspelled_elif.py

@@ -0,0 +1,6 @@
+if True:
+    pass
+elf:
+    pass
+else:
+    pass

crates/ruff_python_parser/resources/inline/err/implicitly_concatenated_unterminated_string.py

@@ -0,0 +1,4 @@
+'hello' 'world
+1 + 1
+'hello' f'world {x}
+2 + 2

crates/ruff_python_parser/resources/inline/err/implicitly_concatenated_unterminated_string_multiline.py

@@ -0,0 +1,11 @@
+(
+    'hello'
+    f'world {x}
+)
+1 + 1
+(
+    'first'
+    'second
+    f'third'
+)
+2 + 2

crates/ruff_python_parser/resources/inline/err/import_alias_missing_asname.py

@@ -0,0 +1 @@
+import x as

crates/ruff_python_parser/resources/inline/err/import_stmt_empty.py

@@ -0,0 +1 @@
+import

crates/ruff_python_parser/resources/inline/err/import_stmt_parenthesized_names.py

@@ -0,0 +1,2 @@
+import (a)
+import (a, b)

crates/ruff_python_parser/resources/inline/err/import_stmt_star_import.py

@@ -0,0 +1,2 @@
+import *
+import x, *, y

crates/ruff_python_parser/resources/inline/err/import_stmt_trailing_comma.py

@@ -0,0 +1,2 @@
+import ,
+import x, y,

crates/ruff_python_parser/resources/inline/err/invalid_byte_literal.py

@@ -0,0 +1,3 @@
+b'123a𝐁c'
+rb"a𝐁c123"
+b"""123a𝐁c"""

crates/ruff_python_parser/resources/inline/err/invalid_del_target.py

@@ -0,0 +1,4 @@
+del x + 1
+del {'x': 1}
+del {'x', 'y'}
+del None, True, False, 1, 1.0, "abc"

crates/ruff_python_parser/resources/inline/err/invalid_fstring_literal_element.py

@@ -0,0 +1,2 @@
+f'hello \N{INVALID} world'
+f"""hello \N{INVALID} world"""

crates/ruff_python_parser/resources/inline/err/invalid_string_literal.py

@@ -0,0 +1,2 @@
+'hello \N{INVALID} world'
+"""hello \N{INVALID} world"""

crates/ruff_python_parser/resources/inline/err/lambda_body_with_starred_expr.py

@@ -0,0 +1,4 @@
+lambda x: *y
+lambda x: *y,
+lambda x: *y, z
+lambda x: *y and z

crates/ruff_python_parser/resources/inline/err/lambda_body_with_yield_expr.py

@@ -0,0 +1,2 @@
+lambda x: yield y
+lambda x: yield from y

crates/ruff_python_parser/resources/inline/err/match_stmt_expect_indented_block.py

@@ -0,0 +1,2 @@
+match foo:
+case _: ...

crates/ruff_python_parser/resources/inline/err/match_stmt_expected_case_block.py

@@ -0,0 +1,5 @@
+match x:
+    x = 1
+match x:
+    match y:
+        case _: ...

crates/ruff_python_parser/resources/inline/err/match_stmt_invalid_guard_expr.py

@@ -0,0 +1,6 @@
+match x:
+    case y if *a: ...
+match x:
+    case y if (*a): ...
+match x:
+    case y if yield x: ...

crates/ruff_python_parser/resources/inline/err/match_stmt_invalid_subject_expr.py

@@ -0,0 +1,7 @@
+match (*x):
+    case _: ...
+# Starred expression precedence test
+match *x and y, z:
+    case _: ...
+match yield x:
+    case _: ...

crates/ruff_python_parser/resources/inline/err/match_stmt_missing_guard_expr.py

@@ -0,0 +1,2 @@
+match x:
+    case y if: ...

crates/ruff_python_parser/resources/inline/err/match_stmt_missing_pattern.py

@@ -0,0 +1,3 @@
+# TODO(dhruvmanila): Here, `case` is a name token because of soft keyword transformer
+match x:
+    case : ...

crates/ruff_python_parser/resources/inline/err/match_stmt_no_newline_before_case.py

@@ -0,0 +1 @@
+match foo: case _: ...

crates/ruff_python_parser/resources/inline/err/match_stmt_single_starred_subject.py

@@ -0,0 +1,2 @@
+match *foo:
+    case _: ...

crates/ruff_python_parser/resources/inline/err/mixed_bytes_and_non_bytes_literals.py

@@ -0,0 +1,3 @@
+'first' b'second'
+f'first' b'second'
+'first' f'second' b'third'

crates/ruff_python_parser/resources/inline/err/multiple_clauses_on_same_line.py

@@ -0,0 +1,6 @@
+if True: pass elif False: pass else: pass
+if True: pass; elif False: pass; else: pass
+for x in iter: break else: pass
+for x in iter: break; else: pass
+try: pass except exc: pass else: pass finally: pass
+try: pass; except exc: pass; else: pass; finally: pass

crates/ruff_python_parser/resources/inline/err/nonlocal_stmt_empty.py

@@ -0,0 +1 @@
+nonlocal

crates/ruff_python_parser/resources/inline/err/nonlocal_stmt_expression.py

@@ -0,0 +1 @@
+nonlocal x + 1

crates/ruff_python_parser/resources/inline/err/nonlocal_stmt_trailing_comma.py

@@ -0,0 +1,3 @@
+nonlocal ,
+nonlocal x,
+nonlocal x, y,

crates/ruff_python_parser/resources/inline/err/param_missing_annotation.py

@@ -0,0 +1,2 @@
+def foo(x:): ...
+def foo(x:,): ...

crates/ruff_python_parser/resources/inline/err/param_missing_default.py

@@ -0,0 +1,2 @@
+def foo(x=): ...
+def foo(x: int = ): ...

crates/ruff_python_parser/resources/inline/err/param_with_invalid_annotation.py

@@ -0,0 +1,3 @@
+def foo(arg: *int): ...
+def foo(arg: yield int): ...
+def foo(arg: x := int): ...