remove entry points to pure parser (#1375)

* rm: ci

* rm: entry point

* fix: tests

* fix: remove combine step from ci

* linter fixes

* omit the _parser

* fix newlines

* fix: remove optional

* fix: linter

---------

Co-authored-by: thereversiblewheel <martin.li@uwaterloo.ca>

.github/workflows/ci.yml

@@ -39,15 +39,8 @@ jobs:
         run: uv sync --locked --dev
       - name: Native Parser Tests
         run: uv run poe test
-      - name: Pure Parser Tests
-        env:
-          COVERAGE_FILE: .coverage.pure
-          LIBCST_PARSER_TYPE: pure
-        run: uv run poe test
       - name: Coverage
-        run: |
-          uv run coverage combine .coverage.pure
-          uv run coverage report
+        run: uv run coverage report
 
   # Run linters
   lint:

libcst/_nodes/tests/test_atom.py

@@ -9,7 +9,6 @@ from typing import Any
 import libcst as cst
 from libcst import parse_expression
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -1184,7 +1183,7 @@ class AtomTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)
 

libcst/_nodes/tests/test_binary_op.py

@@ -8,7 +8,6 @@ from typing import Any
 import libcst as cst
 from libcst import parse_expression
 from libcst._nodes.tests.base import CSTNodeTest
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -189,4 +188,4 @@ class BinaryOperationTest(CSTNodeTest):
         )
     )
     def test_parse_error(self, **kwargs: Any) -> None:
-        self.assert_parses(**kwargs, expect_success=not is_native())
+        self.assert_parses(**kwargs, expect_success=False)

libcst/_nodes/tests/test_classdef.py

@@ -8,7 +8,6 @@ from typing import Any, Callable
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -210,8 +209,6 @@ class ClassDefCreationTest(CSTNodeTest):
         )
     )
     def test_valid_native(self, **kwargs: Any) -> None:
-        if not is_native():
-            self.skipTest("Disabled for pure python parser")
         self.validate_node(**kwargs)
 
     @data_provider(

libcst/_nodes/tests/test_dict.py

@@ -8,7 +8,6 @@ from typing import Any
 import libcst as cst
 from libcst import parse_expression
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -188,6 +187,6 @@ class DictTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_funcdef.py

@@ -8,7 +8,6 @@ from typing import Any, Callable
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest, DummyIndentedBlock, parse_statement_as
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -741,8 +740,6 @@ class FunctionDefCreationTest(CSTNodeTest):
         )
     )
     def test_valid(self, **kwargs: Any) -> None:
-        if not is_native() and kwargs.get("native_only", False):
-            self.skipTest("Disabled for native parser")
         if "native_only" in kwargs:
             kwargs.pop("native_only")
         self.validate_node(**kwargs)
@@ -891,8 +888,6 @@ class FunctionDefCreationTest(CSTNodeTest):
         )
     )
     def test_valid_native(self, **kwargs: Any) -> None:
-        if not is_native():
-            self.skipTest("Disabled for pure python parser")
         self.validate_node(**kwargs)
 
     @data_provider(
@@ -2223,8 +2218,6 @@ class FunctionDefParserTest(CSTNodeTest):
         )
     )
     def test_valid_38(self, node: cst.CSTNode, code: str, **kwargs: Any) -> None:
-        if not is_native() and kwargs.get("native_only", False):
-            self.skipTest("disabled for pure python parser")
         self.validate_node(node, code, _parse_statement_force_38)
 
     @data_provider(
@@ -2252,7 +2245,7 @@ class FunctionDefParserTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)
 
@@ -2271,6 +2264,4 @@ class FunctionDefParserTest(CSTNodeTest):
         )
     )
     def test_parse_error(self, **kwargs: Any) -> None:
-        if not is_native():
-            self.skipTest("Skipped for non-native parser")
         self.assert_parses(**kwargs, expect_success=False, parser=parse_statement)

libcst/_nodes/tests/test_list.py

@@ -8,7 +8,6 @@ from typing import Any, Callable
 import libcst as cst
 from libcst import parse_expression, parse_statement
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -126,6 +125,6 @@ class ListTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_match.py

@@ -3,17 +3,14 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-from typing import Any, Callable, Optional
+from typing import Any, Callable
 
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest
-from libcst._parser.entrypoints import is_native
 from libcst.testing.utils import data_provider
 
-parser: Optional[Callable[[str], cst.CSTNode]] = (
-    parse_statement if is_native() else None
-)
+parser: Callable[[str], cst.CSTNode] = parse_statement
 
 
 class MatchTest(CSTNodeTest):

libcst/_nodes/tests/test_matrix_multiply.py

@@ -11,7 +11,6 @@ from libcst._nodes.tests.base import (
     parse_expression_as,
     parse_statement_as,
 )
-from libcst._parser.entrypoints import is_native
 from libcst.testing.utils import data_provider
 
 
@@ -70,6 +69,6 @@ class NamedExprTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_module.py

@@ -8,7 +8,7 @@ from typing import cast, Tuple
 import libcst as cst
 from libcst import parse_module, parse_statement
 from libcst._nodes.tests.base import CSTNodeTest
-from libcst._parser.entrypoints import is_native
+
 from libcst.metadata import CodeRange, MetadataWrapper, PositionProvider
 from libcst.testing.utils import data_provider
 
@@ -117,7 +117,7 @@ class ModuleTest(CSTNodeTest):
     def test_parser(
         self, *, code: str, expected: cst.Module, enabled_for_native: bool = True
     ) -> None:
-        if is_native() and not enabled_for_native:
+        if not enabled_for_native:
             self.skipTest("Disabled for native parser")
         self.assertEqual(parse_module(code), expected)
 

libcst/_nodes/tests/test_set.py

@@ -8,7 +8,6 @@ from typing import Any, Callable
 import libcst as cst
 from libcst import parse_expression
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
-from libcst._parser.entrypoints import is_native
 from libcst.testing.utils import data_provider
 
 
@@ -133,6 +132,6 @@ class ListTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_try.py

@@ -3,18 +3,15 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-from typing import Any, Callable, Optional
+from typing import Any, Callable
 
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest, DummyIndentedBlock
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
-native_parse_statement: Optional[Callable[[str], cst.CSTNode]] = (
-    parse_statement if is_native() else None
-)
+native_parse_statement: Callable[[str], cst.CSTNode] = parse_statement
 
 
 class TryTest(CSTNodeTest):

libcst/_nodes/tests/test_tuple.py

@@ -8,7 +8,6 @@ from typing import Any, Callable
 import libcst as cst
 from libcst import parse_expression, parse_statement
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -286,6 +285,6 @@ class TupleTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_type_alias.py

@@ -8,7 +8,6 @@ from typing import Any
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -132,8 +131,6 @@ class TypeAliasCreationTest(CSTNodeTest):
         )
     )
     def test_valid(self, **kwargs: Any) -> None:
-        if not is_native():
-            self.skipTest("Disabled in the old parser")
         self.validate_node(**kwargs)
 
 
@@ -252,6 +249,4 @@ class TypeAliasParserTest(CSTNodeTest):
         )
     )
     def test_valid(self, **kwargs: Any) -> None:
-        if not is_native():
-            self.skipTest("Disabled in the old parser")
         self.validate_node(**kwargs)

libcst/_nodes/tests/test_with.py

@@ -7,9 +7,7 @@ from typing import Any
 
 import libcst as cst
 from libcst import parse_statement, PartialParserConfig
-from libcst._maybe_sentinel import MaybeSentinel
 from libcst._nodes.tests.base import CSTNodeTest, DummyIndentedBlock, parse_statement_as
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -187,14 +185,14 @@ class WithTest(CSTNodeTest):
                         cst.WithItem(
                             cst.Call(
                                 cst.Name("context_mgr"),
-                                lpar=() if is_native() else (cst.LeftParen(),),
-                                rpar=() if is_native() else (cst.RightParen(),),
+                                lpar=(),
+                                rpar=(),
                             )
                         ),
                     ),
                     cst.SimpleStatementSuite((cst.Pass(),)),
-                    lpar=(cst.LeftParen() if is_native() else MaybeSentinel.DEFAULT),
-                    rpar=(cst.RightParen() if is_native() else MaybeSentinel.DEFAULT),
+                    lpar=(cst.LeftParen()),
+                    rpar=(cst.RightParen()),
                     whitespace_after_with=cst.SimpleWhitespace(""),
                 ),
                 "code": "with(context_mgr()): pass\n",
@@ -233,7 +231,7 @@ class WithTest(CSTNodeTest):
                     rpar=cst.RightParen(whitespace_before=cst.SimpleWhitespace(" ")),
                 ),
                 "code": ("with ( foo(),\n" "       bar(), ): pass\n"),  # noqa
-                "parser": parse_statement if is_native() else None,
+                "parser": parse_statement,
                 "expected_position": CodeRange((1, 0), (2, 21)),
             },
         )
@@ -310,7 +308,7 @@ class WithTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)
 

libcst/_nodes/tests/test_yield.py

@@ -8,7 +8,6 @@ from typing import Any, Callable, Optional
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest, parse_statement_as
-from libcst._parser.entrypoints import is_native
 from libcst.helpers import ensure_type
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
@@ -241,6 +240,6 @@ class YieldParsingTest(CSTNodeTest):
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
-        if is_native() and not kwargs.get("expect_success", True):
+        if not kwargs.get("expect_success", True):
             self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_parser/entrypoints.py

@@ -9,27 +9,20 @@ parser. A parser entrypoint should take the source code and some configuration
 information
 """
 
-import os
 from functools import partial
 from typing import Union
 
+from libcst import native
 from libcst._nodes.base import CSTNode
 from libcst._nodes.expression import BaseExpression
 from libcst._nodes.module import Module
 from libcst._nodes.statement import BaseCompoundStatement, SimpleStatementLine
-from libcst._parser.detect_config import convert_to_utf8, detect_config
-from libcst._parser.grammar import get_grammar, validate_grammar
-from libcst._parser.python_parser import PythonCSTParser
+from libcst._parser.detect_config import convert_to_utf8
 from libcst._parser.types.config import PartialParserConfig
 
 _DEFAULT_PARTIAL_PARSER_CONFIG: PartialParserConfig = PartialParserConfig()
 
 
-def is_native() -> bool:
-    typ = os.environ.get("LIBCST_PARSER_TYPE")
-    return typ != "pure"
-
-
 def _parse(
     entrypoint: str,
     source: Union[str, bytes],
@@ -38,57 +31,19 @@ def _parse(
     detect_trailing_newline: bool,
     detect_default_newline: bool,
 ) -> CSTNode:
-    if is_native():
-        from libcst.native import parse_expression, parse_module, parse_statement
 
-        encoding, source_str = convert_to_utf8(source, partial=config)
+    encoding, source_str = convert_to_utf8(source, partial=config)
 
-        if entrypoint == "file_input":
-            parse = partial(parse_module, encoding=encoding)
-        elif entrypoint == "stmt_input":
-            parse = parse_statement
-        elif entrypoint == "expression_input":
-            parse = parse_expression
-        else:
-            raise ValueError(f"Unknown parser entry point: {entrypoint}")
+    if entrypoint == "file_input":
+        parse = partial(native.parse_module, encoding=encoding)
+    elif entrypoint == "stmt_input":
+        parse = native.parse_statement
+    elif entrypoint == "expression_input":
+        parse = native.parse_expression
+    else:
+        raise ValueError(f"Unknown parser entry point: {entrypoint}")
 
-        return parse(source_str)
-    return _pure_python_parse(
-        entrypoint,
-        source,
-        config,
-        detect_trailing_newline=detect_trailing_newline,
-        detect_default_newline=detect_default_newline,
-    )
-
-
-def _pure_python_parse(
-    entrypoint: str,
-    source: Union[str, bytes],
-    config: PartialParserConfig,
-    *,
-    detect_trailing_newline: bool,
-    detect_default_newline: bool,
-) -> CSTNode:
-    detection_result = detect_config(
-        source,
-        partial=config,
-        detect_trailing_newline=detect_trailing_newline,
-        detect_default_newline=detect_default_newline,
-    )
-    validate_grammar()
-    grammar = get_grammar(config.parsed_python_version, config.future_imports)
-
-    parser = PythonCSTParser(
-        tokens=detection_result.tokens,
-        config=detection_result.config,
-        pgen_grammar=grammar,
-        start_nonterminal=entrypoint,
-    )
-    # The parser has an Any return type, we can at least refine it to CSTNode here.
-    result = parser.parse()
-    assert isinstance(result, CSTNode)
-    return result
+    return parse(source_str)
 
 
 def parse_module(

libcst/_parser/tests/test_parse_errors.py

@@ -10,7 +10,6 @@ from unittest.mock import patch
 
 import libcst as cst
 from libcst._nodes.base import CSTValidationError
-from libcst._parser.entrypoints import is_native
 from libcst.testing.utils import data_provider, UnitTest
 
 
@@ -174,8 +173,6 @@ class ParseErrorsTest(UnitTest):
             parse_fn()
         # make sure str() doesn't blow up
         self.assertIn("Syntax Error", str(cm.exception))
-        if not is_native():
-            self.assertEqual(str(cm.exception), expected)
 
     def test_native_fallible_into_py(self) -> None:
         with patch("libcst._nodes.expression.Name._validate") as await_validate:

libcst/codemod/tests/test_codemod_cli.py

@@ -12,7 +12,6 @@ import tempfile
 from pathlib import Path
 from unittest import skipIf
 
-from libcst._parser.entrypoints import is_native
 from libcst.codemod import CodemodTest
 from libcst.testing.utils import UnitTest
 
@@ -37,16 +36,10 @@ class TestCodemodCLI(UnitTest):
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
         )
-        if not is_native():
-            self.assertIn(
-                "ParserSyntaxError: Syntax Error @ 14:11.",
-                rlt.stderr.decode("utf-8"),
-            )
-        else:
-            self.assertIn(
-                "error: cannot format -: Cannot parse for target version Python 3.6: 13:10:     async with AsyncExitStack() as stack:",
-                rlt.stderr.decode("utf-8"),
-            )
+        self.assertIn(
+            "error: cannot format -: Cannot parse for target version Python 3.6: 13:10:     async with AsyncExitStack() as stack:",
+            rlt.stderr.decode("utf-8"),
+        )
 
     def test_codemod_external(self) -> None:
         # Test running the NOOP command as an "external command"

libcst/metadata/tests/test_scope_provider.py

@@ -11,7 +11,6 @@ from unittest import mock
 
 import libcst as cst
 from libcst import ensure_type
-from libcst._parser.entrypoints import is_native
 from libcst.metadata import MetadataWrapper
 from libcst.metadata.scope_provider import (
     _gen_dotted_names,
@@ -2029,8 +2028,6 @@ class ScopeProviderTest(UnitTest):
         )
 
     def test_type_alias_scope(self) -> None:
-        if not is_native():
-            self.skipTest("type aliases are only supported in the native parser")
         m, scopes = get_scope_metadata_provider(
             """
                 type A = C
@@ -2052,8 +2049,6 @@ class ScopeProviderTest(UnitTest):
         self.assertIsInstance(scopes[alias.value], AnnotationScope)
 
     def test_type_alias_param(self) -> None:
-        if not is_native():
-            self.skipTest("type parameters are only supported in the native parser")
         m, scopes = get_scope_metadata_provider(
             """
                 B = int
@@ -2084,8 +2079,6 @@ class ScopeProviderTest(UnitTest):
         )
 
     def test_type_alias_tuple_and_paramspec(self) -> None:
-        if not is_native():
-            self.skipTest("type parameters are only supported in the native parser")
         m, scopes = get_scope_metadata_provider(
             """
             type A[*T] = T
@@ -2113,8 +2106,6 @@ class ScopeProviderTest(UnitTest):
         self.assertEqual(t_refs[0].node, alias_paramspec.value)
 
     def test_class_type_params(self) -> None:
-        if not is_native():
-            self.skipTest("type parameters are only supported in the native parser")
         m, scopes = get_scope_metadata_provider(
             """
             class W[T]:
@@ -2149,8 +2140,6 @@ class ScopeProviderTest(UnitTest):
         self.assertEqual(t_refs_in_g[0].node, g.returns.annotation)
 
     def test_nested_class_type_params(self) -> None:
-        if not is_native():
-            self.skipTest("type parameters are only supported in the native parser")
         m, scopes = get_scope_metadata_provider(
             """
             class Outer:
@@ -2168,8 +2157,6 @@ class ScopeProviderTest(UnitTest):
         )
 
     def test_annotation_refers_to_nested_class(self) -> None:
-        if not is_native():
-            self.skipTest("type parameters are only supported in the native parser")
         m, scopes = get_scope_metadata_provider(
             """
                 class Outer:
@@ -2229,8 +2216,6 @@ class ScopeProviderTest(UnitTest):
         )
 
     def test_body_isnt_subject_to_special_annotation_rule(self) -> None:
-        if not is_native():
-            self.skipTest("type parameters are only supported in the native parser")
         m, scopes = get_scope_metadata_provider(
             """
             class Outer:

libcst/tests/__main__.py

@@ -5,11 +5,6 @@
 
 from unittest import main
 
-from libcst._parser.entrypoints import is_native
-
 
 if __name__ == "__main__":
-    parser_type = "native" if is_native() else "pure"
-    print(f"running tests with {parser_type!r} parser")
-
     main(module=None, verbosity=2)

libcst/tests/test_roundtrip.py

@@ -8,7 +8,7 @@ from pathlib import Path
 from unittest import TestCase
 
 from libcst import CSTTransformer, parse_module
-from libcst._parser.entrypoints import is_native
+
 
 fixtures: Path = Path(__file__).parent.parent.parent / "native/libcst/tests/fixtures"
 
@@ -19,8 +19,6 @@ class NOOPTransformer(CSTTransformer):
 
 class RoundTripTests(TestCase):
     def _get_fixtures(self) -> list[Path]:
-        if not is_native():
-            self.skipTest("pure python parser doesn't work with this")
         self.assertTrue(fixtures.exists(), f"{fixtures} should exist")
         files = list(fixtures.iterdir())
         self.assertGreater(len(files), 0)

pyproject.toml

@@ -66,6 +66,7 @@ fail_under = 93
 precision = 1
 show_missing = true
 skip_covered = true
+omit = ["*/_parser/*"] # temporary while I remove the parser
 
 [tool.uv]
 cache-keys = [