LibCST/libcst/_parser/_entrypoints.py

# Copyright (c) Facebook, Inc. and its affiliates.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.

# pyre-strict
"""
Parser entrypoints define the way users of our API are allowed to interact with the
parser. A parser entrypoint should take the source code and some configuration
information
"""

from typing import Union

from libcst._nodes._base import CSTNode
from libcst._nodes._expression import BaseExpression
from libcst._nodes._module import Module
from libcst._nodes._statement import BaseCompoundStatement, SimpleStatementLine
from libcst._parser._detect_config import detect_config
from libcst._parser._grammar import get_grammar, validate_grammar
from libcst._parser._python_parser import PythonCSTParser
from libcst._parser._types.config import PartialParserConfig


_DEFAULT_PARTIAL_PARSER_CONFIG: PartialParserConfig = PartialParserConfig()


def _parse(
    entrypoint: str,
    source: Union[str, bytes],
    config: PartialParserConfig,
    *,
    detect_trailing_newline: bool,
    detect_default_newline: bool,
) -> CSTNode:
    detection_result = detect_config(
        source,
        partial=config,
        detect_trailing_newline=detect_trailing_newline,
        detect_default_newline=detect_default_newline,
    )
    validate_grammar()
    grammar = get_grammar()

    parser = PythonCSTParser(
        tokens=detection_result.tokens,
        config=detection_result.config,
        pgen_grammar=grammar,
        start_nonterminal=entrypoint,
    )
    # The parser has an Any return type, we can at least refine it to CSTNode here.
    result = parser.parse()
    assert isinstance(result, CSTNode)
    return result


def parse_module(
    source: Union[str, bytes],  # the only entrypoint that accepts bytes
    config: PartialParserConfig = _DEFAULT_PARTIAL_PARSER_CONFIG,
) -> Module:
    """
    Accepts an entire python module, including all leading and trailing whitespace.

    If source is ``bytes``, the encoding will be inferred and preserved. If
    the source is a ``string``, we will default to assuming UTF-8 encoding if the
    module is rendered back out to source as bytes. It is recommended that when
    calling :func:`~libcst.parse_module` with a string you access the serialized
    code using :class:`~libcst.Module`'s code attribute, and when calling it with
    bytes you access the serialized code using :class:`~libcst.Module`'s bytes
    attribute.
    """
    result = _parse(
        "file_input",
        source,
        config,
        detect_trailing_newline=True,
        detect_default_newline=True,
    )
    assert isinstance(result, Module)
    return result


def parse_statement(
    source: str, config: PartialParserConfig = _DEFAULT_PARTIAL_PARSER_CONFIG
) -> Union[SimpleStatementLine, BaseCompoundStatement]:
    """
    Accepts a statement followed by a trailing newline. If a trailing newline is not
    provided, one will be added. :func:`parse_statement` is provided mainly as a
    convenience function to generate semi-complex trees from code snippetes. If you
    need to represent a statement exactly, including all leading/trailing comments,
    you should instead use :func:`parse_module`.

    Leading comments and trailing comments (on the same line) are accepted, but
    whitespace (or anything else) after the statement's trailing newline is not valid
    (there's nowhere to store it on the statement node). Note that since there is
    nowhere to store leading and trailing comments/empty lines, code rendered out
    from a parsed statement using ``cst.Module([]).code_for_node(statement)`` will
    not include leading/trailing comments.
    """
    # use detect_trailing_newline to insert a newline
    result = _parse(
        "stmt_input",
        source,
        config,
        detect_trailing_newline=True,
        detect_default_newline=False,
    )
    assert isinstance(result, (SimpleStatementLine, BaseCompoundStatement))
    return result


def parse_expression(
    source: str, config: PartialParserConfig = _DEFAULT_PARTIAL_PARSER_CONFIG
) -> BaseExpression:
    """
    Accepts an expression on a single line. Leading and trailing whitespace is not
    valid (there's nowhere to store it on the expression node).
    :func:`parse_expression` is provided mainly as a convenience function to generate
    semi-complex trees from code snippets. If you need to represent an expression
    exactly, including all leading/trailing comments, you should instead use
    :func:`parse_module`.
    """
    result = _parse(
        "expression_input",
        source,
        config,
        detect_trailing_newline=False,
        detect_default_newline=False,
    )
    assert isinstance(result, BaseExpression)
    return result