mirror of
https://github.com/Instagram/LibCST.git
synced 2025-12-23 10:35:53 +00:00
48668dfabb
#1343 Adds support to parse t-strings Couple things of note: TemplatedString* is largely a copy of FormattedString* Since clients operate of libcst objects I consider this this part of a public API - following the python grammar (where TStrings are distinct from FStrings) seems like a good way to avoid changes to the API in the future. Within the tokenizer we reuse the fstring machinery I consider this an implementation detail, fstrings and tstrings are (for now) identical, we can change this later without changes to the public api. Since 2 -> we have a new FTStringType enum We need to discriminate between f and t strings to know which token to return, a bit clumsy to use in my opinion - so looking for feedback here on how to improve this.
462 lines
8.7 KiB
Python
462 lines
8.7 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
#
|
|
# This source code is licensed under the MIT license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
from libcst._batched_visitor import BatchableCSTVisitor, visit_batched
|
|
from libcst._exceptions import CSTLogicError, MetadataException, ParserSyntaxError
|
|
from libcst._flatten_sentinel import FlattenSentinel
|
|
from libcst._maybe_sentinel import MaybeSentinel
|
|
from libcst._metadata_dependent import MetadataDependent
|
|
from libcst._nodes.base import CSTNode, CSTValidationError
|
|
from libcst._nodes.expression import (
|
|
Annotation,
|
|
Arg,
|
|
Asynchronous,
|
|
Attribute,
|
|
Await,
|
|
BaseAssignTargetExpression,
|
|
BaseComp,
|
|
BaseDelTargetExpression,
|
|
BaseDict,
|
|
BaseDictElement,
|
|
BaseElement,
|
|
BaseExpression,
|
|
BaseFormattedStringContent,
|
|
BaseList,
|
|
BaseNumber,
|
|
BaseSet,
|
|
BaseSimpleComp,
|
|
BaseSlice,
|
|
BaseString,
|
|
BaseTemplatedStringContent,
|
|
BinaryOperation,
|
|
BooleanOperation,
|
|
Call,
|
|
Comparison,
|
|
ComparisonTarget,
|
|
CompFor,
|
|
CompIf,
|
|
ConcatenatedString,
|
|
Dict,
|
|
DictComp,
|
|
DictElement,
|
|
Element,
|
|
Ellipsis,
|
|
Float,
|
|
FormattedString,
|
|
FormattedStringExpression,
|
|
FormattedStringText,
|
|
From,
|
|
GeneratorExp,
|
|
IfExp,
|
|
Imaginary,
|
|
Index,
|
|
Integer,
|
|
Lambda,
|
|
LeftCurlyBrace,
|
|
LeftParen,
|
|
LeftSquareBracket,
|
|
List,
|
|
ListComp,
|
|
Name,
|
|
NamedExpr,
|
|
Param,
|
|
Parameters,
|
|
ParamSlash,
|
|
ParamStar,
|
|
RightCurlyBrace,
|
|
RightParen,
|
|
RightSquareBracket,
|
|
Set,
|
|
SetComp,
|
|
SimpleString,
|
|
Slice,
|
|
StarredDictElement,
|
|
StarredElement,
|
|
Subscript,
|
|
SubscriptElement,
|
|
TemplatedString,
|
|
TemplatedStringExpression,
|
|
TemplatedStringText,
|
|
Tuple,
|
|
UnaryOperation,
|
|
Yield,
|
|
)
|
|
from libcst._nodes.module import Module
|
|
from libcst._nodes.op import (
|
|
Add,
|
|
AddAssign,
|
|
And,
|
|
AssignEqual,
|
|
BaseAugOp,
|
|
BaseBinaryOp,
|
|
BaseBooleanOp,
|
|
BaseCompOp,
|
|
BaseUnaryOp,
|
|
BitAnd,
|
|
BitAndAssign,
|
|
BitInvert,
|
|
BitOr,
|
|
BitOrAssign,
|
|
BitXor,
|
|
BitXorAssign,
|
|
Colon,
|
|
Comma,
|
|
Divide,
|
|
DivideAssign,
|
|
Dot,
|
|
Equal,
|
|
FloorDivide,
|
|
FloorDivideAssign,
|
|
GreaterThan,
|
|
GreaterThanEqual,
|
|
ImportStar,
|
|
In,
|
|
Is,
|
|
IsNot,
|
|
LeftShift,
|
|
LeftShiftAssign,
|
|
LessThan,
|
|
LessThanEqual,
|
|
MatrixMultiply,
|
|
MatrixMultiplyAssign,
|
|
Minus,
|
|
Modulo,
|
|
ModuloAssign,
|
|
Multiply,
|
|
MultiplyAssign,
|
|
Not,
|
|
NotEqual,
|
|
NotIn,
|
|
Or,
|
|
Plus,
|
|
Power,
|
|
PowerAssign,
|
|
RightShift,
|
|
RightShiftAssign,
|
|
Semicolon,
|
|
Subtract,
|
|
SubtractAssign,
|
|
)
|
|
from libcst._nodes.statement import (
|
|
AnnAssign,
|
|
AsName,
|
|
Assert,
|
|
Assign,
|
|
AssignTarget,
|
|
AugAssign,
|
|
BaseCompoundStatement,
|
|
BaseSmallStatement,
|
|
BaseStatement,
|
|
BaseSuite,
|
|
Break,
|
|
ClassDef,
|
|
Continue,
|
|
Decorator,
|
|
Del,
|
|
Else,
|
|
ExceptHandler,
|
|
ExceptStarHandler,
|
|
Expr,
|
|
Finally,
|
|
For,
|
|
FunctionDef,
|
|
Global,
|
|
If,
|
|
Import,
|
|
ImportAlias,
|
|
ImportFrom,
|
|
IndentedBlock,
|
|
Match,
|
|
MatchAs,
|
|
MatchCase,
|
|
MatchClass,
|
|
MatchKeywordElement,
|
|
MatchList,
|
|
MatchMapping,
|
|
MatchMappingElement,
|
|
MatchOr,
|
|
MatchOrElement,
|
|
MatchPattern,
|
|
MatchSequence,
|
|
MatchSequenceElement,
|
|
MatchSingleton,
|
|
MatchStar,
|
|
MatchTuple,
|
|
MatchValue,
|
|
NameItem,
|
|
Nonlocal,
|
|
ParamSpec,
|
|
Pass,
|
|
Raise,
|
|
Return,
|
|
SimpleStatementLine,
|
|
SimpleStatementSuite,
|
|
Try,
|
|
TryStar,
|
|
TypeAlias,
|
|
TypeParam,
|
|
TypeParameters,
|
|
TypeVar,
|
|
TypeVarTuple,
|
|
While,
|
|
With,
|
|
WithItem,
|
|
)
|
|
from libcst._nodes.whitespace import (
|
|
BaseParenthesizableWhitespace,
|
|
Comment,
|
|
EmptyLine,
|
|
Newline,
|
|
ParenthesizedWhitespace,
|
|
SimpleWhitespace,
|
|
TrailingWhitespace,
|
|
)
|
|
from libcst._parser.entrypoints import parse_expression, parse_module, parse_statement
|
|
from libcst._parser.types.config import (
|
|
KNOWN_PYTHON_VERSION_STRINGS,
|
|
PartialParserConfig,
|
|
)
|
|
from libcst._removal_sentinel import RemovalSentinel, RemoveFromParent
|
|
from libcst._visitors import CSTNodeT, CSTTransformer, CSTVisitor, CSTVisitorT
|
|
|
|
try:
|
|
from libcst._version import version as LIBCST_VERSION
|
|
except ImportError:
|
|
LIBCST_VERSION = "unknown"
|
|
from libcst.helpers import ( # from libcst import ensure_type is deprecated, will be removed in 0.4.0
|
|
ensure_type,
|
|
)
|
|
from libcst.metadata.base_provider import (
|
|
BaseMetadataProvider,
|
|
BatchableMetadataProvider,
|
|
VisitorMetadataProvider,
|
|
)
|
|
from libcst.metadata.wrapper import MetadataWrapper
|
|
|
|
__all__ = [
|
|
"KNOWN_PYTHON_VERSION_STRINGS",
|
|
"LIBCST_VERSION",
|
|
"BatchableCSTVisitor",
|
|
"CSTNodeT",
|
|
"CSTTransformer",
|
|
"CSTValidationError",
|
|
"CSTVisitor",
|
|
"CSTVisitorT",
|
|
"FlattenSentinel",
|
|
"MaybeSentinel",
|
|
"CSTLogicError",
|
|
"MetadataException",
|
|
"ParserSyntaxError",
|
|
"PartialParserConfig",
|
|
"RemoveFromParent",
|
|
"RemovalSentinel",
|
|
"ensure_type", # from libcst import ensure_type is deprecated, will be removed in 0.4.0
|
|
"visit_batched",
|
|
"parse_module",
|
|
"parse_expression",
|
|
"parse_statement",
|
|
"CSTNode",
|
|
"Module",
|
|
"Annotation",
|
|
"Arg",
|
|
"Asynchronous",
|
|
"Attribute",
|
|
"Await",
|
|
"BaseAssignTargetExpression",
|
|
"BaseComp",
|
|
"BaseDelTargetExpression",
|
|
"BaseDict",
|
|
"BaseDictElement",
|
|
"BaseElement",
|
|
"BaseExpression",
|
|
"BaseFormattedStringContent",
|
|
"BaseTemplatedStringContent",
|
|
"BaseList",
|
|
"BaseNumber",
|
|
"BaseSet",
|
|
"BaseSimpleComp",
|
|
"BaseSlice",
|
|
"BaseString",
|
|
"BinaryOperation",
|
|
"BooleanOperation",
|
|
"Call",
|
|
"Comparison",
|
|
"ComparisonTarget",
|
|
"CompFor",
|
|
"CompIf",
|
|
"ConcatenatedString",
|
|
"Dict",
|
|
"DictComp",
|
|
"DictElement",
|
|
"Element",
|
|
"Ellipsis",
|
|
"Float",
|
|
"FormattedString",
|
|
"FormattedStringExpression",
|
|
"FormattedStringText",
|
|
"TemplatedString",
|
|
"TemplatedStringText",
|
|
"TemplatedStringExpression",
|
|
"From",
|
|
"GeneratorExp",
|
|
"IfExp",
|
|
"Imaginary",
|
|
"Index",
|
|
"Integer",
|
|
"Lambda",
|
|
"LeftCurlyBrace",
|
|
"LeftParen",
|
|
"LeftSquareBracket",
|
|
"List",
|
|
"ListComp",
|
|
"Name",
|
|
"NamedExpr",
|
|
"Param",
|
|
"Parameters",
|
|
"ParamSlash",
|
|
"ParamStar",
|
|
"RightCurlyBrace",
|
|
"RightParen",
|
|
"RightSquareBracket",
|
|
"Set",
|
|
"SetComp",
|
|
"SimpleString",
|
|
"Slice",
|
|
"StarredDictElement",
|
|
"StarredElement",
|
|
"Subscript",
|
|
"SubscriptElement",
|
|
"Tuple",
|
|
"UnaryOperation",
|
|
"Yield",
|
|
"Add",
|
|
"AddAssign",
|
|
"And",
|
|
"AssignEqual",
|
|
"BaseAugOp",
|
|
"BaseBinaryOp",
|
|
"BaseBooleanOp",
|
|
"BaseCompOp",
|
|
"BaseUnaryOp",
|
|
"BitAnd",
|
|
"BitAndAssign",
|
|
"BitInvert",
|
|
"BitOr",
|
|
"BitOrAssign",
|
|
"BitXor",
|
|
"BitXorAssign",
|
|
"Colon",
|
|
"Comma",
|
|
"Divide",
|
|
"DivideAssign",
|
|
"Dot",
|
|
"Equal",
|
|
"FloorDivide",
|
|
"FloorDivideAssign",
|
|
"GreaterThan",
|
|
"GreaterThanEqual",
|
|
"ImportStar",
|
|
"In",
|
|
"Is",
|
|
"IsNot",
|
|
"LeftShift",
|
|
"LeftShiftAssign",
|
|
"LessThan",
|
|
"LessThanEqual",
|
|
"MatrixMultiply",
|
|
"MatrixMultiplyAssign",
|
|
"Minus",
|
|
"Modulo",
|
|
"ModuloAssign",
|
|
"Multiply",
|
|
"MultiplyAssign",
|
|
"Not",
|
|
"NotEqual",
|
|
"NotIn",
|
|
"Or",
|
|
"Plus",
|
|
"Power",
|
|
"PowerAssign",
|
|
"RightShift",
|
|
"RightShiftAssign",
|
|
"Semicolon",
|
|
"Subtract",
|
|
"SubtractAssign",
|
|
"AnnAssign",
|
|
"AsName",
|
|
"Assert",
|
|
"Assign",
|
|
"AssignTarget",
|
|
"AugAssign",
|
|
"BaseCompoundStatement",
|
|
"BaseSmallStatement",
|
|
"BaseStatement",
|
|
"BaseSuite",
|
|
"Break",
|
|
"ClassDef",
|
|
"Continue",
|
|
"Decorator",
|
|
"Del",
|
|
"Else",
|
|
"ExceptHandler",
|
|
"ExceptStarHandler",
|
|
"Expr",
|
|
"Finally",
|
|
"For",
|
|
"FunctionDef",
|
|
"Global",
|
|
"If",
|
|
"Import",
|
|
"ImportAlias",
|
|
"ImportFrom",
|
|
"IndentedBlock",
|
|
"Match",
|
|
"MatchCase",
|
|
"MatchAs",
|
|
"MatchClass",
|
|
"MatchKeywordElement",
|
|
"MatchList",
|
|
"MatchMapping",
|
|
"MatchMappingElement",
|
|
"MatchOr",
|
|
"MatchOrElement",
|
|
"MatchPattern",
|
|
"MatchSequence",
|
|
"MatchSequenceElement",
|
|
"MatchSingleton",
|
|
"MatchStar",
|
|
"MatchTuple",
|
|
"MatchValue",
|
|
"NameItem",
|
|
"Nonlocal",
|
|
"Pass",
|
|
"Raise",
|
|
"Return",
|
|
"SimpleStatementLine",
|
|
"SimpleStatementSuite",
|
|
"Try",
|
|
"TryStar",
|
|
"While",
|
|
"With",
|
|
"WithItem",
|
|
"BaseParenthesizableWhitespace",
|
|
"Comment",
|
|
"EmptyLine",
|
|
"Newline",
|
|
"ParenthesizedWhitespace",
|
|
"SimpleWhitespace",
|
|
"TrailingWhitespace",
|
|
"BaseMetadataProvider",
|
|
"BatchableMetadataProvider",
|
|
"VisitorMetadataProvider",
|
|
"MetadataDependent",
|
|
"MetadataWrapper",
|
|
"TypeVar",
|
|
"TypeVarTuple",
|
|
"ParamSpec",
|
|
"TypeParam",
|
|
"TypeParameters",
|
|
"TypeAlias",
|
|
]
|