Split Constant to individual literal nodes (#8064)

## Summary

This PR splits the `Constant` enum as individual literal nodes. It
introduces the following new nodes for each variant:
* `ExprStringLiteral`
* `ExprBytesLiteral`
* `ExprNumberLiteral`
* `ExprBooleanLiteral`
* `ExprNoneLiteral`
* `ExprEllipsisLiteral`

The main motivation behind this refactor is to introduce the new AST
node for implicit string concatenation in the coming PR. The elements of
that node will be either a string literal, bytes literal or a f-string
which can be implemented using an enum. This means that a string or
bytes literal cannot be represented by `Constant::Str` /
`Constant::Bytes` which creates an inconsistency.

This PR avoids that inconsistency by splitting the constant nodes into
it's own literal nodes, literal being the more appropriate naming
convention from a static analysis tool perspective.

This also makes working with literals in the linter and formatter much
more ergonomic like, for example, if one would want to check if this is
a string literal, it can be done easily using
`Expr::is_string_literal_expr` or matching against `Expr::StringLiteral`
as oppose to matching against the `ExprConstant` and enum `Constant`. A
few AST helper methods can be simplified as well which will be done in a
follow-up PR.

This introduces a new `Expr::is_literal_expr` method which is the same
as `Expr::is_constant_expr`. There are also intermediary changes related
to implicit string concatenation which are quiet less. This is done so
as to avoid having a huge PR which this already is.

## Test Plan

1. Verify and update all of the existing snapshots (parser, visitor)
2. Verify that the ecosystem check output remains **unchanged** for both
the linter and formatter

### Formatter ecosystem check

#### `main`

| project | similarity index | total files | changed files |

|----------------|------------------:|------------------:|------------------:|
| cpython | 0.75803 | 1799 | 1647 |
| django | 0.99983 | 2772 | 34 |
| home-assistant | 0.99953 | 10596 | 186 |
| poetry | 0.99891 | 317 | 17 |
| transformers | 0.99966 | 2657 | 330 |
| twine | 1.00000 | 33 | 0 |
| typeshed | 0.99978 | 3669 | 20 |
| warehouse | 0.99977 | 654 | 13 |
| zulip | 0.99970 | 1459 | 22 |

#### `dhruv/constant-to-literal`

| project | similarity index | total files | changed files |

|----------------|------------------:|------------------:|------------------:|
| cpython | 0.75803 | 1799 | 1647 |
| django | 0.99983 | 2772 | 34 |
| home-assistant | 0.99953 | 10596 | 186 |
| poetry | 0.99891 | 317 | 17 |
| transformers | 0.99966 | 2657 | 330 |
| twine | 1.00000 | 33 | 0 |
| typeshed | 0.99978 | 3669 | 20 |
| warehouse | 0.99977 | 654 | 13 |
| zulip | 0.99970 | 1459 | 22 |
This commit is contained in:
Dhruv Manilawala 2023-10-30 12:13:23 +05:30 committed by GitHub
parent 78bbf6d403
commit 230c9ce236
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
268 changed files with 6663 additions and 6741 deletions

View file

@ -4,7 +4,9 @@ use bitflags::bitflags;
use ruff_formatter::{format_args, write, FormatError};
use ruff_python_ast::AnyNodeRef;
use ruff_python_ast::{self as ast, Constant, ExprConstant, ExprFString, ExpressionRef};
use ruff_python_ast::{
self as ast, ExprBytesLiteral, ExprFString, ExprStringLiteral, ExpressionRef,
};
use ruff_python_parser::lexer::{lex_starts_at, LexicalError, LexicalErrorType};
use ruff_python_parser::{Mode, Tok};
use ruff_source_file::Locator;
@ -26,19 +28,16 @@ enum Quoting {
#[derive(Clone, Debug)]
pub(super) enum AnyString<'a> {
Constant(&'a ExprConstant),
String(&'a ExprStringLiteral),
Bytes(&'a ExprBytesLiteral),
FString(&'a ExprFString),
}
impl<'a> AnyString<'a> {
pub(crate) fn from_expression(expression: &'a Expr) -> Option<AnyString<'a>> {
match expression {
Expr::Constant(
constant @ ExprConstant {
value: Constant::Str(_) | Constant::Bytes(_),
..
},
) => Some(AnyString::Constant(constant)),
Expr::StringLiteral(string) => Some(AnyString::String(string)),
Expr::BytesLiteral(bytes) => Some(AnyString::Bytes(bytes)),
Expr::FString(fstring) => Some(AnyString::FString(fstring)),
_ => None,
}
@ -46,7 +45,7 @@ impl<'a> AnyString<'a> {
fn quoting(&self, locator: &Locator) -> Quoting {
match self {
Self::Constant(_) => Quoting::CanChange,
Self::String(_) | Self::Bytes(_) => Quoting::CanChange,
Self::FString(f_string) => {
let unprefixed = locator
.slice(f_string.range)
@ -75,7 +74,14 @@ impl<'a> AnyString<'a> {
/// Returns `true` if the string is implicitly concatenated.
pub(super) fn is_implicit_concatenated(&self) -> bool {
match self {
Self::Constant(ExprConstant { value, .. }) => value.is_implicit_concatenated(),
Self::String(ExprStringLiteral {
implicit_concatenated,
..
}) => *implicit_concatenated,
Self::Bytes(ExprBytesLiteral {
implicit_concatenated,
..
}) => *implicit_concatenated,
Self::FString(ExprFString {
implicit_concatenated,
..
@ -87,7 +93,8 @@ impl<'a> AnyString<'a> {
impl Ranged for AnyString<'_> {
fn range(&self) -> TextRange {
match self {
Self::Constant(expr) => expr.range(),
Self::String(expr) => expr.range(),
Self::Bytes(expr) => expr.range(),
Self::FString(expr) => expr.range(),
}
}
@ -96,7 +103,8 @@ impl Ranged for AnyString<'_> {
impl<'a> From<&AnyString<'a>> for AnyNodeRef<'a> {
fn from(value: &AnyString<'a>) -> Self {
match value {
AnyString::Constant(expr) => AnyNodeRef::ExprConstant(expr),
AnyString::String(expr) => AnyNodeRef::ExprStringLiteral(expr),
AnyString::Bytes(expr) => AnyNodeRef::ExprBytesLiteral(expr),
AnyString::FString(expr) => AnyNodeRef::ExprFString(expr),
}
}
@ -105,7 +113,8 @@ impl<'a> From<&AnyString<'a>> for AnyNodeRef<'a> {
impl<'a> From<&AnyString<'a>> for ExpressionRef<'a> {
fn from(value: &AnyString<'a>) -> Self {
match value {
AnyString::Constant(expr) => ExpressionRef::Constant(expr),
AnyString::String(expr) => ExpressionRef::StringLiteral(expr),
AnyString::Bytes(expr) => ExpressionRef::BytesLiteral(expr),
AnyString::FString(expr) => ExpressionRef::FString(expr),
}
}
@ -130,9 +139,6 @@ pub enum StringLayout {
impl<'a> FormatString<'a> {
pub(super) fn new(string: &'a AnyString<'a>) -> Self {
if let AnyString::Constant(constant) = string {
debug_assert!(constant.value.is_str() || constant.value.is_bytes());
}
Self {
string,
layout: StringLayout::Default,
@ -247,9 +253,6 @@ struct FormatStringContinuation<'a> {
impl<'a> FormatStringContinuation<'a> {
fn new(string: &'a AnyString<'a>) -> Self {
if let AnyString::Constant(constant) = string {
debug_assert!(constant.value.is_str() || constant.value.is_bytes());
}
Self { string }
}
}