ruff/crates/ruff_python_ast/tests/preorder.rs
Dhruv Manilawala 230c9ce236
Split Constant to individual literal nodes (#8064)
## Summary

This PR splits the `Constant` enum as individual literal nodes. It
introduces the following new nodes for each variant:
* `ExprStringLiteral`
* `ExprBytesLiteral`
* `ExprNumberLiteral`
* `ExprBooleanLiteral`
* `ExprNoneLiteral`
* `ExprEllipsisLiteral`

The main motivation behind this refactor is to introduce the new AST
node for implicit string concatenation in the coming PR. The elements of
that node will be either a string literal, bytes literal or a f-string
which can be implemented using an enum. This means that a string or
bytes literal cannot be represented by `Constant::Str` /
`Constant::Bytes` which creates an inconsistency.

This PR avoids that inconsistency by splitting the constant nodes into
it's own literal nodes, literal being the more appropriate naming
convention from a static analysis tool perspective.

This also makes working with literals in the linter and formatter much
more ergonomic like, for example, if one would want to check if this is
a string literal, it can be done easily using
`Expr::is_string_literal_expr` or matching against `Expr::StringLiteral`
as oppose to matching against the `ExprConstant` and enum `Constant`. A
few AST helper methods can be simplified as well which will be done in a
follow-up PR.

This introduces a new `Expr::is_literal_expr` method which is the same
as `Expr::is_constant_expr`. There are also intermediary changes related
to implicit string concatenation which are quiet less. This is done so
as to avoid having a huge PR which this already is.

## Test Plan

1. Verify and update all of the existing snapshots (parser, visitor)
2. Verify that the ecosystem check output remains **unchanged** for both
the linter and formatter

### Formatter ecosystem check

#### `main`

| project | similarity index | total files | changed files |

|----------------|------------------:|------------------:|------------------:|
| cpython | 0.75803 | 1799 | 1647 |
| django | 0.99983 | 2772 | 34 |
| home-assistant | 0.99953 | 10596 | 186 |
| poetry | 0.99891 | 317 | 17 |
| transformers | 0.99966 | 2657 | 330 |
| twine | 1.00000 | 33 | 0 |
| typeshed | 0.99978 | 3669 | 20 |
| warehouse | 0.99977 | 654 | 13 |
| zulip | 0.99970 | 1459 | 22 |

#### `dhruv/constant-to-literal`

| project | similarity index | total files | changed files |

|----------------|------------------:|------------------:|------------------:|
| cpython | 0.75803 | 1799 | 1647 |
| django | 0.99983 | 2772 | 34 |
| home-assistant | 0.99953 | 10596 | 186 |
| poetry | 0.99891 | 317 | 17 |
| transformers | 0.99966 | 2657 | 330 |
| twine | 1.00000 | 33 | 0 |
| typeshed | 0.99978 | 3669 | 20 |
| warehouse | 0.99977 | 654 | 13 |
| zulip | 0.99970 | 1459 | 22 |
2023-10-30 12:13:23 +05:30

281 lines
6.4 KiB
Rust

use std::fmt::{Debug, Write};
use insta::assert_snapshot;
use ruff_python_ast::visitor::preorder::{
walk_alias, walk_comprehension, walk_except_handler, walk_expr, walk_keyword, walk_match_case,
walk_module, walk_parameter, walk_parameters, walk_pattern, walk_stmt, walk_type_param,
walk_with_item, PreorderVisitor,
};
use ruff_python_ast::AnyNodeRef;
use ruff_python_ast::{
Alias, BoolOp, CmpOp, Comprehension, ExceptHandler, Expr, Keyword, MatchCase, Mod, Operator,
Parameter, Parameters, Pattern, Singleton, Stmt, TypeParam, UnaryOp, WithItem,
};
use ruff_python_parser::lexer::lex;
use ruff_python_parser::{parse_tokens, Mode};
#[test]
fn function_arguments() {
let source = r#"def a(b, c,/, d, e = 20, *args, named=5, other=20, **kwargs): pass"#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn function_positional_only_with_default() {
let source = r#"def a(b, c = 34,/, e = 20, *args): pass"#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn compare() {
let source = r#"4 < x < 5"#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn list_comprehension() {
let source = "[x for x in numbers]";
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn dict_comprehension() {
let source = "{x: x**2 for x in numbers}";
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn set_comprehension() {
let source = "{x for x in numbers}";
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn match_class_pattern() {
let source = r#"
match x:
case Point2D(0, 0):
...
case Point3D(x=0, y=0, z=0):
...
"#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn decorators() {
let source = r#"
@decorator
def a():
pass
@test
class A:
pass
"#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn type_aliases() {
let source = r#"type X[T: str, U, *Ts, **P] = list[T]"#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn class_type_parameters() {
let source = r#"class X[T: str, U, *Ts, **P]: ..."#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
#[test]
fn function_type_parameters() {
let source = r#"def X[T: str, U, *Ts, **P](): ..."#;
let trace = trace_preorder_visitation(source);
assert_snapshot!(trace);
}
fn trace_preorder_visitation(source: &str) -> String {
let tokens = lex(source, Mode::Module);
let parsed = parse_tokens(tokens, source, Mode::Module, "test.py").unwrap();
let mut visitor = RecordVisitor::default();
visitor.visit_mod(&parsed);
visitor.output
}
/// Emits a `tree` with a node for every visited AST node (labelled by the AST node's kind)
/// and leafs for attributes.
#[derive(Default)]
struct RecordVisitor {
depth: usize,
output: String,
}
impl RecordVisitor {
fn enter_node<'a, T>(&mut self, node: T)
where
T: Into<AnyNodeRef<'a>>,
{
self.emit(&node.into().kind());
self.depth += 1;
}
fn exit_node(&mut self) {
self.depth -= 1;
}
fn emit(&mut self, text: &dyn Debug) {
for _ in 0..self.depth {
self.output.push_str(" ");
}
writeln!(self.output, "- {text:?}").unwrap();
}
}
impl PreorderVisitor<'_> for RecordVisitor {
fn visit_mod(&mut self, module: &Mod) {
self.enter_node(module);
walk_module(self, module);
self.exit_node();
}
fn visit_stmt(&mut self, stmt: &Stmt) {
self.enter_node(stmt);
walk_stmt(self, stmt);
self.exit_node();
}
fn visit_annotation(&mut self, expr: &Expr) {
self.enter_node(expr);
walk_expr(self, expr);
self.exit_node();
}
fn visit_expr(&mut self, expr: &Expr) {
self.enter_node(expr);
walk_expr(self, expr);
self.exit_node();
}
fn visit_singleton(&mut self, singleton: &Singleton) {
self.emit(&singleton);
}
fn visit_bool_op(&mut self, bool_op: &BoolOp) {
self.emit(&bool_op);
}
fn visit_operator(&mut self, operator: &Operator) {
self.emit(&operator);
}
fn visit_unary_op(&mut self, unary_op: &UnaryOp) {
self.emit(&unary_op);
}
fn visit_cmp_op(&mut self, cmp_op: &CmpOp) {
self.emit(&cmp_op);
}
fn visit_comprehension(&mut self, comprehension: &Comprehension) {
self.enter_node(comprehension);
walk_comprehension(self, comprehension);
self.exit_node();
}
fn visit_except_handler(&mut self, except_handler: &ExceptHandler) {
self.enter_node(except_handler);
walk_except_handler(self, except_handler);
self.exit_node();
}
fn visit_format_spec(&mut self, format_spec: &Expr) {
self.enter_node(format_spec);
walk_expr(self, format_spec);
self.exit_node();
}
fn visit_parameters(&mut self, parameters: &Parameters) {
self.enter_node(parameters);
walk_parameters(self, parameters);
self.exit_node();
}
fn visit_parameter(&mut self, parameter: &Parameter) {
self.enter_node(parameter);
walk_parameter(self, parameter);
self.exit_node();
}
fn visit_keyword(&mut self, keyword: &Keyword) {
self.enter_node(keyword);
walk_keyword(self, keyword);
self.exit_node();
}
fn visit_alias(&mut self, alias: &Alias) {
self.enter_node(alias);
walk_alias(self, alias);
self.exit_node();
}
fn visit_with_item(&mut self, with_item: &WithItem) {
self.enter_node(with_item);
walk_with_item(self, with_item);
self.exit_node();
}
fn visit_match_case(&mut self, match_case: &MatchCase) {
self.enter_node(match_case);
walk_match_case(self, match_case);
self.exit_node();
}
fn visit_pattern(&mut self, pattern: &Pattern) {
self.enter_node(pattern);
walk_pattern(self, pattern);
self.exit_node();
}
fn visit_type_param(&mut self, type_param: &TypeParam) {
self.enter_node(type_param);
walk_type_param(self, type_param);
self.exit_node();
}
}