mirror of
				https://github.com/astral-sh/ruff.git
				synced 2025-10-31 03:55:09 +00:00 
			
		
		
		
	 008bbfdf5a
			
		
	
	
		008bbfdf5a
		
			
		
	
	
	
		
			
	
		
	
	
		
			Some checks are pending
		
		
	
	CI / Determine changes (push) Waiting to run
				
			CI / cargo fmt (push) Waiting to run
				
			CI / cargo clippy (push) Blocked by required conditions
				
			CI / cargo test (linux) (push) Blocked by required conditions
				
			CI / cargo test (linux, release) (push) Blocked by required conditions
				
			CI / cargo test (windows) (push) Blocked by required conditions
				
			CI / cargo test (wasm) (push) Blocked by required conditions
				
			CI / cargo build (release) (push) Waiting to run
				
			CI / cargo build (msrv) (push) Blocked by required conditions
				
			CI / cargo fuzz build (push) Blocked by required conditions
				
			CI / fuzz parser (push) Blocked by required conditions
				
			CI / test scripts (push) Blocked by required conditions
				
			CI / ecosystem (push) Blocked by required conditions
				
			CI / Fuzz for new ty panics (push) Blocked by required conditions
				
			CI / cargo shear (push) Blocked by required conditions
				
			CI / python package (push) Waiting to run
				
			CI / pre-commit (push) Waiting to run
				
			CI / mkdocs (push) Waiting to run
				
			CI / formatter instabilities and black similarity (push) Blocked by required conditions
				
			CI / test ruff-lsp (push) Blocked by required conditions
				
			CI / check playground (push) Blocked by required conditions
				
			CI / benchmarks-instrumented (push) Blocked by required conditions
				
			CI / benchmarks-walltime (push) Blocked by required conditions
				
			[ty Playground] Release / publish (push) Waiting to run
				
			As of [this cpython PR](https://github.com/python/cpython/pull/135996), it is not allowed to concatenate t-strings with non-t-strings, implicitly or explicitly. Expressions such as `"foo" t"{bar}"` are now syntax errors. This PR updates some AST nodes and parsing to reflect this change. The structural change is that `TStringPart` is no longer needed, since, as in the case of `BytesStringLiteral`, the only possibilities are that we have a single `TString` or a vector of such (representing an implicit concatenation of t-strings). This removes a level of nesting from many AST expressions (which is what all the snapshot changes reflect), and simplifies some logic in the implementation of visitors, for example. The other change of note is in the parser. When we meet an implicit concatenation of string-like literals, we now count the number of t-string literals. If these do not exhaust the total number of implicitly concatenated pieces, then we emit a syntax error. To recover from this syntax error, we encode any t-string pieces as _invalid_ string literals (which means we flag them as invalid, record their range, and record the value as `""`). Note that if at least one of the pieces is an f-string we prefer to parse the entire string as an f-string; otherwise we parse it as a string. This logic is exactly the same as how we currently treat `BytesStringLiteral` parsing and error recovery - and carries with it the same pros and cons. Finally, note that I have not implemented any changes in the implementation of the formatter. As far as I can tell, none are needed. I did change a few of the fixtures so that we are always concatenating t-strings with t-strings.
		
			
				
	
	
		
			350 lines
		
	
	
	
		
			7.9 KiB
		
	
	
	
		
			Rust
		
	
	
	
	
	
			
		
		
	
	
			350 lines
		
	
	
	
		
			7.9 KiB
		
	
	
	
		
			Rust
		
	
	
	
	
	
| use std::fmt::{Debug, Write};
 | |
| 
 | |
| use insta::assert_snapshot;
 | |
| 
 | |
| use ruff_python_ast::visitor::{
 | |
|     Visitor, walk_alias, walk_bytes_literal, walk_comprehension, walk_except_handler, walk_expr,
 | |
|     walk_f_string, walk_interpolated_string_element, walk_keyword, walk_match_case, walk_parameter,
 | |
|     walk_parameters, walk_pattern, walk_stmt, walk_string_literal, walk_t_string, walk_type_param,
 | |
|     walk_with_item,
 | |
| };
 | |
| use ruff_python_ast::{
 | |
|     self as ast, Alias, AnyNodeRef, BoolOp, BytesLiteral, CmpOp, Comprehension, ExceptHandler,
 | |
|     Expr, FString, InterpolatedStringElement, Keyword, MatchCase, Operator, Parameter, Parameters,
 | |
|     Pattern, Stmt, StringLiteral, TString, TypeParam, UnaryOp, WithItem,
 | |
| };
 | |
| use ruff_python_parser::{Mode, ParseOptions, parse};
 | |
| 
 | |
| #[test]
 | |
| fn function_arguments() {
 | |
|     let source = r"def a(b, c,/, d, e = 20, *args, named=5, other=20, **kwargs): pass";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn function_positional_only_with_default() {
 | |
|     let source = r"def a(b, c = 34,/, e = 20, *args): pass";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn compare() {
 | |
|     let source = r"4 < x < 5";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn list_comprehension() {
 | |
|     let source = "[x for x in numbers]";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn dict_comprehension() {
 | |
|     let source = "{x: x**2 for x in numbers}";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn set_comprehension() {
 | |
|     let source = "{x for x in numbers}";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn match_class_pattern() {
 | |
|     let source = r"
 | |
| match x:
 | |
|     case Point2D(0, 0):
 | |
|         ...
 | |
|     case Point3D(x=0, y=0, z=0):
 | |
|         ...
 | |
| ";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn decorators() {
 | |
|     let source = r"
 | |
| @decorator
 | |
| def a():
 | |
|     pass
 | |
| 
 | |
| @test
 | |
| class A:
 | |
|     pass
 | |
| ";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn type_aliases() {
 | |
|     let source = r"type X[T: str, U, *Ts, **P] = list[T]";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn class_type_parameters() {
 | |
|     let source = r"class X[T: str, U, *Ts, **P]: ...";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn function_type_parameters() {
 | |
|     let source = r"def X[T: str, U, *Ts, **P](): ...";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn string_literals() {
 | |
|     let source = r"'a' 'b' 'c'";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn bytes_literals() {
 | |
|     let source = r"b'a' b'b' b'c'";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn f_strings() {
 | |
|     let source = r"'pre' f'foo {bar:.{x}f} baz'";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| #[test]
 | |
| fn t_strings() {
 | |
|     let source = r"t'pre' t'foo {bar:.{x}f} baz'";
 | |
| 
 | |
|     let trace = trace_visitation(source);
 | |
| 
 | |
|     assert_snapshot!(trace);
 | |
| }
 | |
| 
 | |
| fn trace_visitation(source: &str) -> String {
 | |
|     let parsed = parse(source, ParseOptions::from(Mode::Module)).unwrap();
 | |
| 
 | |
|     let mut visitor = RecordVisitor::default();
 | |
|     walk_module(&mut visitor, parsed.syntax());
 | |
| 
 | |
|     visitor.output
 | |
| }
 | |
| 
 | |
| fn walk_module<'a, V>(visitor: &mut V, module: &'a ast::Mod)
 | |
| where
 | |
|     V: Visitor<'a> + ?Sized,
 | |
| {
 | |
|     match module {
 | |
|         ast::Mod::Module(ast::ModModule {
 | |
|             body,
 | |
|             range: _,
 | |
|             node_index: _,
 | |
|         }) => {
 | |
|             visitor.visit_body(body);
 | |
|         }
 | |
|         ast::Mod::Expression(ast::ModExpression {
 | |
|             body,
 | |
|             range: _,
 | |
|             node_index: _,
 | |
|         }) => visitor.visit_expr(body),
 | |
|     }
 | |
| }
 | |
| 
 | |
| /// Emits a `tree` with a node for every visited AST node (labelled by the AST node's kind)
 | |
| /// and leaves for attributes.
 | |
| #[derive(Default)]
 | |
| struct RecordVisitor {
 | |
|     depth: usize,
 | |
|     output: String,
 | |
| }
 | |
| 
 | |
| impl RecordVisitor {
 | |
|     fn enter_node<'a, T>(&mut self, node: T)
 | |
|     where
 | |
|         T: Into<AnyNodeRef<'a>>,
 | |
|     {
 | |
|         self.emit(&node.into().kind());
 | |
|         self.depth += 1;
 | |
|     }
 | |
| 
 | |
|     fn exit_node(&mut self) {
 | |
|         self.depth -= 1;
 | |
|     }
 | |
| 
 | |
|     fn emit(&mut self, text: &dyn Debug) {
 | |
|         for _ in 0..self.depth {
 | |
|             self.output.push_str("  ");
 | |
|         }
 | |
| 
 | |
|         writeln!(self.output, "- {text:?}").unwrap();
 | |
|     }
 | |
| }
 | |
| 
 | |
| impl Visitor<'_> for RecordVisitor {
 | |
|     fn visit_stmt(&mut self, stmt: &Stmt) {
 | |
|         self.enter_node(stmt);
 | |
|         walk_stmt(self, stmt);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_annotation(&mut self, expr: &Expr) {
 | |
|         self.enter_node(expr);
 | |
|         walk_expr(self, expr);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_expr(&mut self, expr: &Expr) {
 | |
|         self.enter_node(expr);
 | |
|         walk_expr(self, expr);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_bool_op(&mut self, bool_op: &BoolOp) {
 | |
|         self.emit(&bool_op);
 | |
|     }
 | |
| 
 | |
|     fn visit_operator(&mut self, operator: &Operator) {
 | |
|         self.emit(&operator);
 | |
|     }
 | |
| 
 | |
|     fn visit_unary_op(&mut self, unary_op: &UnaryOp) {
 | |
|         self.emit(&unary_op);
 | |
|     }
 | |
| 
 | |
|     fn visit_cmp_op(&mut self, cmp_op: &CmpOp) {
 | |
|         self.emit(&cmp_op);
 | |
|     }
 | |
| 
 | |
|     fn visit_comprehension(&mut self, comprehension: &Comprehension) {
 | |
|         self.enter_node(comprehension);
 | |
|         walk_comprehension(self, comprehension);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_except_handler(&mut self, except_handler: &ExceptHandler) {
 | |
|         self.enter_node(except_handler);
 | |
|         walk_except_handler(self, except_handler);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_parameters(&mut self, parameters: &Parameters) {
 | |
|         self.enter_node(parameters);
 | |
|         walk_parameters(self, parameters);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_parameter(&mut self, parameter: &Parameter) {
 | |
|         self.enter_node(parameter);
 | |
|         walk_parameter(self, parameter);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_keyword(&mut self, keyword: &Keyword) {
 | |
|         self.enter_node(keyword);
 | |
|         walk_keyword(self, keyword);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_alias(&mut self, alias: &Alias) {
 | |
|         self.enter_node(alias);
 | |
|         walk_alias(self, alias);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_with_item(&mut self, with_item: &WithItem) {
 | |
|         self.enter_node(with_item);
 | |
|         walk_with_item(self, with_item);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_match_case(&mut self, match_case: &MatchCase) {
 | |
|         self.enter_node(match_case);
 | |
|         walk_match_case(self, match_case);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_pattern(&mut self, pattern: &Pattern) {
 | |
|         self.enter_node(pattern);
 | |
|         walk_pattern(self, pattern);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_type_param(&mut self, type_param: &TypeParam) {
 | |
|         self.enter_node(type_param);
 | |
|         walk_type_param(self, type_param);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_string_literal(&mut self, string_literal: &StringLiteral) {
 | |
|         self.enter_node(string_literal);
 | |
|         walk_string_literal(self, string_literal);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_bytes_literal(&mut self, bytes_literal: &BytesLiteral) {
 | |
|         self.enter_node(bytes_literal);
 | |
|         walk_bytes_literal(self, bytes_literal);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_f_string(&mut self, f_string: &FString) {
 | |
|         self.enter_node(f_string);
 | |
|         walk_f_string(self, f_string);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_interpolated_string_element(&mut self, f_string_element: &InterpolatedStringElement) {
 | |
|         self.enter_node(f_string_element);
 | |
|         walk_interpolated_string_element(self, f_string_element);
 | |
|         self.exit_node();
 | |
|     }
 | |
| 
 | |
|     fn visit_t_string(&mut self, t_string: &TString) {
 | |
|         self.enter_node(t_string);
 | |
|         walk_t_string(self, t_string);
 | |
|         self.exit_node();
 | |
|     }
 | |
| }
 |