Remove cyclic dev dependency with the parser crate (#11261)

## Summary

This PR removes the cyclic dev dependency some of the crates had with
the parser crate.

The cyclic dependencies are:
* `ruff_python_ast` has a **dev dependency** on `ruff_python_parser` and
`ruff_python_parser` directly depends on `ruff_python_ast`
* `ruff_python_trivia` has a **dev dependency** on `ruff_python_parser`
and `ruff_python_parser` has an indirect dependency on
`ruff_python_trivia` (`ruff_python_parser` - `ruff_python_ast` -
`ruff_python_trivia`)

Specifically, this PR does the following:
* Introduce two new crates
* `ruff_python_ast_integration_tests` and move the tests from the
`ruff_python_ast` crate which uses the parser in this crate
* `ruff_python_trivia_integration_tests` and move the tests from the
`ruff_python_trivia` crate which uses the parser in this crate

### Motivation

The main motivation for this PR is to help development. Before this PR,
`rust-analyzer` wouldn't provide any intellisense in the
`ruff_python_parser` crate regarding the symbols in `ruff_python_ast`
crate.

```
[ERROR][2024-05-03 13:47:06] .../vim/lsp/rpc.lua:770	"rpc"	"/Users/dhruv/.cargo/bin/rust-analyzer"	"stderr"	"[ERROR project_model::workspace] cyclic deps: ruff_python_parser(Idx::<CrateData>(50)) -> ruff_python_ast(Idx::<CrateData>(37)), alternative path: ruff_python_ast(Idx::<CrateData>(37)) -> ruff_python_parser(Idx::<CrateData>(50))\n"
```

## Test Plan

Check the logs of `rust-analyzer` to not see any signs of cyclic
dependency.

crates/ruff_python_ast_integration_tests/tests/identifier.rs

@@ -0,0 +1,23 @@
+use ruff_python_ast::identifier;
+use ruff_python_parser::{parse_suite, ParseError};
+use ruff_text_size::{TextRange, TextSize};
+
+#[test]
+fn extract_else_range() -> Result<(), ParseError> {
+    let contents = r"
+for x in y:
+    pass
+else:
+    pass
+"
+    .trim();
+    let stmts = parse_suite(contents)?;
+    let stmt = stmts.first().unwrap();
+    let range = identifier::else_(stmt, contents).unwrap();
+    assert_eq!(&contents[range], "else");
+    assert_eq!(
+        range,
+        TextRange::new(TextSize::from(21), TextSize::from(25))
+    );
+    Ok(())
+}

crates/ruff_python_ast_integration_tests/tests/parenthesize.rs

@@ -0,0 +1,143 @@
+use ruff_python_ast::parenthesize::parenthesized_range;
+use ruff_python_parser::parse_expression;
+use ruff_python_trivia::CommentRanges;
+use ruff_text_size::TextRange;
+
+#[test]
+fn test_parenthesized_name() {
+    let source_code = r"(x) + 1";
+    let expr = parse_expression(source_code).unwrap();
+
+    let bin_op = expr.as_bin_op_expr().unwrap();
+    let name = bin_op.left.as_ref();
+
+    let parenthesized = parenthesized_range(
+        name.into(),
+        bin_op.into(),
+        &CommentRanges::default(),
+        source_code,
+    );
+    assert_eq!(parenthesized, Some(TextRange::new(0.into(), 3.into())));
+}
+
+#[test]
+fn test_non_parenthesized_name() {
+    let source_code = r"x + 1";
+    let expr = parse_expression(source_code).unwrap();
+
+    let bin_op = expr.as_bin_op_expr().unwrap();
+    let name = bin_op.left.as_ref();
+
+    let parenthesized = parenthesized_range(
+        name.into(),
+        bin_op.into(),
+        &CommentRanges::default(),
+        source_code,
+    );
+    assert_eq!(parenthesized, None);
+}
+
+#[test]
+fn test_parenthesized_argument() {
+    let source_code = r"f((a))";
+    let expr = parse_expression(source_code).unwrap();
+
+    let call = expr.as_call_expr().unwrap();
+    let arguments = &call.arguments;
+    let argument = arguments.args.first().unwrap();
+
+    let parenthesized = parenthesized_range(
+        argument.into(),
+        arguments.into(),
+        &CommentRanges::default(),
+        source_code,
+    );
+    assert_eq!(parenthesized, Some(TextRange::new(2.into(), 5.into())));
+}
+
+#[test]
+fn test_non_parenthesized_argument() {
+    let source_code = r"f(a)";
+    let expr = parse_expression(source_code).unwrap();
+
+    let call = expr.as_call_expr().unwrap();
+    let arguments = &call.arguments;
+    let argument = arguments.args.first().unwrap();
+
+    let parenthesized = parenthesized_range(
+        argument.into(),
+        arguments.into(),
+        &CommentRanges::default(),
+        source_code,
+    );
+    assert_eq!(parenthesized, None);
+}
+
+#[test]
+fn test_parenthesized_tuple_member() {
+    let source_code = r"(a, (b))";
+    let expr = parse_expression(source_code).unwrap();
+
+    let tuple = expr.as_tuple_expr().unwrap();
+    let member = tuple.elts.last().unwrap();
+
+    let parenthesized = parenthesized_range(
+        member.into(),
+        tuple.into(),
+        &CommentRanges::default(),
+        source_code,
+    );
+    assert_eq!(parenthesized, Some(TextRange::new(4.into(), 7.into())));
+}
+
+#[test]
+fn test_non_parenthesized_tuple_member() {
+    let source_code = r"(a, b)";
+    let expr = parse_expression(source_code).unwrap();
+
+    let tuple = expr.as_tuple_expr().unwrap();
+    let member = tuple.elts.last().unwrap();
+
+    let parenthesized = parenthesized_range(
+        member.into(),
+        tuple.into(),
+        &CommentRanges::default(),
+        source_code,
+    );
+    assert_eq!(parenthesized, None);
+}
+
+#[test]
+fn test_twice_parenthesized_name() {
+    let source_code = r"((x)) + 1";
+    let expr = parse_expression(source_code).unwrap();
+
+    let bin_op = expr.as_bin_op_expr().unwrap();
+    let name = bin_op.left.as_ref();
+
+    let parenthesized = parenthesized_range(
+        name.into(),
+        bin_op.into(),
+        &CommentRanges::default(),
+        source_code,
+    );
+    assert_eq!(parenthesized, Some(TextRange::new(0.into(), 5.into())));
+}
+
+#[test]
+fn test_twice_parenthesized_argument() {
+    let source_code = r"f(((a + 1)))";
+    let expr = parse_expression(source_code).unwrap();
+
+    let call = expr.as_call_expr().unwrap();
+    let arguments = &call.arguments;
+    let argument = arguments.args.first().unwrap();
+
+    let parenthesized = parenthesized_range(
+        argument.into(),
+        arguments.into(),
+        &CommentRanges::default(),
+        source_code,
+    );
+    assert_eq!(parenthesized, Some(TextRange::new(2.into(), 11.into())));
+}

crates/ruff_python_ast_integration_tests/tests/preorder.rs

@@ -0,0 +1,209 @@
+use std::fmt::{Debug, Write};
+
+use insta::assert_snapshot;
+
+use ruff_python_ast::visitor::preorder::{PreorderVisitor, TraversalSignal};
+use ruff_python_ast::{AnyNodeRef, BoolOp, CmpOp, Operator, Singleton, UnaryOp};
+use ruff_python_parser::lexer::lex;
+use ruff_python_parser::{parse_tokens, Mode};
+
+#[test]
+fn function_arguments() {
+    let source = r"def a(b, c,/, d, e = 20, *args, named=5, other=20, **kwargs): pass";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn function_positional_only_with_default() {
+    let source = r"def a(b, c = 34,/, e = 20, *args): pass";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn compare() {
+    let source = r"4 < x < 5";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn list_comprehension() {
+    let source = "[x for x in numbers]";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn dict_comprehension() {
+    let source = "{x: x**2 for x in numbers}";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn set_comprehension() {
+    let source = "{x for x in numbers}";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn match_class_pattern() {
+    let source = r"
+match x:
+    case Point2D(0, 0):
+        ...
+    case Point3D(x=0, y=0, z=0):
+        ...
+";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn decorators() {
+    let source = r"
+@decorator
+def a():
+    pass
+
+@test
+class A:
+    pass
+";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn type_aliases() {
+    let source = r"type X[T: str, U, *Ts, **P] = list[T]";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn class_type_parameters() {
+    let source = r"class X[T: str, U, *Ts, **P]: ...";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn function_type_parameters() {
+    let source = r"def X[T: str, U, *Ts, **P](): ...";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn string_literals() {
+    let source = r"'a' 'b' 'c'";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn bytes_literals() {
+    let source = r"b'a' b'b' b'c'";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn f_strings() {
+    let source = r"'pre' f'foo {bar:.{x}f} baz'";
+
+    let trace = trace_preorder_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+fn trace_preorder_visitation(source: &str) -> String {
+    let tokens = lex(source, Mode::Module);
+    let parsed = parse_tokens(tokens.collect(), source, Mode::Module).unwrap();
+
+    let mut visitor = RecordVisitor::default();
+    visitor.visit_mod(&parsed);
+
+    visitor.output
+}
+
+/// Emits a `tree` with a node for every visited AST node (labelled by the AST node's kind)
+/// and leaves for attributes.
+#[derive(Default)]
+struct RecordVisitor {
+    depth: usize,
+    output: String,
+}
+
+impl RecordVisitor {
+    fn emit(&mut self, text: &dyn Debug) {
+        for _ in 0..self.depth {
+            self.output.push_str("  ");
+        }
+
+        writeln!(self.output, "- {text:?}").unwrap();
+    }
+}
+
+impl<'a> PreorderVisitor<'a> for RecordVisitor {
+    fn enter_node(&mut self, node: AnyNodeRef<'a>) -> TraversalSignal {
+        self.emit(&node.kind());
+        self.depth += 1;
+
+        TraversalSignal::Traverse
+    }
+
+    fn leave_node(&mut self, _node: AnyNodeRef<'a>) {
+        self.depth -= 1;
+    }
+
+    fn visit_singleton(&mut self, singleton: &Singleton) {
+        self.emit(&singleton);
+    }
+
+    fn visit_bool_op(&mut self, bool_op: &BoolOp) {
+        self.emit(&bool_op);
+    }
+
+    fn visit_operator(&mut self, operator: &Operator) {
+        self.emit(&operator);
+    }
+
+    fn visit_unary_op(&mut self, unary_op: &UnaryOp) {
+        self.emit(&unary_op);
+    }
+
+    fn visit_cmp_op(&mut self, cmp_op: &CmpOp) {
+        self.emit(&cmp_op);
+    }
+}

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__bytes_literals.snap

@@ -0,0 +1,10 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtExpr
+    - ExprBytesLiteral
+      - BytesLiteral
+      - BytesLiteral
+      - BytesLiteral

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__class_type_parameters.snap

@@ -0,0 +1,14 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtClassDef
+    - TypeParams
+      - TypeParamTypeVar
+        - ExprName
+      - TypeParamTypeVar
+      - TypeParamTypeVarTuple
+      - TypeParamParamSpec
+    - StmtExpr
+      - ExprEllipsisLiteral

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__compare.snap

@@ -0,0 +1,12 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtExpr
+    - ExprCompare
+      - ExprNumberLiteral
+      - Lt
+      - ExprName
+      - Lt
+      - ExprNumberLiteral

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__decorators.snap

@@ -0,0 +1,14 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtFunctionDef
+    - Decorator
+      - ExprName
+    - Parameters
+    - StmtPass
+  - StmtClassDef
+    - Decorator
+      - ExprName
+    - StmtPass

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__dict_comprehension.snap

@@ -0,0 +1,15 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtExpr
+    - ExprDictComp
+      - ExprName
+      - ExprBinOp
+        - ExprName
+        - Pow
+        - ExprNumberLiteral
+      - Comprehension
+        - ExprName
+        - ExprName

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__f_strings.snap

@@ -0,0 +1,17 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtExpr
+    - ExprFString
+      - StringLiteral
+      - FString
+        - FStringLiteralElement
+        - FStringExpressionElement
+          - ExprName
+          - FStringLiteralElement
+          - FStringExpressionElement
+            - ExprName
+          - FStringLiteralElement
+        - FStringLiteralElement

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__function_arguments.snap

@@ -0,0 +1,25 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtFunctionDef
+    - Parameters
+      - ParameterWithDefault
+        - Parameter
+      - ParameterWithDefault
+        - Parameter
+      - ParameterWithDefault
+        - Parameter
+      - ParameterWithDefault
+        - Parameter
+        - ExprNumberLiteral
+      - Parameter
+      - ParameterWithDefault
+        - Parameter
+        - ExprNumberLiteral
+      - ParameterWithDefault
+        - Parameter
+        - ExprNumberLiteral
+      - Parameter
+    - StmtPass

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__function_positional_only_with_default.snap

@@ -0,0 +1,17 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtFunctionDef
+    - Parameters
+      - ParameterWithDefault
+        - Parameter
+      - ParameterWithDefault
+        - Parameter
+        - ExprNumberLiteral
+      - ParameterWithDefault
+        - Parameter
+        - ExprNumberLiteral
+      - Parameter
+    - StmtPass

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__function_type_parameters.snap

@@ -0,0 +1,15 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtFunctionDef
+    - TypeParams
+      - TypeParamTypeVar
+        - ExprName
+      - TypeParamTypeVar
+      - TypeParamTypeVarTuple
+      - TypeParamParamSpec
+    - Parameters
+    - StmtExpr
+      - ExprEllipsisLiteral

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__list_comprehension.snap

@@ -0,0 +1,11 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtExpr
+    - ExprListComp
+      - ExprName
+      - Comprehension
+        - ExprName
+        - ExprName

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__match_class_pattern.snap

@@ -0,0 +1,32 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtMatch
+    - ExprName
+    - MatchCase
+      - PatternMatchClass
+        - ExprName
+        - PatternArguments
+          - PatternMatchValue
+            - ExprNumberLiteral
+          - PatternMatchValue
+            - ExprNumberLiteral
+      - StmtExpr
+        - ExprEllipsisLiteral
+    - MatchCase
+      - PatternMatchClass
+        - ExprName
+        - PatternArguments
+          - PatternKeyword
+            - PatternMatchValue
+              - ExprNumberLiteral
+          - PatternKeyword
+            - PatternMatchValue
+              - ExprNumberLiteral
+          - PatternKeyword
+            - PatternMatchValue
+              - ExprNumberLiteral
+      - StmtExpr
+        - ExprEllipsisLiteral

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__set_comprehension.snap

@@ -0,0 +1,11 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtExpr
+    - ExprSetComp
+      - ExprName
+      - Comprehension
+        - ExprName
+        - ExprName

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__string_literals.snap

@@ -0,0 +1,10 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtExpr
+    - ExprStringLiteral
+      - StringLiteral
+      - StringLiteral
+      - StringLiteral

crates/ruff_python_ast_integration_tests/tests/snapshots/preorder__type_aliases.snap

@@ -0,0 +1,16 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/preorder.rs
+expression: trace
+---
+- ModModule
+  - StmtTypeAlias
+    - ExprName
+    - TypeParams
+      - TypeParamTypeVar
+        - ExprName
+      - TypeParamTypeVar
+      - TypeParamTypeVarTuple
+      - TypeParamParamSpec
+    - ExprSubscript
+      - ExprName
+      - ExprName

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__bytes_literals.snap

@@ -0,0 +1,9 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtExpr
+  - ExprBytesLiteral
+    - BytesLiteral
+    - BytesLiteral
+    - BytesLiteral

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__class_type_parameters.snap

@@ -0,0 +1,12 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtClassDef
+  - TypeParamTypeVar
+    - ExprName
+  - TypeParamTypeVar
+  - TypeParamTypeVarTuple
+  - TypeParamParamSpec
+  - StmtExpr
+    - ExprEllipsisLiteral

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__compare.snap

@@ -0,0 +1,11 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtExpr
+  - ExprCompare
+    - ExprNumberLiteral
+    - Lt
+    - Lt
+    - ExprName
+    - ExprNumberLiteral

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__decorators.snap

@@ -0,0 +1,11 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtFunctionDef
+  - ExprName
+  - Parameters
+  - StmtPass
+- StmtClassDef
+  - ExprName
+  - StmtPass

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__dict_comprehension.snap

@@ -0,0 +1,14 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtExpr
+  - ExprDictComp
+    - Comprehension
+      - ExprName
+      - ExprName
+    - ExprName
+    - ExprBinOp
+      - ExprName
+      - Pow
+      - ExprNumberLiteral

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__f_strings.snap

@@ -0,0 +1,16 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtExpr
+  - ExprFString
+    - StringLiteral
+    - FString
+      - FStringLiteralElement
+      - FStringExpressionElement
+        - ExprName
+        - FStringLiteralElement
+        - FStringExpressionElement
+          - ExprName
+        - FStringLiteralElement
+      - FStringLiteralElement

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__function_arguments.snap

@@ -0,0 +1,18 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtFunctionDef
+  - Parameters
+    - ExprNumberLiteral
+    - ExprNumberLiteral
+    - ExprNumberLiteral
+    - Parameter
+    - Parameter
+    - Parameter
+    - Parameter
+    - Parameter
+    - Parameter
+    - Parameter
+    - Parameter
+  - StmtPass

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__function_positional_only_with_default.snap

@@ -0,0 +1,13 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtFunctionDef
+  - Parameters
+    - ExprNumberLiteral
+    - ExprNumberLiteral
+    - Parameter
+    - Parameter
+    - Parameter
+    - Parameter
+  - StmtPass

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__function_type_parameters.snap

@@ -0,0 +1,13 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtFunctionDef
+  - TypeParamTypeVar
+    - ExprName
+  - TypeParamTypeVar
+  - TypeParamTypeVarTuple
+  - TypeParamParamSpec
+  - Parameters
+  - StmtExpr
+    - ExprEllipsisLiteral

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__list_comprehension.snap

@@ -0,0 +1,10 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtExpr
+  - ExprListComp
+    - Comprehension
+      - ExprName
+      - ExprName
+    - ExprName

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__match_class_pattern.snap

@@ -0,0 +1,26 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtMatch
+  - ExprName
+  - MatchCase
+    - PatternMatchClass
+      - ExprName
+      - PatternMatchValue
+        - ExprNumberLiteral
+      - PatternMatchValue
+        - ExprNumberLiteral
+    - StmtExpr
+      - ExprEllipsisLiteral
+  - MatchCase
+    - PatternMatchClass
+      - ExprName
+      - PatternMatchValue
+        - ExprNumberLiteral
+      - PatternMatchValue
+        - ExprNumberLiteral
+      - PatternMatchValue
+        - ExprNumberLiteral
+    - StmtExpr
+      - ExprEllipsisLiteral

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__set_comprehension.snap

@@ -0,0 +1,10 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtExpr
+  - ExprSetComp
+    - Comprehension
+      - ExprName
+      - ExprName
+    - ExprName

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__string_literals.snap

@@ -0,0 +1,9 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtExpr
+  - ExprStringLiteral
+    - StringLiteral
+    - StringLiteral
+    - StringLiteral

crates/ruff_python_ast_integration_tests/tests/snapshots/visitor__type_aliases.snap

@@ -0,0 +1,14 @@
+---
+source: crates/ruff_python_ast_integration_tests/tests/visitor.rs
+expression: trace
+---
+- StmtTypeAlias
+  - ExprSubscript
+    - ExprName
+    - ExprName
+  - TypeParamTypeVar
+    - ExprName
+  - TypeParamTypeVar
+  - TypeParamTypeVarTuple
+  - TypeParamParamSpec
+  - ExprName

crates/ruff_python_ast_integration_tests/tests/stmt_if.rs

@@ -0,0 +1,36 @@
+use ruff_python_ast::stmt_if::elif_else_range;
+use ruff_python_parser::{parse_suite, ParseError};
+use ruff_text_size::TextSize;
+
+#[test]
+fn extract_elif_else_range() -> Result<(), ParseError> {
+    let contents = "if a:
+    ...
+elif b:
+    ...
+";
+    let mut stmts = parse_suite(contents)?;
+    let stmt = stmts
+        .pop()
+        .and_then(ruff_python_ast::Stmt::if_stmt)
+        .unwrap();
+    let range = elif_else_range(&stmt.elif_else_clauses[0], contents).unwrap();
+    assert_eq!(range.start(), TextSize::from(14));
+    assert_eq!(range.end(), TextSize::from(18));
+
+    let contents = "if a:
+    ...
+else:
+    ...
+";
+    let mut stmts = parse_suite(contents)?;
+    let stmt = stmts
+        .pop()
+        .and_then(ruff_python_ast::Stmt::if_stmt)
+        .unwrap();
+    let range = elif_else_range(&stmt.elif_else_clauses[0], contents).unwrap();
+    assert_eq!(range.start(), TextSize::from(14));
+    assert_eq!(range.end(), TextSize::from(18));
+
+    Ok(())
+}

crates/ruff_python_ast_integration_tests/tests/visitor.rs

@@ -0,0 +1,329 @@
+use std::fmt::{Debug, Write};
+
+use insta::assert_snapshot;
+
+use ruff_python_ast::visitor::{
+    walk_alias, walk_bytes_literal, walk_comprehension, walk_except_handler, walk_expr,
+    walk_f_string, walk_f_string_element, walk_keyword, walk_match_case, walk_parameter,
+    walk_parameters, walk_pattern, walk_stmt, walk_string_literal, walk_type_param, walk_with_item,
+    Visitor,
+};
+use ruff_python_ast::{
+    self as ast, Alias, AnyNodeRef, BoolOp, BytesLiteral, CmpOp, Comprehension, ExceptHandler,
+    Expr, FString, FStringElement, Keyword, MatchCase, Operator, Parameter, Parameters, Pattern,
+    Stmt, StringLiteral, TypeParam, UnaryOp, WithItem,
+};
+use ruff_python_parser::lexer::lex;
+use ruff_python_parser::{parse_tokens, Mode};
+
+#[test]
+fn function_arguments() {
+    let source = r"def a(b, c,/, d, e = 20, *args, named=5, other=20, **kwargs): pass";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn function_positional_only_with_default() {
+    let source = r"def a(b, c = 34,/, e = 20, *args): pass";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn compare() {
+    let source = r"4 < x < 5";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn list_comprehension() {
+    let source = "[x for x in numbers]";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn dict_comprehension() {
+    let source = "{x: x**2 for x in numbers}";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn set_comprehension() {
+    let source = "{x for x in numbers}";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn match_class_pattern() {
+    let source = r"
+match x:
+    case Point2D(0, 0):
+        ...
+    case Point3D(x=0, y=0, z=0):
+        ...
+";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn decorators() {
+    let source = r"
+@decorator
+def a():
+    pass
+
+@test
+class A:
+    pass
+";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn type_aliases() {
+    let source = r"type X[T: str, U, *Ts, **P] = list[T]";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn class_type_parameters() {
+    let source = r"class X[T: str, U, *Ts, **P]: ...";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn function_type_parameters() {
+    let source = r"def X[T: str, U, *Ts, **P](): ...";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn string_literals() {
+    let source = r"'a' 'b' 'c'";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn bytes_literals() {
+    let source = r"b'a' b'b' b'c'";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+#[test]
+fn f_strings() {
+    let source = r"'pre' f'foo {bar:.{x}f} baz'";
+
+    let trace = trace_visitation(source);
+
+    assert_snapshot!(trace);
+}
+
+fn trace_visitation(source: &str) -> String {
+    let tokens = lex(source, Mode::Module);
+    let parsed = parse_tokens(tokens.collect(), source, Mode::Module).unwrap();
+
+    let mut visitor = RecordVisitor::default();
+    walk_module(&mut visitor, &parsed);
+
+    visitor.output
+}
+
+fn walk_module<'a, V>(visitor: &mut V, module: &'a ast::Mod)
+where
+    V: Visitor<'a> + ?Sized,
+{
+    match module {
+        ast::Mod::Module(ast::ModModule { body, range: _ }) => {
+            visitor.visit_body(body);
+        }
+        ast::Mod::Expression(ast::ModExpression { body, range: _ }) => visitor.visit_expr(body),
+    }
+}
+
+/// Emits a `tree` with a node for every visited AST node (labelled by the AST node's kind)
+/// and leaves for attributes.
+#[derive(Default)]
+struct RecordVisitor {
+    depth: usize,
+    output: String,
+}
+
+impl RecordVisitor {
+    fn enter_node<'a, T>(&mut self, node: T)
+    where
+        T: Into<AnyNodeRef<'a>>,
+    {
+        self.emit(&node.into().kind());
+        self.depth += 1;
+    }
+
+    fn exit_node(&mut self) {
+        self.depth -= 1;
+    }
+
+    fn emit(&mut self, text: &dyn Debug) {
+        for _ in 0..self.depth {
+            self.output.push_str("  ");
+        }
+
+        writeln!(self.output, "- {text:?}").unwrap();
+    }
+}
+
+impl Visitor<'_> for RecordVisitor {
+    fn visit_stmt(&mut self, stmt: &Stmt) {
+        self.enter_node(stmt);
+        walk_stmt(self, stmt);
+        self.exit_node();
+    }
+
+    fn visit_annotation(&mut self, expr: &Expr) {
+        self.enter_node(expr);
+        walk_expr(self, expr);
+        self.exit_node();
+    }
+
+    fn visit_expr(&mut self, expr: &Expr) {
+        self.enter_node(expr);
+        walk_expr(self, expr);
+        self.exit_node();
+    }
+
+    fn visit_bool_op(&mut self, bool_op: &BoolOp) {
+        self.emit(&bool_op);
+    }
+
+    fn visit_operator(&mut self, operator: &Operator) {
+        self.emit(&operator);
+    }
+
+    fn visit_unary_op(&mut self, unary_op: &UnaryOp) {
+        self.emit(&unary_op);
+    }
+
+    fn visit_cmp_op(&mut self, cmp_op: &CmpOp) {
+        self.emit(&cmp_op);
+    }
+
+    fn visit_comprehension(&mut self, comprehension: &Comprehension) {
+        self.enter_node(comprehension);
+        walk_comprehension(self, comprehension);
+        self.exit_node();
+    }
+
+    fn visit_except_handler(&mut self, except_handler: &ExceptHandler) {
+        self.enter_node(except_handler);
+        walk_except_handler(self, except_handler);
+        self.exit_node();
+    }
+
+    fn visit_parameters(&mut self, parameters: &Parameters) {
+        self.enter_node(parameters);
+        walk_parameters(self, parameters);
+        self.exit_node();
+    }
+
+    fn visit_parameter(&mut self, parameter: &Parameter) {
+        self.enter_node(parameter);
+        walk_parameter(self, parameter);
+        self.exit_node();
+    }
+
+    fn visit_keyword(&mut self, keyword: &Keyword) {
+        self.enter_node(keyword);
+        walk_keyword(self, keyword);
+        self.exit_node();
+    }
+
+    fn visit_alias(&mut self, alias: &Alias) {
+        self.enter_node(alias);
+        walk_alias(self, alias);
+        self.exit_node();
+    }
+
+    fn visit_with_item(&mut self, with_item: &WithItem) {
+        self.enter_node(with_item);
+        walk_with_item(self, with_item);
+        self.exit_node();
+    }
+
+    fn visit_match_case(&mut self, match_case: &MatchCase) {
+        self.enter_node(match_case);
+        walk_match_case(self, match_case);
+        self.exit_node();
+    }
+
+    fn visit_pattern(&mut self, pattern: &Pattern) {
+        self.enter_node(pattern);
+        walk_pattern(self, pattern);
+        self.exit_node();
+    }
+
+    fn visit_type_param(&mut self, type_param: &TypeParam) {
+        self.enter_node(type_param);
+        walk_type_param(self, type_param);
+        self.exit_node();
+    }
+
+    fn visit_string_literal(&mut self, string_literal: &StringLiteral) {
+        self.enter_node(string_literal);
+        walk_string_literal(self, string_literal);
+        self.exit_node();
+    }
+
+    fn visit_bytes_literal(&mut self, bytes_literal: &BytesLiteral) {
+        self.enter_node(bytes_literal);
+        walk_bytes_literal(self, bytes_literal);
+        self.exit_node();
+    }
+
+    fn visit_f_string(&mut self, f_string: &FString) {
+        self.enter_node(f_string);
+        walk_f_string(self, f_string);
+        self.exit_node();
+    }
+
+    fn visit_f_string_element(&mut self, f_string_element: &FStringElement) {
+        self.enter_node(f_string_element);
+        walk_f_string_element(self, f_string_element);
+        self.exit_node();
+    }
+}