mirror of
https://github.com/astral-sh/ruff.git
synced 2025-10-30 19:47:52 +00:00
c9dff5c7d5
## Summary Garbage collect ASTs once we are done checking a given file. Queries with a cross-file dependency on the AST will reparse the file on demand. This reduces ty's peak memory usage by ~20-30%. The primary change of this PR is adding a `node_index` field to every AST node, that is assigned by the parser. `ParsedModule` can use this to create a flat index of AST nodes any time the file is parsed (or reparsed). This allows `AstNodeRef` to simply index into the current instance of the `ParsedModule`, instead of storing a pointer directly. The indices are somewhat hackily (using an atomic integer) assigned by the `parsed_module` query instead of by the parser directly. Assigning the indices in source-order in the (recursive) parser turns out to be difficult, and collecting the nodes during semantic indexing is impossible as `SemanticIndex` does not hold onto a specific `ParsedModuleRef`, which the pointers in the flat AST are tied to. This means that we have to do an extra AST traversal to assign and collect the nodes into a flat index, but the small performance impact (~3% on cold runs) seems worth it for the memory savings. Part of https://github.com/astral-sh/ty/issues/214.
530 lines
24 KiB
Text
530 lines
24 KiB
Text
---
|
|
source: crates/ruff_python_parser/tests/fixtures.rs
|
|
input_file: crates/ruff_python_parser/resources/valid/expressions/yield.py
|
|
---
|
|
## AST
|
|
|
|
```
|
|
Module(
|
|
ModModule {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 0..166,
|
|
body: [
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 0..5,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 0..5,
|
|
value: None,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 6..13,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 6..13,
|
|
value: Some(
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 12..13,
|
|
id: Name("x"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 14..25,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 14..25,
|
|
value: Some(
|
|
BinOp(
|
|
ExprBinOp {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 20..25,
|
|
left: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 20..21,
|
|
id: Name("x"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
op: Add,
|
|
right: NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 24..25,
|
|
value: Int(
|
|
1,
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 26..39,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 26..39,
|
|
value: Some(
|
|
BoolOp(
|
|
ExprBoolOp {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 32..39,
|
|
op: And,
|
|
values: [
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 32..33,
|
|
id: Name("x"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 38..39,
|
|
id: Name("y"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
],
|
|
},
|
|
),
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 40..52,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 40..52,
|
|
value: Some(
|
|
Call(
|
|
ExprCall {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 46..52,
|
|
func: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 46..50,
|
|
id: Name("call"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
arguments: Arguments {
|
|
range: 50..52,
|
|
node_index: AtomicNodeIndex(..),
|
|
args: [],
|
|
keywords: [],
|
|
},
|
|
},
|
|
),
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 53..65,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 53..65,
|
|
value: Some(
|
|
List(
|
|
ExprList {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 59..65,
|
|
elts: [
|
|
NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 60..61,
|
|
value: Int(
|
|
1,
|
|
),
|
|
},
|
|
),
|
|
NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 63..64,
|
|
value: Int(
|
|
2,
|
|
),
|
|
},
|
|
),
|
|
],
|
|
ctx: Load,
|
|
},
|
|
),
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 66..78,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 66..78,
|
|
value: Some(
|
|
Set(
|
|
ExprSet {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 72..78,
|
|
elts: [
|
|
NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 73..74,
|
|
value: Int(
|
|
3,
|
|
),
|
|
},
|
|
),
|
|
NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 76..77,
|
|
value: Int(
|
|
4,
|
|
),
|
|
},
|
|
),
|
|
],
|
|
},
|
|
),
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 79..91,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 79..91,
|
|
value: Some(
|
|
Dict(
|
|
ExprDict {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 85..91,
|
|
items: [
|
|
DictItem {
|
|
key: Some(
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 86..87,
|
|
id: Name("x"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
),
|
|
value: NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 89..90,
|
|
value: Int(
|
|
5,
|
|
),
|
|
},
|
|
),
|
|
},
|
|
],
|
|
},
|
|
),
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 92..102,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 92..102,
|
|
value: Some(
|
|
Tuple(
|
|
ExprTuple {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 98..102,
|
|
elts: [
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 98..99,
|
|
id: Name("x"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 101..102,
|
|
id: Name("y"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
],
|
|
ctx: Load,
|
|
parenthesized: false,
|
|
},
|
|
),
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 103..115,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 103..115,
|
|
value: Some(
|
|
Tuple(
|
|
ExprTuple {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 109..115,
|
|
elts: [
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 110..111,
|
|
id: Name("x"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 113..114,
|
|
id: Name("y"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
],
|
|
ctx: Load,
|
|
parenthesized: true,
|
|
},
|
|
),
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 116..128,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 116..128,
|
|
value: Some(
|
|
Compare(
|
|
ExprCompare {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 122..128,
|
|
left: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 122..123,
|
|
id: Name("x"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
ops: [
|
|
Eq,
|
|
],
|
|
comparators: [
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 127..128,
|
|
id: Name("y"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
],
|
|
},
|
|
),
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 129..143,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 129..143,
|
|
value: Some(
|
|
Named(
|
|
ExprNamed {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 136..142,
|
|
target: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 136..137,
|
|
id: Name("x"),
|
|
ctx: Store,
|
|
},
|
|
),
|
|
value: NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 141..142,
|
|
value: Int(
|
|
1,
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 144..155,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 144..155,
|
|
value: Some(
|
|
Tuple(
|
|
ExprTuple {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 150..155,
|
|
elts: [
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 150..151,
|
|
id: Name("x"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
Starred(
|
|
ExprStarred {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 153..155,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 154..155,
|
|
id: Name("y"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
],
|
|
ctx: Load,
|
|
parenthesized: false,
|
|
},
|
|
),
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 156..165,
|
|
value: Yield(
|
|
ExprYield {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 156..165,
|
|
value: Some(
|
|
Tuple(
|
|
ExprTuple {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 162..165,
|
|
elts: [
|
|
Starred(
|
|
ExprStarred {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 162..164,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 163..164,
|
|
id: Name("x"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
],
|
|
ctx: Load,
|
|
parenthesized: false,
|
|
},
|
|
),
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
],
|
|
},
|
|
)
|
|
```
|