mirror of
https://github.com/astral-sh/ruff.git
synced 2025-10-29 19:17:20 +00:00
c9dff5c7d5
## Summary Garbage collect ASTs once we are done checking a given file. Queries with a cross-file dependency on the AST will reparse the file on demand. This reduces ty's peak memory usage by ~20-30%. The primary change of this PR is adding a `node_index` field to every AST node, that is assigned by the parser. `ParsedModule` can use this to create a flat index of AST nodes any time the file is parsed (or reparsed). This allows `AstNodeRef` to simply index into the current instance of the `ParsedModule`, instead of storing a pointer directly. The indices are somewhat hackily (using an atomic integer) assigned by the `parsed_module` query instead of by the parser directly. Assigning the indices in source-order in the (recursive) parser turns out to be difficult, and collecting the nodes during semantic indexing is impossible as `SemanticIndex` does not hold onto a specific `ParsedModuleRef`, which the pointers in the flat AST are tied to. This means that we have to do an extra AST traversal to assign and collect the nodes into a flat index, but the small performance impact (~3% on cold runs) seems worth it for the memory savings. Part of https://github.com/astral-sh/ty/issues/214.
405 lines
20 KiB
Text
405 lines
20 KiB
Text
---
|
|
source: crates/ruff_python_parser/tests/fixtures.rs
|
|
input_file: crates/ruff_python_parser/resources/valid/expressions/starred.py
|
|
---
|
|
## AST
|
|
|
|
```
|
|
Module(
|
|
ModModule {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 0..172,
|
|
body: [
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 0..2,
|
|
value: Starred(
|
|
ExprStarred {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 0..2,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 1..2,
|
|
id: Name("a"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 3..11,
|
|
value: Starred(
|
|
ExprStarred {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 3..11,
|
|
value: BinOp(
|
|
ExprBinOp {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 5..10,
|
|
left: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 5..6,
|
|
id: Name("a"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
op: Add,
|
|
right: NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 9..10,
|
|
value: Int(
|
|
1,
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 12..19,
|
|
value: Starred(
|
|
ExprStarred {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 12..19,
|
|
value: Attribute(
|
|
ExprAttribute {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 13..19,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 13..14,
|
|
id: Name("x"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
attr: Identifier {
|
|
id: Name("attr"),
|
|
range: 15..19,
|
|
node_index: AtomicNodeIndex(..),
|
|
},
|
|
ctx: Load,
|
|
},
|
|
),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Assign(
|
|
StmtAssign {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 21..57,
|
|
targets: [
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 21..32,
|
|
id: Name("array_slice"),
|
|
ctx: Store,
|
|
},
|
|
),
|
|
],
|
|
value: Subscript(
|
|
ExprSubscript {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 35..57,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 35..40,
|
|
id: Name("array"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
slice: Tuple(
|
|
ExprTuple {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 41..56,
|
|
elts: [
|
|
NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 41..42,
|
|
value: Int(
|
|
0,
|
|
),
|
|
},
|
|
),
|
|
Starred(
|
|
ExprStarred {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 44..52,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 45..52,
|
|
id: Name("indexes"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
UnaryOp(
|
|
ExprUnaryOp {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 54..56,
|
|
op: USub,
|
|
operand: NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 55..56,
|
|
value: Int(
|
|
1,
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
],
|
|
ctx: Load,
|
|
parenthesized: false,
|
|
},
|
|
),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Assign(
|
|
StmtAssign {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 58..94,
|
|
targets: [
|
|
Subscript(
|
|
ExprSubscript {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 58..80,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 58..63,
|
|
id: Name("array"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
slice: Tuple(
|
|
ExprTuple {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 64..79,
|
|
elts: [
|
|
NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 64..65,
|
|
value: Int(
|
|
0,
|
|
),
|
|
},
|
|
),
|
|
Starred(
|
|
ExprStarred {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 67..75,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 68..75,
|
|
id: Name("indexes"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
UnaryOp(
|
|
ExprUnaryOp {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 77..79,
|
|
op: USub,
|
|
operand: NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 78..79,
|
|
value: Int(
|
|
1,
|
|
),
|
|
},
|
|
),
|
|
},
|
|
),
|
|
],
|
|
ctx: Load,
|
|
parenthesized: false,
|
|
},
|
|
),
|
|
ctx: Store,
|
|
},
|
|
),
|
|
],
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 83..94,
|
|
id: Name("array_slice"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 95..140,
|
|
value: Subscript(
|
|
ExprSubscript {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 95..140,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 95..100,
|
|
id: Name("array"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
slice: Tuple(
|
|
ExprTuple {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 101..139,
|
|
elts: [
|
|
Starred(
|
|
ExprStarred {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 101..119,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 102..119,
|
|
id: Name("indexes_to_select"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
Starred(
|
|
ExprStarred {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 121..139,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 122..139,
|
|
id: Name("indexes_to_select"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
],
|
|
ctx: Load,
|
|
parenthesized: false,
|
|
},
|
|
),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 141..171,
|
|
value: Subscript(
|
|
ExprSubscript {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 141..171,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 141..146,
|
|
id: Name("array"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
slice: Tuple(
|
|
ExprTuple {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 147..170,
|
|
elts: [
|
|
Slice(
|
|
ExprSlice {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 147..150,
|
|
lower: Some(
|
|
NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 147..148,
|
|
value: Int(
|
|
3,
|
|
),
|
|
},
|
|
),
|
|
),
|
|
upper: Some(
|
|
NumberLiteral(
|
|
ExprNumberLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 149..150,
|
|
value: Int(
|
|
5,
|
|
),
|
|
},
|
|
),
|
|
),
|
|
step: None,
|
|
},
|
|
),
|
|
Starred(
|
|
ExprStarred {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 152..170,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 153..170,
|
|
id: Name("indexes_to_select"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
],
|
|
ctx: Load,
|
|
parenthesized: false,
|
|
},
|
|
),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
],
|
|
},
|
|
)
|
|
```
|