mirror of
https://github.com/astral-sh/ruff.git
synced 2025-11-02 04:48:07 +00:00
[ty] AST garbage collection (#18482)
## Summary Garbage collect ASTs once we are done checking a given file. Queries with a cross-file dependency on the AST will reparse the file on demand. This reduces ty's peak memory usage by ~20-30%. The primary change of this PR is adding a `node_index` field to every AST node, that is assigned by the parser. `ParsedModule` can use this to create a flat index of AST nodes any time the file is parsed (or reparsed). This allows `AstNodeRef` to simply index into the current instance of the `ParsedModule`, instead of storing a pointer directly. The indices are somewhat hackily (using an atomic integer) assigned by the `parsed_module` query instead of by the parser directly. Assigning the indices in source-order in the (recursive) parser turns out to be difficult, and collecting the nodes during semantic indexing is impossible as `SemanticIndex` does not hold onto a specific `ParsedModuleRef`, which the pointers in the flat AST are tied to. This means that we have to do an extra AST traversal to assign and collect the nodes into a flat index, but the small performance impact (~3% on cold runs) seems worth it for the memory savings. Part of https://github.com/astral-sh/ty/issues/214.
This commit is contained in:
Ibraheem Ahmed
GitHub
parent
76d9009a6e
commit
c9dff5c7d5
824 changed files with 25243 additions and 804 deletions
|
|
@ -1,20 +1,22 @@
|
|||
---
|
||||
source: crates/ruff_python_parser/tests/fixtures.rs
|
||||
input_file: crates/ruff_python_parser/resources/valid/statement/if.py
|
||||
snapshot_kind: text
|
||||
---
|
||||
## AST
|
||||
|
||||
```
|
||||
Module(
|
||||
ModModule {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 0..375,
|
||||
body: [
|
||||
If(
|
||||
StmtIf {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 0..28,
|
||||
test: NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 3..4,
|
||||
value: Int(
|
||||
1,
|
||||
|
|
@ -24,9 +26,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 6..8,
|
||||
value: NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 6..8,
|
||||
value: Int(
|
||||
10,
|
||||
|
|
@ -39,9 +43,11 @@ Module(
|
|||
elif_else_clauses: [
|
||||
ElifElseClause {
|
||||
range: 9..19,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
test: Some(
|
||||
NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 14..15,
|
||||
value: Int(
|
||||
2,
|
||||
|
|
@ -52,9 +58,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 17..19,
|
||||
value: NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 17..19,
|
||||
value: Int(
|
||||
20,
|
||||
|
|
@ -67,13 +75,16 @@ Module(
|
|||
},
|
||||
ElifElseClause {
|
||||
range: 20..28,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
test: None,
|
||||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 26..28,
|
||||
value: NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 26..28,
|
||||
value: Int(
|
||||
30,
|
||||
|
|
@ -89,9 +100,11 @@ Module(
|
|||
),
|
||||
If(
|
||||
StmtIf {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 30..52,
|
||||
test: BooleanLiteral(
|
||||
ExprBooleanLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 33..37,
|
||||
value: true,
|
||||
},
|
||||
|
|
@ -99,9 +112,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 43..44,
|
||||
value: NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 43..44,
|
||||
value: Int(
|
||||
1,
|
||||
|
|
@ -112,9 +127,11 @@ Module(
|
|||
),
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 49..52,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 49..52,
|
||||
},
|
||||
),
|
||||
|
|
@ -126,12 +143,15 @@ Module(
|
|||
),
|
||||
If(
|
||||
StmtIf {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 53..85,
|
||||
test: Compare(
|
||||
ExprCompare {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 56..61,
|
||||
left: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 56..57,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -143,6 +163,7 @@ Module(
|
|||
comparators: [
|
||||
NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 60..61,
|
||||
value: Int(
|
||||
1,
|
||||
|
|
@ -155,9 +176,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 67..70,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 67..70,
|
||||
},
|
||||
),
|
||||
|
|
@ -167,10 +190,12 @@ Module(
|
|||
elif_else_clauses: [
|
||||
ElifElseClause {
|
||||
range: 71..85,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
test: None,
|
||||
body: [
|
||||
Pass(
|
||||
StmtPass {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 81..85,
|
||||
},
|
||||
),
|
||||
|
|
@ -181,9 +206,11 @@ Module(
|
|||
),
|
||||
If(
|
||||
StmtIf {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 87..117,
|
||||
test: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 90..91,
|
||||
id: Name("a"),
|
||||
ctx: Load,
|
||||
|
|
@ -192,6 +219,7 @@ Module(
|
|||
body: [
|
||||
Pass(
|
||||
StmtPass {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 97..101,
|
||||
},
|
||||
),
|
||||
|
|
@ -199,9 +227,11 @@ Module(
|
|||
elif_else_clauses: [
|
||||
ElifElseClause {
|
||||
range: 102..117,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
test: Some(
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 107..108,
|
||||
id: Name("b"),
|
||||
ctx: Load,
|
||||
|
|
@ -211,9 +241,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 114..117,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 114..117,
|
||||
},
|
||||
),
|
||||
|
|
@ -226,14 +258,17 @@ Module(
|
|||
),
|
||||
If(
|
||||
StmtIf {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 119..203,
|
||||
test: BoolOp(
|
||||
ExprBoolOp {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 122..129,
|
||||
op: And,
|
||||
values: [
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 122..123,
|
||||
id: Name("a"),
|
||||
ctx: Load,
|
||||
|
|
@ -241,6 +276,7 @@ Module(
|
|||
),
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 128..129,
|
||||
id: Name("b"),
|
||||
ctx: Load,
|
||||
|
|
@ -252,9 +288,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 135..138,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 135..138,
|
||||
},
|
||||
),
|
||||
|
|
@ -264,9 +302,11 @@ Module(
|
|||
elif_else_clauses: [
|
||||
ElifElseClause {
|
||||
range: 139..157,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
test: Some(
|
||||
BooleanLiteral(
|
||||
ExprBooleanLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 144..148,
|
||||
value: true,
|
||||
},
|
||||
|
|
@ -275,9 +315,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 154..157,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 154..157,
|
||||
},
|
||||
),
|
||||
|
|
@ -287,9 +329,11 @@ Module(
|
|||
},
|
||||
ElifElseClause {
|
||||
range: 158..173,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
test: Some(
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 163..164,
|
||||
id: Name("c"),
|
||||
ctx: Load,
|
||||
|
|
@ -299,9 +343,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 170..173,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 170..173,
|
||||
},
|
||||
),
|
||||
|
|
@ -311,9 +357,11 @@ Module(
|
|||
},
|
||||
ElifElseClause {
|
||||
range: 174..189,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
test: Some(
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 179..180,
|
||||
id: Name("d"),
|
||||
ctx: Load,
|
||||
|
|
@ -323,9 +371,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 186..189,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 186..189,
|
||||
},
|
||||
),
|
||||
|
|
@ -335,16 +385,20 @@ Module(
|
|||
},
|
||||
ElifElseClause {
|
||||
range: 190..203,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
test: None,
|
||||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 200..203,
|
||||
value: Call(
|
||||
ExprCall {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 200..203,
|
||||
func: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 200..201,
|
||||
id: Name("f"),
|
||||
ctx: Load,
|
||||
|
|
@ -352,6 +406,7 @@ Module(
|
|||
),
|
||||
arguments: Arguments {
|
||||
range: 201..203,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
args: [],
|
||||
keywords: [],
|
||||
},
|
||||
|
|
@ -366,12 +421,15 @@ Module(
|
|||
),
|
||||
If(
|
||||
StmtIf {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 229..260,
|
||||
test: Named(
|
||||
ExprNamed {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 232..238,
|
||||
target: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 232..233,
|
||||
id: Name("a"),
|
||||
ctx: Store,
|
||||
|
|
@ -379,6 +437,7 @@ Module(
|
|||
),
|
||||
value: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 237..238,
|
||||
id: Name("b"),
|
||||
ctx: Load,
|
||||
|
|
@ -389,9 +448,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 240..243,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 240..243,
|
||||
},
|
||||
),
|
||||
|
|
@ -401,12 +462,15 @@ Module(
|
|||
elif_else_clauses: [
|
||||
ElifElseClause {
|
||||
range: 244..260,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
test: Some(
|
||||
Named(
|
||||
ExprNamed {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 249..255,
|
||||
target: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 249..250,
|
||||
id: Name("a"),
|
||||
ctx: Store,
|
||||
|
|
@ -414,6 +478,7 @@ Module(
|
|||
),
|
||||
value: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 254..255,
|
||||
id: Name("b"),
|
||||
ctx: Load,
|
||||
|
|
@ -425,9 +490,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 257..260,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 257..260,
|
||||
},
|
||||
),
|
||||
|
|
@ -440,22 +507,30 @@ Module(
|
|||
),
|
||||
If(
|
||||
StmtIf {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 261..302,
|
||||
test: Lambda(
|
||||
ExprLambda {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 264..275,
|
||||
parameters: Some(
|
||||
Parameters {
|
||||
range: 271..272,
|
||||
node_index: AtomicNodeIndex(
|
||||
0,
|
||||
),
|
||||
posonlyargs: [],
|
||||
args: [
|
||||
ParameterWithDefault {
|
||||
range: 271..272,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
parameter: Parameter {
|
||||
range: 271..272,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
name: Identifier {
|
||||
id: Name("x"),
|
||||
range: 271..272,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
},
|
||||
annotation: None,
|
||||
},
|
||||
|
|
@ -469,6 +544,7 @@ Module(
|
|||
),
|
||||
body: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 274..275,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -479,9 +555,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 277..280,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 277..280,
|
||||
},
|
||||
),
|
||||
|
|
@ -491,22 +569,30 @@ Module(
|
|||
elif_else_clauses: [
|
||||
ElifElseClause {
|
||||
range: 281..302,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
test: Some(
|
||||
Lambda(
|
||||
ExprLambda {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 286..297,
|
||||
parameters: Some(
|
||||
Parameters {
|
||||
range: 293..294,
|
||||
node_index: AtomicNodeIndex(
|
||||
0,
|
||||
),
|
||||
posonlyargs: [],
|
||||
args: [
|
||||
ParameterWithDefault {
|
||||
range: 293..294,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
parameter: Parameter {
|
||||
range: 293..294,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
name: Identifier {
|
||||
id: Name("x"),
|
||||
range: 293..294,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
},
|
||||
annotation: None,
|
||||
},
|
||||
|
|
@ -520,6 +606,7 @@ Module(
|
|||
),
|
||||
body: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 296..297,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -531,9 +618,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 299..302,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 299..302,
|
||||
},
|
||||
),
|
||||
|
|
@ -546,12 +635,15 @@ Module(
|
|||
),
|
||||
If(
|
||||
StmtIf {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 303..336,
|
||||
test: Await(
|
||||
ExprAwait {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 306..313,
|
||||
value: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 312..313,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -562,9 +654,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 315..318,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 315..318,
|
||||
},
|
||||
),
|
||||
|
|
@ -574,12 +668,15 @@ Module(
|
|||
elif_else_clauses: [
|
||||
ElifElseClause {
|
||||
range: 319..336,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
test: Some(
|
||||
Await(
|
||||
ExprAwait {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 324..331,
|
||||
value: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 330..331,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -591,9 +688,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 333..336,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 333..336,
|
||||
},
|
||||
),
|
||||
|
|
@ -606,13 +705,16 @@ Module(
|
|||
),
|
||||
If(
|
||||
StmtIf {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 337..374,
|
||||
test: Yield(
|
||||
ExprYield {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 341..348,
|
||||
value: Some(
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 347..348,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -624,9 +726,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 351..354,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 351..354,
|
||||
},
|
||||
),
|
||||
|
|
@ -636,13 +740,16 @@ Module(
|
|||
elif_else_clauses: [
|
||||
ElifElseClause {
|
||||
range: 355..374,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
test: Some(
|
||||
Yield(
|
||||
ExprYield {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 361..368,
|
||||
value: Some(
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 367..368,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -655,9 +762,11 @@ Module(
|
|||
body: [
|
||||
Expr(
|
||||
StmtExpr {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 371..374,
|
||||
value: EllipsisLiteral(
|
||||
ExprEllipsisLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 371..374,
|
||||
},
|
||||
),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue