mirror of
https://github.com/astral-sh/ruff.git
synced 2025-08-04 02:39:12 +00:00
[ty] AST garbage collection (#18482)
## Summary Garbage collect ASTs once we are done checking a given file. Queries with a cross-file dependency on the AST will reparse the file on demand. This reduces ty's peak memory usage by ~20-30%. The primary change of this PR is adding a `node_index` field to every AST node, that is assigned by the parser. `ParsedModule` can use this to create a flat index of AST nodes any time the file is parsed (or reparsed). This allows `AstNodeRef` to simply index into the current instance of the `ParsedModule`, instead of storing a pointer directly. The indices are somewhat hackily (using an atomic integer) assigned by the `parsed_module` query instead of by the parser directly. Assigning the indices in source-order in the (recursive) parser turns out to be difficult, and collecting the nodes during semantic indexing is impossible as `SemanticIndex` does not hold onto a specific `ParsedModuleRef`, which the pointers in the flat AST are tied to. This means that we have to do an extra AST traversal to assign and collect the nodes into a flat index, but the small performance impact (~3% on cold runs) seems worth it for the memory savings. Part of https://github.com/astral-sh/ty/issues/214.
This commit is contained in:
Ibraheem Ahmed
GitHub
parent
76d9009a6e
commit
c9dff5c7d5
824 changed files with 25243 additions and 804 deletions
|
|
@ -1,23 +1,26 @@
|
|||
---
|
||||
source: crates/ruff_python_parser/tests/fixtures.rs
|
||||
input_file: crates/ruff_python_parser/resources/valid/statement/assert.py
|
||||
snapshot_kind: text
|
||||
---
|
||||
## AST
|
||||
|
||||
```
|
||||
Module(
|
||||
ModModule {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 0..186,
|
||||
body: [
|
||||
Assert(
|
||||
StmtAssert {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 0..12,
|
||||
test: Compare(
|
||||
ExprCompare {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 7..12,
|
||||
left: NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 7..8,
|
||||
value: Int(
|
||||
1,
|
||||
|
|
@ -30,6 +33,7 @@ Module(
|
|||
comparators: [
|
||||
NumberLiteral(
|
||||
ExprNumberLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 11..12,
|
||||
value: Int(
|
||||
2,
|
||||
|
|
@ -44,12 +48,15 @@ Module(
|
|||
),
|
||||
Assert(
|
||||
StmtAssert {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 13..26,
|
||||
test: Call(
|
||||
ExprCall {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 20..26,
|
||||
func: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 20..24,
|
||||
id: Name("call"),
|
||||
ctx: Load,
|
||||
|
|
@ -57,6 +64,7 @@ Module(
|
|||
),
|
||||
arguments: Arguments {
|
||||
range: 24..26,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
args: [],
|
||||
keywords: [],
|
||||
},
|
||||
|
|
@ -67,14 +75,17 @@ Module(
|
|||
),
|
||||
Assert(
|
||||
StmtAssert {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 27..41,
|
||||
test: BoolOp(
|
||||
ExprBoolOp {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 34..41,
|
||||
op: And,
|
||||
values: [
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 34..35,
|
||||
id: Name("a"),
|
||||
ctx: Load,
|
||||
|
|
@ -82,6 +93,7 @@ Module(
|
|||
),
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 40..41,
|
||||
id: Name("b"),
|
||||
ctx: Load,
|
||||
|
|
@ -95,22 +107,30 @@ Module(
|
|||
),
|
||||
Assert(
|
||||
StmtAssert {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 42..60,
|
||||
test: Lambda(
|
||||
ExprLambda {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 49..60,
|
||||
parameters: Some(
|
||||
Parameters {
|
||||
range: 56..57,
|
||||
node_index: AtomicNodeIndex(
|
||||
0,
|
||||
),
|
||||
posonlyargs: [],
|
||||
args: [
|
||||
ParameterWithDefault {
|
||||
range: 56..57,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
parameter: Parameter {
|
||||
range: 56..57,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
name: Identifier {
|
||||
id: Name("x"),
|
||||
range: 56..57,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
},
|
||||
annotation: None,
|
||||
},
|
||||
|
|
@ -124,6 +144,7 @@ Module(
|
|||
),
|
||||
body: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 59..60,
|
||||
id: Name("y"),
|
||||
ctx: Load,
|
||||
|
|
@ -136,12 +157,15 @@ Module(
|
|||
),
|
||||
Assert(
|
||||
StmtAssert {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 61..75,
|
||||
test: Await(
|
||||
ExprAwait {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 68..75,
|
||||
value: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 74..75,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -154,18 +178,22 @@ Module(
|
|||
),
|
||||
Assert(
|
||||
StmtAssert {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 76..99,
|
||||
test: If(
|
||||
ExprIf {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 83..99,
|
||||
test: BooleanLiteral(
|
||||
ExprBooleanLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 88..92,
|
||||
value: true,
|
||||
},
|
||||
),
|
||||
body: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 83..84,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -173,6 +201,7 @@ Module(
|
|||
),
|
||||
orelse: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 98..99,
|
||||
id: Name("y"),
|
||||
ctx: Load,
|
||||
|
|
@ -185,9 +214,11 @@ Module(
|
|||
),
|
||||
Assert(
|
||||
StmtAssert {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 101..118,
|
||||
test: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 108..109,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -196,11 +227,13 @@ Module(
|
|||
msg: Some(
|
||||
StringLiteral(
|
||||
ExprStringLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 111..118,
|
||||
value: StringLiteralValue {
|
||||
inner: Single(
|
||||
StringLiteral {
|
||||
range: 111..118,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
value: "error",
|
||||
flags: StringLiteralFlags {
|
||||
quote_style: Double,
|
||||
|
|
@ -217,9 +250,11 @@ Module(
|
|||
),
|
||||
Assert(
|
||||
StmtAssert {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 119..140,
|
||||
test: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 126..127,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -228,19 +263,26 @@ Module(
|
|||
msg: Some(
|
||||
Lambda(
|
||||
ExprLambda {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 129..140,
|
||||
parameters: Some(
|
||||
Parameters {
|
||||
range: 136..137,
|
||||
node_index: AtomicNodeIndex(
|
||||
0,
|
||||
),
|
||||
posonlyargs: [],
|
||||
args: [
|
||||
ParameterWithDefault {
|
||||
range: 136..137,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
parameter: Parameter {
|
||||
range: 136..137,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
name: Identifier {
|
||||
id: Name("x"),
|
||||
range: 136..137,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
},
|
||||
annotation: None,
|
||||
},
|
||||
|
|
@ -254,6 +296,7 @@ Module(
|
|||
),
|
||||
body: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 139..140,
|
||||
id: Name("y"),
|
||||
ctx: Load,
|
||||
|
|
@ -266,9 +309,11 @@ Module(
|
|||
),
|
||||
Assert(
|
||||
StmtAssert {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 141..158,
|
||||
test: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 148..149,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -277,9 +322,11 @@ Module(
|
|||
msg: Some(
|
||||
Await(
|
||||
ExprAwait {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 151..158,
|
||||
value: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 157..158,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -292,9 +339,11 @@ Module(
|
|||
),
|
||||
Assert(
|
||||
StmtAssert {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 159..185,
|
||||
test: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 166..167,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -303,15 +352,18 @@ Module(
|
|||
msg: Some(
|
||||
If(
|
||||
ExprIf {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 169..185,
|
||||
test: BooleanLiteral(
|
||||
ExprBooleanLiteral {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 174..178,
|
||||
value: true,
|
||||
},
|
||||
),
|
||||
body: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 169..170,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -319,6 +371,7 @@ Module(
|
|||
),
|
||||
orelse: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 184..185,
|
||||
id: Name("y"),
|
||||
ctx: Load,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue