mirror of
https://github.com/astral-sh/ruff.git
synced 2025-11-01 20:31:57 +00:00
[ty] AST garbage collection (#18482)
## Summary Garbage collect ASTs once we are done checking a given file. Queries with a cross-file dependency on the AST will reparse the file on demand. This reduces ty's peak memory usage by ~20-30%. The primary change of this PR is adding a `node_index` field to every AST node, that is assigned by the parser. `ParsedModule` can use this to create a flat index of AST nodes any time the file is parsed (or reparsed). This allows `AstNodeRef` to simply index into the current instance of the `ParsedModule`, instead of storing a pointer directly. The indices are somewhat hackily (using an atomic integer) assigned by the `parsed_module` query instead of by the parser directly. Assigning the indices in source-order in the (recursive) parser turns out to be difficult, and collecting the nodes during semantic indexing is impossible as `SemanticIndex` does not hold onto a specific `ParsedModuleRef`, which the pointers in the flat AST are tied to. This means that we have to do an extra AST traversal to assign and collect the nodes into a flat index, but the small performance impact (~3% on cold runs) seems worth it for the memory savings. Part of https://github.com/astral-sh/ty/issues/214.
This commit is contained in:
Ibraheem Ahmed
GitHub
parent
76d9009a6e
commit
c9dff5c7d5
824 changed files with 25243 additions and 804 deletions
|
|
@ -1,21 +1,23 @@
|
|||
---
|
||||
source: crates/ruff_python_parser/tests/fixtures.rs
|
||||
input_file: crates/ruff_python_parser/resources/valid/statement/delete.py
|
||||
snapshot_kind: text
|
||||
---
|
||||
## AST
|
||||
|
||||
```
|
||||
Module(
|
||||
ModModule {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 0..122,
|
||||
body: [
|
||||
Delete(
|
||||
StmtDelete {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 0..5,
|
||||
targets: [
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 4..5,
|
||||
id: Name("x"),
|
||||
ctx: Del,
|
||||
|
|
@ -26,10 +28,12 @@ Module(
|
|||
),
|
||||
Delete(
|
||||
StmtDelete {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 6..13,
|
||||
targets: [
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 11..12,
|
||||
id: Name("x"),
|
||||
ctx: Del,
|
||||
|
|
@ -40,10 +44,12 @@ Module(
|
|||
),
|
||||
Delete(
|
||||
StmtDelete {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 14..23,
|
||||
targets: [
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 18..19,
|
||||
id: Name("a"),
|
||||
ctx: Del,
|
||||
|
|
@ -51,6 +57,7 @@ Module(
|
|||
),
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 21..22,
|
||||
id: Name("b"),
|
||||
ctx: Del,
|
||||
|
|
@ -61,10 +68,12 @@ Module(
|
|||
),
|
||||
Delete(
|
||||
StmtDelete {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 24..40,
|
||||
targets: [
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 28..29,
|
||||
id: Name("a"),
|
||||
ctx: Del,
|
||||
|
|
@ -72,10 +81,12 @@ Module(
|
|||
),
|
||||
Tuple(
|
||||
ExprTuple {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 31..37,
|
||||
elts: [
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 32..33,
|
||||
id: Name("b"),
|
||||
ctx: Del,
|
||||
|
|
@ -83,6 +94,7 @@ Module(
|
|||
),
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 35..36,
|
||||
id: Name("c"),
|
||||
ctx: Del,
|
||||
|
|
@ -95,6 +107,7 @@ Module(
|
|||
),
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 39..40,
|
||||
id: Name("d"),
|
||||
ctx: Del,
|
||||
|
|
@ -105,14 +118,17 @@ Module(
|
|||
),
|
||||
Delete(
|
||||
StmtDelete {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 41..51,
|
||||
targets: [
|
||||
List(
|
||||
ExprList {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 45..51,
|
||||
elts: [
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 46..47,
|
||||
id: Name("a"),
|
||||
ctx: Del,
|
||||
|
|
@ -120,6 +136,7 @@ Module(
|
|||
),
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 49..50,
|
||||
id: Name("b"),
|
||||
ctx: Del,
|
||||
|
|
@ -134,14 +151,17 @@ Module(
|
|||
),
|
||||
Delete(
|
||||
StmtDelete {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 52..70,
|
||||
targets: [
|
||||
List(
|
||||
ExprList {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 56..70,
|
||||
elts: [
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 57..58,
|
||||
id: Name("a"),
|
||||
ctx: Del,
|
||||
|
|
@ -149,10 +169,12 @@ Module(
|
|||
),
|
||||
List(
|
||||
ExprList {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 60..66,
|
||||
elts: [
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 61..62,
|
||||
id: Name("b"),
|
||||
ctx: Del,
|
||||
|
|
@ -160,6 +182,7 @@ Module(
|
|||
),
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 64..65,
|
||||
id: Name("c"),
|
||||
ctx: Del,
|
||||
|
|
@ -171,6 +194,7 @@ Module(
|
|||
),
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 68..69,
|
||||
id: Name("d"),
|
||||
ctx: Del,
|
||||
|
|
@ -185,13 +209,16 @@ Module(
|
|||
),
|
||||
Delete(
|
||||
StmtDelete {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 71..78,
|
||||
targets: [
|
||||
Attribute(
|
||||
ExprAttribute {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 75..78,
|
||||
value: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 75..76,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -200,6 +227,7 @@ Module(
|
|||
attr: Identifier {
|
||||
id: Name("y"),
|
||||
range: 77..78,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
},
|
||||
ctx: Del,
|
||||
},
|
||||
|
|
@ -209,13 +237,16 @@ Module(
|
|||
),
|
||||
Delete(
|
||||
StmtDelete {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 79..87,
|
||||
targets: [
|
||||
Subscript(
|
||||
ExprSubscript {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 83..87,
|
||||
value: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 83..84,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -223,6 +254,7 @@ Module(
|
|||
),
|
||||
slice: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 85..86,
|
||||
id: Name("y"),
|
||||
ctx: Load,
|
||||
|
|
@ -236,14 +268,17 @@ Module(
|
|||
),
|
||||
Delete(
|
||||
StmtDelete {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 88..121,
|
||||
targets: [
|
||||
Tuple(
|
||||
ExprTuple {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 92..121,
|
||||
elts: [
|
||||
Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 98..99,
|
||||
id: Name("x"),
|
||||
ctx: Del,
|
||||
|
|
@ -251,9 +286,11 @@ Module(
|
|||
),
|
||||
Attribute(
|
||||
ExprAttribute {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 105..108,
|
||||
value: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 105..106,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -262,15 +299,18 @@ Module(
|
|||
attr: Identifier {
|
||||
id: Name("y"),
|
||||
range: 107..108,
|
||||
node_index: AtomicNodeIndex(..),
|
||||
},
|
||||
ctx: Del,
|
||||
},
|
||||
),
|
||||
Subscript(
|
||||
ExprSubscript {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 114..118,
|
||||
value: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 114..115,
|
||||
id: Name("x"),
|
||||
ctx: Load,
|
||||
|
|
@ -278,6 +318,7 @@ Module(
|
|||
),
|
||||
slice: Name(
|
||||
ExprName {
|
||||
node_index: AtomicNodeIndex(..),
|
||||
range: 116..117,
|
||||
id: Name("y"),
|
||||
ctx: Load,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue