mirror of
https://github.com/astral-sh/ruff.git
synced 2025-07-15 17:15:22 +00:00
c9dff5c7d5
## Summary Garbage collect ASTs once we are done checking a given file. Queries with a cross-file dependency on the AST will reparse the file on demand. This reduces ty's peak memory usage by ~20-30%. The primary change of this PR is adding a `node_index` field to every AST node, that is assigned by the parser. `ParsedModule` can use this to create a flat index of AST nodes any time the file is parsed (or reparsed). This allows `AstNodeRef` to simply index into the current instance of the `ParsedModule`, instead of storing a pointer directly. The indices are somewhat hackily (using an atomic integer) assigned by the `parsed_module` query instead of by the parser directly. Assigning the indices in source-order in the (recursive) parser turns out to be difficult, and collecting the nodes during semantic indexing is impossible as `SemanticIndex` does not hold onto a specific `ParsedModuleRef`, which the pointers in the flat AST are tied to. This means that we have to do an extra AST traversal to assign and collect the nodes into a flat index, but the small performance impact (~3% on cold runs) seems worth it for the memory savings. Part of https://github.com/astral-sh/ty/issues/214.
462 lines
20 KiB
Text
462 lines
20 KiB
Text
---
|
|
source: crates/ruff_python_parser/tests/fixtures.rs
|
|
input_file: crates/ruff_python_parser/resources/valid/statement/with.py
|
|
---
|
|
## AST
|
|
|
|
```
|
|
Module(
|
|
ModModule {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 0..361,
|
|
body: [
|
|
With(
|
|
StmtWith {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 137..151,
|
|
is_async: false,
|
|
items: [
|
|
WithItem {
|
|
range: 142..146,
|
|
node_index: AtomicNodeIndex(..),
|
|
context_expr: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 142..146,
|
|
id: Name("item"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
optional_vars: None,
|
|
},
|
|
],
|
|
body: [
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 148..151,
|
|
value: EllipsisLiteral(
|
|
ExprEllipsisLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 148..151,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
],
|
|
},
|
|
),
|
|
With(
|
|
StmtWith {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 152..171,
|
|
is_async: false,
|
|
items: [
|
|
WithItem {
|
|
range: 157..166,
|
|
node_index: AtomicNodeIndex(..),
|
|
context_expr: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 157..161,
|
|
id: Name("item"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
optional_vars: Some(
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 165..166,
|
|
id: Name("f"),
|
|
ctx: Store,
|
|
},
|
|
),
|
|
),
|
|
},
|
|
],
|
|
body: [
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 168..171,
|
|
value: EllipsisLiteral(
|
|
ExprEllipsisLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 168..171,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
],
|
|
},
|
|
),
|
|
With(
|
|
StmtWith {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 172..194,
|
|
is_async: false,
|
|
items: [
|
|
WithItem {
|
|
range: 177..182,
|
|
node_index: AtomicNodeIndex(..),
|
|
context_expr: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 177..182,
|
|
id: Name("item1"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
optional_vars: None,
|
|
},
|
|
WithItem {
|
|
range: 184..189,
|
|
node_index: AtomicNodeIndex(..),
|
|
context_expr: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 184..189,
|
|
id: Name("item2"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
optional_vars: None,
|
|
},
|
|
],
|
|
body: [
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 191..194,
|
|
value: EllipsisLiteral(
|
|
ExprEllipsisLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 191..194,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
],
|
|
},
|
|
),
|
|
With(
|
|
StmtWith {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 195..229,
|
|
is_async: false,
|
|
items: [
|
|
WithItem {
|
|
range: 200..211,
|
|
node_index: AtomicNodeIndex(..),
|
|
context_expr: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 200..205,
|
|
id: Name("item1"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
optional_vars: Some(
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 209..211,
|
|
id: Name("f1"),
|
|
ctx: Store,
|
|
},
|
|
),
|
|
),
|
|
},
|
|
WithItem {
|
|
range: 213..224,
|
|
node_index: AtomicNodeIndex(..),
|
|
context_expr: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 213..218,
|
|
id: Name("item2"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
optional_vars: Some(
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 222..224,
|
|
id: Name("f2"),
|
|
ctx: Store,
|
|
},
|
|
),
|
|
),
|
|
},
|
|
],
|
|
body: [
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 226..229,
|
|
value: EllipsisLiteral(
|
|
ExprEllipsisLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 226..229,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
],
|
|
},
|
|
),
|
|
With(
|
|
StmtWith {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 231..257,
|
|
is_async: false,
|
|
items: [
|
|
WithItem {
|
|
range: 236..252,
|
|
node_index: AtomicNodeIndex(..),
|
|
context_expr: If(
|
|
ExprIf {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 236..252,
|
|
test: BooleanLiteral(
|
|
ExprBooleanLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 241..245,
|
|
value: true,
|
|
},
|
|
),
|
|
body: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 236..237,
|
|
id: Name("x"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
orelse: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 251..252,
|
|
id: Name("y"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
optional_vars: None,
|
|
},
|
|
],
|
|
body: [
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 254..257,
|
|
value: EllipsisLiteral(
|
|
ExprEllipsisLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 254..257,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
],
|
|
},
|
|
),
|
|
With(
|
|
StmtWith {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 258..289,
|
|
is_async: false,
|
|
items: [
|
|
WithItem {
|
|
range: 263..284,
|
|
node_index: AtomicNodeIndex(..),
|
|
context_expr: If(
|
|
ExprIf {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 263..279,
|
|
test: BooleanLiteral(
|
|
ExprBooleanLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 268..272,
|
|
value: true,
|
|
},
|
|
),
|
|
body: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 263..264,
|
|
id: Name("x"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
orelse: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 278..279,
|
|
id: Name("y"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
optional_vars: Some(
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 283..284,
|
|
id: Name("f"),
|
|
ctx: Store,
|
|
},
|
|
),
|
|
),
|
|
},
|
|
],
|
|
body: [
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 286..289,
|
|
value: EllipsisLiteral(
|
|
ExprEllipsisLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 286..289,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
],
|
|
},
|
|
),
|
|
With(
|
|
StmtWith {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 313..334,
|
|
is_async: false,
|
|
items: [
|
|
WithItem {
|
|
range: 318..329,
|
|
node_index: AtomicNodeIndex(..),
|
|
context_expr: Call(
|
|
ExprCall {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 318..324,
|
|
func: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 318..322,
|
|
id: Name("open"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
arguments: Arguments {
|
|
range: 322..324,
|
|
node_index: AtomicNodeIndex(..),
|
|
args: [],
|
|
keywords: [],
|
|
},
|
|
},
|
|
),
|
|
optional_vars: Some(
|
|
Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 328..329,
|
|
id: Name("f"),
|
|
ctx: Store,
|
|
},
|
|
),
|
|
),
|
|
},
|
|
],
|
|
body: [
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 331..334,
|
|
value: EllipsisLiteral(
|
|
ExprEllipsisLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 331..334,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
],
|
|
},
|
|
),
|
|
With(
|
|
StmtWith {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 335..361,
|
|
is_async: false,
|
|
items: [
|
|
WithItem {
|
|
range: 340..356,
|
|
node_index: AtomicNodeIndex(..),
|
|
context_expr: Call(
|
|
ExprCall {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 340..346,
|
|
func: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 340..344,
|
|
id: Name("open"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
arguments: Arguments {
|
|
range: 344..346,
|
|
node_index: AtomicNodeIndex(..),
|
|
args: [],
|
|
keywords: [],
|
|
},
|
|
},
|
|
),
|
|
optional_vars: Some(
|
|
Attribute(
|
|
ExprAttribute {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 350..356,
|
|
value: Name(
|
|
ExprName {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 350..351,
|
|
id: Name("f"),
|
|
ctx: Load,
|
|
},
|
|
),
|
|
attr: Identifier {
|
|
id: Name("attr"),
|
|
range: 352..356,
|
|
node_index: AtomicNodeIndex(..),
|
|
},
|
|
ctx: Store,
|
|
},
|
|
),
|
|
),
|
|
},
|
|
],
|
|
body: [
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 358..361,
|
|
value: EllipsisLiteral(
|
|
ExprEllipsisLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 358..361,
|
|
},
|
|
),
|
|
},
|
|
),
|
|
],
|
|
},
|
|
),
|
|
],
|
|
},
|
|
)
|
|
```
|