mirror of
https://github.com/astral-sh/ruff.git
synced 2025-07-17 01:55:20 +00:00
c9dff5c7d5
## Summary Garbage collect ASTs once we are done checking a given file. Queries with a cross-file dependency on the AST will reparse the file on demand. This reduces ty's peak memory usage by ~20-30%. The primary change of this PR is adding a `node_index` field to every AST node, that is assigned by the parser. `ParsedModule` can use this to create a flat index of AST nodes any time the file is parsed (or reparsed). This allows `AstNodeRef` to simply index into the current instance of the `ParsedModule`, instead of storing a pointer directly. The indices are somewhat hackily (using an atomic integer) assigned by the `parsed_module` query instead of by the parser directly. Assigning the indices in source-order in the (recursive) parser turns out to be difficult, and collecting the nodes during semantic indexing is impossible as `SemanticIndex` does not hold onto a specific `ParsedModuleRef`, which the pointers in the flat AST are tied to. This means that we have to do an extra AST traversal to assign and collect the nodes into a flat index, but the small performance impact (~3% on cold runs) seems worth it for the memory savings. Part of https://github.com/astral-sh/ty/issues/214.
348 lines
16 KiB
Text
348 lines
16 KiB
Text
---
|
|
source: crates/ruff_python_parser/tests/fixtures.rs
|
|
input_file: crates/ruff_python_parser/resources/valid/expressions/string.py
|
|
---
|
|
## AST
|
|
|
|
```
|
|
Module(
|
|
ModModule {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 0..163,
|
|
body: [
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 0..13,
|
|
value: StringLiteral(
|
|
ExprStringLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 0..13,
|
|
value: StringLiteralValue {
|
|
inner: Single(
|
|
StringLiteral {
|
|
range: 0..13,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: "Hello World",
|
|
flags: StringLiteralFlags {
|
|
quote_style: Single,
|
|
prefix: Empty,
|
|
triple_quoted: false,
|
|
},
|
|
},
|
|
),
|
|
},
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 14..20,
|
|
value: StringLiteral(
|
|
ExprStringLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 14..20,
|
|
value: StringLiteralValue {
|
|
inner: Single(
|
|
StringLiteral {
|
|
range: 14..20,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: "😎",
|
|
flags: StringLiteralFlags {
|
|
quote_style: Double,
|
|
prefix: Empty,
|
|
triple_quoted: false,
|
|
},
|
|
},
|
|
),
|
|
},
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 21..32,
|
|
value: StringLiteral(
|
|
ExprStringLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 21..32,
|
|
value: StringLiteralValue {
|
|
inner: Concatenated(
|
|
ConcatenatedStringLiteral {
|
|
strings: [
|
|
StringLiteral {
|
|
range: 21..26,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: "Foo",
|
|
flags: StringLiteralFlags {
|
|
quote_style: Single,
|
|
prefix: Empty,
|
|
triple_quoted: false,
|
|
},
|
|
},
|
|
StringLiteral {
|
|
range: 27..32,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: "Bar",
|
|
flags: StringLiteralFlags {
|
|
quote_style: Single,
|
|
prefix: Empty,
|
|
triple_quoted: false,
|
|
},
|
|
},
|
|
],
|
|
value: "FooBar",
|
|
},
|
|
),
|
|
},
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 33..60,
|
|
value: StringLiteral(
|
|
ExprStringLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 39..58,
|
|
value: StringLiteralValue {
|
|
inner: Concatenated(
|
|
ConcatenatedStringLiteral {
|
|
strings: [
|
|
StringLiteral {
|
|
range: 39..42,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: "A",
|
|
flags: StringLiteralFlags {
|
|
quote_style: Single,
|
|
prefix: Empty,
|
|
triple_quoted: false,
|
|
},
|
|
},
|
|
StringLiteral {
|
|
range: 47..50,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: "B",
|
|
flags: StringLiteralFlags {
|
|
quote_style: Single,
|
|
prefix: Empty,
|
|
triple_quoted: false,
|
|
},
|
|
},
|
|
StringLiteral {
|
|
range: 55..58,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: "C",
|
|
flags: StringLiteralFlags {
|
|
quote_style: Single,
|
|
prefix: Empty,
|
|
triple_quoted: false,
|
|
},
|
|
},
|
|
],
|
|
value: "ABC",
|
|
},
|
|
),
|
|
},
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 61..79,
|
|
value: StringLiteral(
|
|
ExprStringLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 61..79,
|
|
value: StringLiteralValue {
|
|
inner: Single(
|
|
StringLiteral {
|
|
range: 61..79,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: "Olá, Mundo!",
|
|
flags: StringLiteralFlags {
|
|
quote_style: Single,
|
|
prefix: Empty,
|
|
triple_quoted: true,
|
|
},
|
|
},
|
|
),
|
|
},
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 80..91,
|
|
value: StringLiteral(
|
|
ExprStringLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 80..91,
|
|
value: StringLiteralValue {
|
|
inner: Single(
|
|
StringLiteral {
|
|
range: 80..91,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: "ABCDE",
|
|
flags: StringLiteralFlags {
|
|
quote_style: Double,
|
|
prefix: Empty,
|
|
triple_quoted: true,
|
|
},
|
|
},
|
|
),
|
|
},
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 92..121,
|
|
value: StringLiteral(
|
|
ExprStringLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 98..119,
|
|
value: StringLiteralValue {
|
|
inner: Concatenated(
|
|
ConcatenatedStringLiteral {
|
|
strings: [
|
|
StringLiteral {
|
|
range: 98..106,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: "aB",
|
|
flags: StringLiteralFlags {
|
|
quote_style: Single,
|
|
prefix: Empty,
|
|
triple_quoted: true,
|
|
},
|
|
},
|
|
StringLiteral {
|
|
range: 111..119,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: "cD",
|
|
flags: StringLiteralFlags {
|
|
quote_style: Single,
|
|
prefix: Empty,
|
|
triple_quoted: true,
|
|
},
|
|
},
|
|
],
|
|
value: "aBcD",
|
|
},
|
|
),
|
|
},
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 122..136,
|
|
value: BytesLiteral(
|
|
ExprBytesLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 122..136,
|
|
value: BytesLiteralValue {
|
|
inner: Single(
|
|
BytesLiteral {
|
|
range: 122..136,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: [
|
|
104,
|
|
101,
|
|
108,
|
|
108,
|
|
111,
|
|
32,
|
|
119,
|
|
111,
|
|
114,
|
|
108,
|
|
100,
|
|
],
|
|
flags: BytesLiteralFlags {
|
|
quote_style: Single,
|
|
prefix: Regular,
|
|
triple_quoted: false,
|
|
},
|
|
},
|
|
),
|
|
},
|
|
},
|
|
),
|
|
},
|
|
),
|
|
Expr(
|
|
StmtExpr {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 137..161,
|
|
value: BytesLiteral(
|
|
ExprBytesLiteral {
|
|
node_index: AtomicNodeIndex(..),
|
|
range: 137..161,
|
|
value: BytesLiteralValue {
|
|
inner: Concatenated(
|
|
[
|
|
BytesLiteral {
|
|
range: 137..145,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: [
|
|
98,
|
|
121,
|
|
116,
|
|
101,
|
|
115,
|
|
],
|
|
flags: BytesLiteralFlags {
|
|
quote_style: Single,
|
|
prefix: Regular,
|
|
triple_quoted: false,
|
|
},
|
|
},
|
|
BytesLiteral {
|
|
range: 146..161,
|
|
node_index: AtomicNodeIndex(..),
|
|
value: [
|
|
99,
|
|
111,
|
|
110,
|
|
99,
|
|
97,
|
|
116,
|
|
101,
|
|
110,
|
|
97,
|
|
116,
|
|
101,
|
|
100,
|
|
],
|
|
flags: BytesLiteralFlags {
|
|
quote_style: Single,
|
|
prefix: Regular,
|
|
triple_quoted: false,
|
|
},
|
|
},
|
|
],
|
|
),
|
|
},
|
|
},
|
|
),
|
|
},
|
|
),
|
|
],
|
|
},
|
|
)
|
|
```
|