mirror of
https://github.com/astral-sh/ruff.git
synced 2025-10-30 19:47:52 +00:00
[ty] AST garbage collection (#18482)
## Summary Garbage collect ASTs once we are done checking a given file. Queries with a cross-file dependency on the AST will reparse the file on demand. This reduces ty's peak memory usage by ~20-30%. The primary change of this PR is adding a `node_index` field to every AST node, that is assigned by the parser. `ParsedModule` can use this to create a flat index of AST nodes any time the file is parsed (or reparsed). This allows `AstNodeRef` to simply index into the current instance of the `ParsedModule`, instead of storing a pointer directly. The indices are somewhat hackily (using an atomic integer) assigned by the `parsed_module` query instead of by the parser directly. Assigning the indices in source-order in the (recursive) parser turns out to be difficult, and collecting the nodes during semantic indexing is impossible as `SemanticIndex` does not hold onto a specific `ParsedModuleRef`, which the pointers in the flat AST are tied to. This means that we have to do an extra AST traversal to assign and collect the nodes into a flat index, but the small performance impact (~3% on cold runs) seems worth it for the memory savings. Part of https://github.com/astral-sh/ty/issues/214.
This commit is contained in:
Ibraheem Ahmed
GitHub
parent
76d9009a6e
commit
c9dff5c7d5
824 changed files with 25243 additions and 804 deletions
|
|
@ -13,6 +13,7 @@ impl ast::ElifElseClause {
|
|||
{
|
||||
let ast::ElifElseClause {
|
||||
range: _,
|
||||
node_index: _,
|
||||
test,
|
||||
body,
|
||||
} = self;
|
||||
|
|
@ -28,7 +29,11 @@ impl ast::ExprDict {
|
|||
where
|
||||
V: SourceOrderVisitor<'a> + ?Sized,
|
||||
{
|
||||
let ast::ExprDict { items, range: _ } = self;
|
||||
let ast::ExprDict {
|
||||
items,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
|
||||
for ast::DictItem { key, value } in items {
|
||||
if let Some(key) = key {
|
||||
|
|
@ -48,6 +53,7 @@ impl ast::ExprBoolOp {
|
|||
op,
|
||||
values,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
match values.as_slice() {
|
||||
[left, rest @ ..] => {
|
||||
|
|
@ -74,6 +80,7 @@ impl ast::ExprCompare {
|
|||
ops,
|
||||
comparators,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
|
||||
visitor.visit_expr(left);
|
||||
|
|
@ -121,7 +128,11 @@ impl ast::InterpolatedStringLiteralElement {
|
|||
where
|
||||
V: SourceOrderVisitor<'a> + ?Sized,
|
||||
{
|
||||
let ast::InterpolatedStringLiteralElement { range: _, value: _ } = self;
|
||||
let ast::InterpolatedStringLiteralElement {
|
||||
range: _,
|
||||
node_index: _,
|
||||
value: _,
|
||||
} = self;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -130,7 +141,11 @@ impl ast::ExprFString {
|
|||
where
|
||||
V: SourceOrderVisitor<'a> + ?Sized,
|
||||
{
|
||||
let ast::ExprFString { value, range: _ } = self;
|
||||
let ast::ExprFString {
|
||||
value,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
|
||||
for f_string_part in value {
|
||||
match f_string_part {
|
||||
|
|
@ -150,7 +165,11 @@ impl ast::ExprTString {
|
|||
where
|
||||
V: SourceOrderVisitor<'a> + ?Sized,
|
||||
{
|
||||
let ast::ExprTString { value, range: _ } = self;
|
||||
let ast::ExprTString {
|
||||
value,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
|
||||
for t_string_part in value {
|
||||
match t_string_part {
|
||||
|
|
@ -173,7 +192,11 @@ impl ast::ExprStringLiteral {
|
|||
where
|
||||
V: SourceOrderVisitor<'a> + ?Sized,
|
||||
{
|
||||
let ast::ExprStringLiteral { value, range: _ } = self;
|
||||
let ast::ExprStringLiteral {
|
||||
value,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
|
||||
for string_literal in value {
|
||||
visitor.visit_string_literal(string_literal);
|
||||
|
|
@ -186,7 +209,11 @@ impl ast::ExprBytesLiteral {
|
|||
where
|
||||
V: SourceOrderVisitor<'a> + ?Sized,
|
||||
{
|
||||
let ast::ExprBytesLiteral { value, range: _ } = self;
|
||||
let ast::ExprBytesLiteral {
|
||||
value,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
|
||||
for bytes_literal in value {
|
||||
visitor.visit_bytes_literal(bytes_literal);
|
||||
|
|
@ -201,6 +228,7 @@ impl ast::ExceptHandlerExceptHandler {
|
|||
{
|
||||
let ast::ExceptHandlerExceptHandler {
|
||||
range: _,
|
||||
node_index: _,
|
||||
type_,
|
||||
name,
|
||||
body,
|
||||
|
|
@ -222,7 +250,11 @@ impl ast::PatternMatchValue {
|
|||
where
|
||||
V: SourceOrderVisitor<'a> + ?Sized,
|
||||
{
|
||||
let ast::PatternMatchValue { value, range: _ } = self;
|
||||
let ast::PatternMatchValue {
|
||||
value,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
visitor.visit_expr(value);
|
||||
}
|
||||
}
|
||||
|
|
@ -232,7 +264,11 @@ impl ast::PatternMatchSingleton {
|
|||
where
|
||||
V: SourceOrderVisitor<'a> + ?Sized,
|
||||
{
|
||||
let ast::PatternMatchSingleton { value, range: _ } = self;
|
||||
let ast::PatternMatchSingleton {
|
||||
value,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
visitor.visit_singleton(value);
|
||||
}
|
||||
}
|
||||
|
|
@ -242,7 +278,11 @@ impl ast::PatternMatchSequence {
|
|||
where
|
||||
V: SourceOrderVisitor<'a> + ?Sized,
|
||||
{
|
||||
let ast::PatternMatchSequence { patterns, range: _ } = self;
|
||||
let ast::PatternMatchSequence {
|
||||
patterns,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
for pattern in patterns {
|
||||
visitor.visit_pattern(pattern);
|
||||
}
|
||||
|
|
@ -259,6 +299,7 @@ impl ast::PatternMatchMapping {
|
|||
patterns,
|
||||
rest,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
|
||||
let mut rest = rest.as_ref();
|
||||
|
|
@ -289,6 +330,7 @@ impl ast::PatternMatchClass {
|
|||
cls,
|
||||
arguments: parameters,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
visitor.visit_expr(cls);
|
||||
visitor.visit_pattern_arguments(parameters);
|
||||
|
|
@ -300,7 +342,11 @@ impl ast::PatternMatchStar {
|
|||
where
|
||||
V: SourceOrderVisitor<'a> + ?Sized,
|
||||
{
|
||||
let ast::PatternMatchStar { range: _, name } = self;
|
||||
let ast::PatternMatchStar {
|
||||
range: _,
|
||||
node_index: _,
|
||||
name,
|
||||
} = self;
|
||||
|
||||
if let Some(name) = name {
|
||||
visitor.visit_identifier(name);
|
||||
|
|
@ -316,6 +362,7 @@ impl ast::PatternMatchAs {
|
|||
let ast::PatternMatchAs {
|
||||
pattern,
|
||||
range: _,
|
||||
node_index: _,
|
||||
name,
|
||||
} = self;
|
||||
if let Some(pattern) = pattern {
|
||||
|
|
@ -333,7 +380,11 @@ impl ast::PatternMatchOr {
|
|||
where
|
||||
V: SourceOrderVisitor<'a> + ?Sized,
|
||||
{
|
||||
let ast::PatternMatchOr { patterns, range: _ } = self;
|
||||
let ast::PatternMatchOr {
|
||||
patterns,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
for pattern in patterns {
|
||||
visitor.visit_pattern(pattern);
|
||||
}
|
||||
|
|
@ -347,6 +398,7 @@ impl ast::PatternArguments {
|
|||
{
|
||||
let PatternArguments {
|
||||
range: _,
|
||||
node_index: _,
|
||||
patterns,
|
||||
keywords,
|
||||
} = self;
|
||||
|
|
@ -368,6 +420,7 @@ impl ast::PatternKeyword {
|
|||
{
|
||||
let PatternKeyword {
|
||||
range: _,
|
||||
node_index: _,
|
||||
attr,
|
||||
pattern,
|
||||
} = self;
|
||||
|
|
@ -384,6 +437,7 @@ impl ast::Comprehension {
|
|||
{
|
||||
let ast::Comprehension {
|
||||
range: _,
|
||||
node_index: _,
|
||||
target,
|
||||
iter,
|
||||
ifs,
|
||||
|
|
@ -435,6 +489,7 @@ impl ast::Parameter {
|
|||
{
|
||||
let ast::Parameter {
|
||||
range: _,
|
||||
node_index: _,
|
||||
name,
|
||||
annotation,
|
||||
} = self;
|
||||
|
|
@ -453,6 +508,7 @@ impl ast::ParameterWithDefault {
|
|||
{
|
||||
let ast::ParameterWithDefault {
|
||||
range: _,
|
||||
node_index: _,
|
||||
parameter,
|
||||
default,
|
||||
} = self;
|
||||
|
|
@ -470,6 +526,7 @@ impl ast::Keyword {
|
|||
{
|
||||
let ast::Keyword {
|
||||
range: _,
|
||||
node_index: _,
|
||||
arg,
|
||||
value,
|
||||
} = self;
|
||||
|
|
@ -488,6 +545,7 @@ impl Alias {
|
|||
{
|
||||
let ast::Alias {
|
||||
range: _,
|
||||
node_index: _,
|
||||
name,
|
||||
asname,
|
||||
} = self;
|
||||
|
|
@ -506,6 +564,7 @@ impl ast::WithItem {
|
|||
{
|
||||
let ast::WithItem {
|
||||
range: _,
|
||||
node_index: _,
|
||||
context_expr,
|
||||
optional_vars,
|
||||
} = self;
|
||||
|
|
@ -525,6 +584,7 @@ impl ast::MatchCase {
|
|||
{
|
||||
let ast::MatchCase {
|
||||
range: _,
|
||||
node_index: _,
|
||||
pattern,
|
||||
guard,
|
||||
body,
|
||||
|
|
@ -545,6 +605,7 @@ impl ast::Decorator {
|
|||
{
|
||||
let ast::Decorator {
|
||||
range: _,
|
||||
node_index: _,
|
||||
expression,
|
||||
} = self;
|
||||
|
||||
|
|
@ -559,6 +620,7 @@ impl ast::TypeParams {
|
|||
{
|
||||
let ast::TypeParams {
|
||||
range: _,
|
||||
node_index: _,
|
||||
type_params,
|
||||
} = self;
|
||||
|
||||
|
|
@ -578,6 +640,7 @@ impl ast::TypeParamTypeVar {
|
|||
default,
|
||||
name,
|
||||
range: _,
|
||||
node_index: _,
|
||||
} = self;
|
||||
|
||||
visitor.visit_identifier(name);
|
||||
|
|
@ -598,6 +661,7 @@ impl ast::TypeParamTypeVarTuple {
|
|||
{
|
||||
let ast::TypeParamTypeVarTuple {
|
||||
range: _,
|
||||
node_index: _,
|
||||
name,
|
||||
default,
|
||||
} = self;
|
||||
|
|
@ -616,6 +680,7 @@ impl ast::TypeParamParamSpec {
|
|||
{
|
||||
let ast::TypeParamParamSpec {
|
||||
range: _,
|
||||
node_index: _,
|
||||
name,
|
||||
default,
|
||||
} = self;
|
||||
|
|
@ -634,6 +699,7 @@ impl ast::FString {
|
|||
let ast::FString {
|
||||
elements,
|
||||
range: _,
|
||||
node_index: _,
|
||||
flags: _,
|
||||
} = self;
|
||||
|
||||
|
|
@ -651,6 +717,7 @@ impl ast::TString {
|
|||
let ast::TString {
|
||||
elements,
|
||||
range: _,
|
||||
node_index: _,
|
||||
flags: _,
|
||||
} = self;
|
||||
|
||||
|
|
@ -668,6 +735,7 @@ impl ast::StringLiteral {
|
|||
{
|
||||
let ast::StringLiteral {
|
||||
range: _,
|
||||
node_index: _,
|
||||
value: _,
|
||||
flags: _,
|
||||
} = self;
|
||||
|
|
@ -682,6 +750,7 @@ impl ast::BytesLiteral {
|
|||
{
|
||||
let ast::BytesLiteral {
|
||||
range: _,
|
||||
node_index: _,
|
||||
value: _,
|
||||
flags: _,
|
||||
} = self;
|
||||
|
|
@ -694,7 +763,11 @@ impl ast::Identifier {
|
|||
where
|
||||
V: SourceOrderVisitor<'a> + ?Sized,
|
||||
{
|
||||
let ast::Identifier { range: _, id: _ } = self;
|
||||
let ast::Identifier {
|
||||
range: _,
|
||||
node_index: _,
|
||||
id: _,
|
||||
} = self;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue