mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-08-25 12:55:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Separate byteoffset ast and located ast

Browse source
This commit is contained in:
Jeong YunWon 2023-05-06 21:35:43 +09:00
parent f47dfca4e3
commit a14e43e03a
21 changed files with 893 additions and 562 deletions
Download patch file Download diff file Expand all files Collapse all files

3
ast/Cargo.toml
View file

@ -8,8 +8,9 @@ repository = "https://github.com/RustPython/RustPython"
license = "MIT"
[features]
default = ["constant-optimization", "fold"]
default = ["constant-optimization", "fold", "location"]
constant-optimization = ["fold"]
location = []
fold = []
unparse = ["rustpython-literal"]

423
ast/asdl_rs.py
View file

@ -7,6 +7,8 @@ import textwrap
from argparse import ArgumentParser
from pathlib import Path
from typing import Optional, Dict
from attr import dataclass
import asdl
@ -62,38 +64,62 @@ def asdl_of(name, obj):
return "{} = {}".format(name, types)
class EmitVisitor(asdl.VisitorBase):
"""Visit that emits lines"""
def __init__(self, file):
self.file = file
self.identifiers = set()
super(EmitVisitor, self).__init__()
def emit_identifier(self, name):
name = str(name)
if name in self.identifiers:
return
self.emit("_Py_IDENTIFIER(%s);" % name, 0)
self.identifiers.add(name)
def emit(self, line, depth):
if line:
line = (" " * TABSIZE * depth) + line
self.file.write(line + "\n")
class TypeInfo:
name: str
enum_name: Optional[str]
has_userdata: Optional[bool]
has_attributes: bool
children: set
boxed: bool
product: bool
has_expr: bool = False
def __init__(self, name):
self.name = name
self.enum_name = None
self.has_userdata = None
self.has_attributes = False
self.children = set()
self.boxed = False
self.product = False
self.product_has_expr = False
def __repr__(self):
return f"<TypeInfo: {self.name}>"
@property
def rust_name(self):
return get_rust_type(self.name)
@property
def sum_name(self):
if self.enum_name is None:
return self.name
else:
return f"{self.enum_name}_{self.name}"
@property
def rust_sum_name(self):
rust_name = get_rust_type(self.name)
if self.enum_name is None:
return rust_name
else:
name = get_rust_type(self.enum_name) + rust_name
return name
@property
def rust_suffix(self):
if self.product:
if self.has_attributes:
return "Data"
else:
return ""
else:
if self.has_attributes:
return "Kind"
else:
return ""
def determine_userdata(self, typeinfo, stack):
if self.name in stack:
return None
@ -110,6 +136,41 @@ class TypeInfo:
return self.has_userdata
class TypeInfoMixin:
typeinfo: Dict[str, TypeInfo]
def has_userdata(self, typ):
return self.typeinfo[typ].has_userdata
def get_generics(self, typ, *generics):
if self.has_userdata(typ):
return [f"<{g}>" for g in generics]
else:
return ["" for g in generics]
class EmitVisitor(asdl.VisitorBase, TypeInfoMixin):
"""Visit that emits lines"""
def __init__(self, file, typeinfo):
self.file = file
self.typeinfo = typeinfo
self.identifiers = set()
super(EmitVisitor, self).__init__()
def emit_identifier(self, name):
name = str(name)
if name in self.identifiers:
return
self.emit("_Py_IDENTIFIER(%s);" % name, 0)
self.identifiers.add(name)
def emit(self, line, depth):
if line:
line = (" " * TABSIZE * depth) + line
self.file.write(line + "\n")
class FindUserdataTypesVisitor(asdl.VisitorBase):
def __init__(self, typeinfo):
self.typeinfo = typeinfo
@ -132,21 +193,29 @@ class FindUserdataTypesVisitor(asdl.VisitorBase):
info.has_userdata = False
else:
for t in sum.types:
self.typeinfo[t.name] = TypeInfo(t.name)
if not t.fields:
continue
t_info = TypeInfo(t.name)
t_info.enum_name = name
self.typeinfo[t.name] = t_info
self.add_children(t.name, t.fields)
if len(sum.types) > 1:
info.boxed = True
if sum.attributes:
# attributes means Located, which has the `custom: U` field
# attributes means located, which has the `custom: U` field
info.has_userdata = True
info.has_attributes = True
for variant in sum.types:
self.add_children(name, variant.fields)
def visitProduct(self, product, name):
info = self.typeinfo[name]
if product.attributes:
# attributes means Located, which has the `custom: U` field
# attributes means located, which has the `custom: U` field
info.has_userdata = True
info.has_attributes = True
info.has_expr = product_has_expr(product)
if len(product.fields) > 2:
info.boxed = True
info.product = True
@ -163,24 +232,17 @@ def rust_field(field_name):
return field_name
class TypeInfoEmitVisitor(EmitVisitor):
def __init__(self, file, typeinfo):
self.typeinfo = typeinfo
super().__init__(file)
def has_userdata(self, typ):
return self.typeinfo[typ].has_userdata
def get_generics(self, typ, *generics):
if self.has_userdata(typ):
return [f"<{g}>" for g in generics]
else:
return ["" for g in generics]
def product_has_expr(product):
return any(f.type != "identifier" for f in product.fields)
class StructVisitor(TypeInfoEmitVisitor):
class StructVisitor(EmitVisitor):
"""Visitor to generate typedefs for AST."""
def __init__(self, *args, **kw):
super().__init__(*args, **kw)
self.rust_type_defs = []
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
@ -208,57 +270,56 @@ class StructVisitor(TypeInfoEmitVisitor):
def sum_with_constructors(self, sum, name, depth):
typeinfo = self.typeinfo[name]
enumname = rustname = get_rust_type(name)
suffix = typeinfo.rust_suffix
rustname = get_rust_type(name)
# all the attributes right now are for location, so if it has attrs we
# can just wrap it in Located<>
if sum.attributes:
enumname = rustname + "Kind"
# can just wrap it in Attributed<>
for t in sum.types:
if not t.fields:
continue
self.emit_attrs(depth)
self.typeinfo[t] = TypeInfo(t)
t_generics, t_generics_applied = self.get_generics(t.name, "U = ()", "U")
payload_name = f"{rustname}{t.name}"
self.emit(f"pub struct {payload_name}{t_generics} {{", depth)
for f in t.fields:
self.visit(f, typeinfo, "pub ", depth + 1, t.name)
self.emit("}", depth)
self.emit(
textwrap.dedent(
f"""
impl{t_generics_applied} From<{payload_name}{t_generics_applied}> for {enumname}{t_generics_applied} {{
fn from(payload: {payload_name}{t_generics_applied}) -> Self {{
{enumname}::{t.name}(payload)
}}
}}
"""
),
depth,
)
self.sum_subtype_struct(typeinfo, t, rustname, depth)
generics, generics_applied = self.get_generics(name, "U = ()", "U")
self.emit_attrs(depth)
self.emit(f"pub enum {enumname}{generics} {{", depth)
self.emit(f"pub enum {rustname}{suffix}{generics} {{", depth)
for t in sum.types:
if t.fields:
t_generics, t_generics_applied = self.get_generics(
t.name, "U = ()", "U"
)
(t_generics_applied,) = self.get_generics(t.name, "U")
self.emit(
f"{t.name}({rustname}{t.name}{t_generics_applied}),", depth + 1
)
else:
self.emit(f"{t.name},", depth + 1)
self.emit("}", depth)
if sum.attributes:
if typeinfo.has_attributes:
self.emit(
f"pub type {rustname}<U = ()> = Located<{enumname}{generics_applied}, U>;",
f"pub type {rustname}<U = ()> = Attributed<{rustname}{suffix}{generics_applied}, U>;",
depth,
)
self.emit("", depth)
def sum_subtype_struct(self, sum_typeinfo, t, rustname, depth):
self.emit_attrs(depth)
generics, generics_applied = self.get_generics(t.name, "U = ()", "U")
payload_name = f"{rustname}{t.name}"
self.emit(f"pub struct {payload_name}{generics} {{", depth)
for f in t.fields:
self.visit(f, sum_typeinfo, "pub ", depth + 1, t.name)
self.emit("}", depth)
self.emit(
textwrap.dedent(
f"""
impl{generics_applied} From<{payload_name}{generics_applied}> for {rustname}{sum_typeinfo.rust_suffix}{generics_applied} {{
fn from(payload: {payload_name}{generics_applied}) -> Self {{
{rustname}{sum_typeinfo.rust_suffix}::{t.name}(payload)
}}
}}
"""
),
depth,
)
def visitConstructor(self, cons, parent, depth):
if cons.fields:
self.emit(f"{cons.name} {{", depth)
@ -300,7 +361,7 @@ class StructVisitor(TypeInfoEmitVisitor):
if product.attributes:
dataname = rustname + "Data"
self.emit_attrs(depth)
has_expr = any(f.type != "identifier" for f in product.fields)
has_expr = product_has_expr(product)
if has_expr:
datadef = f"{dataname}{generics}"
else:
@ -314,20 +375,24 @@ class StructVisitor(TypeInfoEmitVisitor):
if not has_expr:
generics_applied = ""
self.emit(
f"pub type {rustname}<U = ()> = Located<{dataname}{generics_applied}, U>;",
f"pub type {rustname}<U = ()> = Attributed<{dataname}{generics_applied}, U>;",
depth,
)
self.emit("", depth)
class FoldTraitDefVisitor(TypeInfoEmitVisitor):
class FoldTraitDefVisitor(EmitVisitor):
def visitModule(self, mod, depth):
self.emit("pub trait Fold<U> {", depth)
self.emit("type TargetU;", depth + 1)
self.emit("type Error;", depth + 1)
self.emit(
"fn map_user(&mut self, user: U) -> Result<Self::TargetU, Self::Error>;",
depth + 2,
depth + 1,
)
self.emit(
"fn map_located<T>(&mut self, located: Attributed<T, U>) -> Result<Attributed<T, Self::TargetU>, Self::Error> { let custom = self.map_user(located.custom)?; Ok(Attributed { range: located.range, custom, node: located.node }) }",
depth + 1,
)
for dfn in mod.dfns:
self.visit(dfn, depth + 2)
@ -345,14 +410,14 @@ class FoldTraitDefVisitor(TypeInfoEmitVisitor):
self.emit("}", depth)
class FoldImplVisitor(TypeInfoEmitVisitor):
class FoldImplVisitor(EmitVisitor):
def visitModule(self, mod, depth):
self.emit(
"fn fold_located<U, F: Fold<U> + ?Sized, T, MT>(folder: &mut F, node: Located<T, U>, f: impl FnOnce(&mut F, T) -> Result<MT, F::Error>) -> Result<Located<MT, F::TargetU>, F::Error> {",
"fn fold_located<U, F: Fold<U> + ?Sized, T, MT>(folder: &mut F, node: Attributed<T, U>, f: impl FnOnce(&mut F, T) -> Result<MT, F::Error>) -> Result<Attributed<MT, F::TargetU>, F::Error> {",
depth,
)
self.emit(
"Ok(Located { custom: folder.map_user(node.custom)?, range: node.range, node: f(folder, node.node)? })",
"let node = folder.map_located(node)?; Ok(Attributed { custom: node.custom, range: node.range, node: f(folder, node.node)? })",
depth + 1,
)
self.emit("}", depth)
@ -363,11 +428,11 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
self.visit(type.value, type.name, depth)
def visitSum(self, sum, name, depth):
typeinfo = self.typeinfo[name]
apply_t, apply_u, apply_target_u = self.get_generics(
name, "T", "U", "F::TargetU"
)
enumname = get_rust_type(name)
is_located = bool(sum.attributes)
self.emit(f"impl<T, U> Foldable<T, U> for {enumname}{apply_t} {{", depth)
self.emit(f"type Mapped = {enumname}{apply_u};", depth + 1)
@ -383,15 +448,13 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
f"pub fn fold_{name}<U, F: Fold<U> + ?Sized>(#[allow(unused)] folder: &mut F, node: {enumname}{apply_u}) -> Result<{enumname}{apply_target_u}, F::Error> {{",
depth,
)
if is_located:
if typeinfo.has_attributes:
self.emit("fold_located(folder, node, |folder, node| {", depth)
rustname = enumname + "Kind"
else:
rustname = enumname
self.emit("match node {", depth + 1)
for cons in sum.types:
fields_pattern = self.make_pattern(
enumname, rustname, cons.name, cons.fields
enumname, typeinfo.rust_suffix, cons.name, cons.fields
)
self.emit(
f"{fields_pattern[0]} {{ {fields_pattern[1]} }} {fields_pattern[2]} => {{",
@ -402,7 +465,7 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
)
self.emit("}", depth + 2)
self.emit("}", depth + 1)
if is_located:
if typeinfo.has_attributes:
self.emit("})", depth)
self.emit("}", depth)
@ -411,7 +474,7 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
name, "T", "U", "F::TargetU"
)
structname = get_rust_type(name)
is_located = bool(product.attributes)
has_attributes = bool(product.attributes)
self.emit(f"impl<T, U> Foldable<T, U> for {structname}{apply_t} {{", depth)
self.emit(f"type Mapped = {structname}{apply_u};", depth + 1)
@ -427,7 +490,7 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
f"pub fn fold_{name}<U, F: Fold<U> + ?Sized>(#[allow(unused)] folder: &mut F, node: {structname}{apply_u}) -> Result<{structname}{apply_target_u}, F::Error> {{",
depth,
)
if is_located:
if has_attributes:
self.emit("fold_located(folder, node, |folder, node| {", depth)
rustname = structname + "Data"
else:
@ -435,16 +498,16 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
fields_pattern = self.make_pattern(rustname, structname, None, product.fields)
self.emit(f"let {rustname} {{ {fields_pattern[1]} }} = node;", depth + 1)
self.gen_construction(rustname, product.fields, "", depth + 1)
if is_located:
if has_attributes:
self.emit("})", depth)
self.emit("}", depth)
def make_pattern(self, rustname, pyname, fieldname, fields):
def make_pattern(self, rustname, suffix, fieldname, fields):
if fields:
header = f"{pyname}::{fieldname}({rustname}{fieldname}"
header = f"{rustname}{suffix}::{fieldname}({rustname}{fieldname}"
footer = ")"
else:
header = f"{pyname}::{fieldname}"
header = f"{rustname}{suffix}::{fieldname}"
footer = ""
body = ",".join(rust_field(f.name) for f in fields)
@ -458,7 +521,7 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
self.emit(f"}}{footer})", depth)
class FoldModuleVisitor(TypeInfoEmitVisitor):
class FoldModuleVisitor(EmitVisitor):
def visitModule(self, mod):
depth = 0
self.emit('#[cfg(feature = "fold")]', depth)
@ -576,10 +639,10 @@ class TraitImplVisitor(EmitVisitor):
if sum.attributes:
rustname = enumname + "Kind"
self.emit(f"impl NamedNode for ast::{rustname} {{", depth)
self.emit(f"impl NamedNode for ast::located::{rustname} {{", depth)
self.emit(f"const NAME: &'static str = {json.dumps(name)};", depth + 1)
self.emit("}", depth)
self.emit(f"impl Node for ast::{rustname} {{", depth)
self.emit(f"impl Node for ast::located::{rustname} {{", depth)
self.emit(
"fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef {", depth + 1
)
@ -597,10 +660,12 @@ class TraitImplVisitor(EmitVisitor):
self.emit("}", depth)
def constructor_to_object(self, cons, enumname, rustname, depth):
self.emit(f"ast::{rustname}::{cons.name}", depth)
self.emit(f"ast::located::{rustname}::{cons.name}", depth)
if cons.fields:
fields_pattern = self.make_pattern(cons.fields)
self.emit(f"( ast::{enumname}{cons.name} {{ {fields_pattern} }} )", depth)
self.emit(
f"( ast::located::{enumname}{cons.name} {{ {fields_pattern} }} )", depth
)
self.emit(" => {", depth)
self.make_node(cons.name, cons.fields, depth + 1)
self.emit("}", depth)
@ -610,15 +675,17 @@ class TraitImplVisitor(EmitVisitor):
if product.attributes:
structname += "Data"
self.emit(f"impl NamedNode for ast::{structname} {{", depth)
self.emit(f"impl NamedNode for ast::located::{structname} {{", depth)
self.emit(f"const NAME: &'static str = {json.dumps(name)};", depth + 1)
self.emit("}", depth)
self.emit(f"impl Node for ast::{structname} {{", depth)
self.emit(f"impl Node for ast::located::{structname} {{", depth)
self.emit(
"fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef {", depth + 1
)
fields_pattern = self.make_pattern(product.fields)
self.emit(f"let ast::{structname} {{ {fields_pattern} }} = self;", depth + 2)
self.emit(
f"let ast::located::{structname} {{ {fields_pattern} }} = self;", depth + 2
)
self.make_node(name, product.fields, depth + 2)
self.emit("}", depth + 1)
self.emit(
@ -656,11 +723,14 @@ class TraitImplVisitor(EmitVisitor):
for cons in sum.types:
self.emit(f"if _cls.is(Node{cons.name}::static_type()) {{", depth)
if cons.fields:
self.emit(f"ast::{rustname}::{cons.name} (ast::{enumname}{cons.name} {{", depth + 1)
self.emit(
f"ast::located::{rustname}::{cons.name} (ast::located::{enumname}{cons.name} {{",
depth + 1,
)
self.gen_construction_fields(cons, sumname, depth + 1)
self.emit("})", depth + 1)
else:
self.emit(f"ast::{rustname}::{cons.name}", depth + 1)
self.emit(f"ast::located::{rustname}::{cons.name}", depth + 1)
self.emit("} else", depth)
self.emit("{", depth)
@ -684,14 +754,14 @@ class TraitImplVisitor(EmitVisitor):
)
def gen_construction(self, cons_path, cons, name, depth):
self.emit(f"ast::{cons_path} {{", depth)
self.emit(f"ast::located::{cons_path} {{", depth)
self.gen_construction_fields(cons, name, depth + 1)
self.emit("}", depth)
def extract_location(self, typename, depth):
row = self.decode_field(asdl.Field("int", "lineno"), typename)
column = self.decode_field(asdl.Field("int", "col_offset"), typename)
self.emit(f"let _location = ast::Location::new({row}, {column});", depth)
self.emit(f"let _location = Location::new({row}, {column});", depth)
def decode_field(self, field, typename):
name = json.dumps(field.name)
@ -711,84 +781,18 @@ class ChainOfVisitors:
v.emit("", 0)
def write_ast_def(mod, typeinfo, f):
def write_generic_def(mod, typeinfo, f):
f.write(
textwrap.dedent(
"""
#![allow(clippy::derive_partial_eq_without_eq)]
pub use crate::constant::*;
pub use rustpython_compiler_core::text_size::{TextSize, TextRange};
pub use crate::{Attributed, constant::*};
use rustpython_compiler_core::{text_size::{TextSize, TextRange}};
type Ident = String;
\n
"""
)
)
StructVisitor(f, typeinfo).emit_attrs(0)
f.write(
textwrap.dedent(
"""
pub struct Located<T, U = ()> {
pub range: TextRange,
pub custom: U,
pub node: T,
}
impl<T> Located<T> {
pub fn new(start: TextSize, end: TextSize, node: T) -> Self {
Self { range: TextRange::new(start, end), custom: (), node }
}
/// Creates a new node that spans the position specified by `range`.
pub fn with_range(node: T, range: TextRange) -> Self {
Self {
range,
custom: (),
node,
}
}
/// Returns the absolute start position of the node from the beginning of the document.
#[inline]
pub const fn start(&self) -> TextSize {
self.range.start()
}
/// Returns the node
#[inline]
pub fn node(&self) -> &T {
&self.node
}
/// Consumes self and returns the node.
#[inline]
pub fn into_node(self) -> T {
self.node
}
/// Returns the `range` of the node. The range offsets are absolute to the start of the document.
#[inline]
pub const fn range(&self) -> TextRange {
self.range
}
/// Returns the absolute position at which the node ends in the source document.
#[inline]
pub const fn end(&self) -> TextSize {
self.range.end()
}
}
impl<T, U> std::ops::Deref for Located<T, U> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.node
}
}
\n
""".lstrip()
"""
)
)
@ -796,24 +800,59 @@ def write_ast_def(mod, typeinfo, f):
c.visit(mod)
def write_ast_mod(mod, f):
def write_located_def(typeinfo, f):
f.write(
textwrap.dedent(
"""
#![allow(clippy::all)]
use rustpython_compiler_core::LocationRange;
use super::*;
use crate::common::ascii;
pub type Located<T> = super::generic::Attributed<T, LocationRange>;
"""
)
)
for info in typeinfo.values():
if info.has_userdata:
generics = "::<LocationRange>"
else:
generics = ""
f.write(
f"pub type {info.rust_sum_name} = super::generic::{info.rust_sum_name}{generics};\n"
)
if info.rust_suffix:
if info.rust_suffix == "Data" and not info.has_expr:
generics = ""
f.write(
f"pub type {info.rust_sum_name}{info.rust_suffix} = super::generic::{info.rust_sum_name}{info.rust_suffix}{generics};\n"
)
"""
def write_ast_mod(mod, typeinfo, f):
f.write(
textwrap.dedent(
"""
#![allow(clippy::all)]
use super::*;
use crate::common::ascii;
"""
)
)
c = ChainOfVisitors(ClassDefVisitor(f), TraitImplVisitor(f), ExtendModuleVisitor(f))
c = ChainOfVisitors(
ClassDefVisitor(f, typeinfo),
TraitImplVisitor(f, typeinfo),
ExtendModuleVisitor(f, typeinfo),
)
c.visit(mod)
def main(input_filename, ast_mod_filename, ast_def_filename, dump_module=False):
def main(
input_filename,
generic_filename,
located_filename,
module_filename,
dump_module=False,
):
auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:]))
mod = asdl.parse(input_filename)
if dump_module:
@ -825,22 +864,34 @@ def main(input_filename, ast_mod_filename, ast_def_filename, dump_module=False):
typeinfo = {}
FindUserdataTypesVisitor(typeinfo).visit(mod)
with ast_def_filename.open("w") as def_file, ast_mod_filename.open("w") as mod_file:
def_file.write(auto_gen_msg)
write_ast_def(mod, typeinfo, def_file)
with generic_filename.open("w") as generic_file, located_filename.open(
"w"
) as located_file:
generic_file.write(auto_gen_msg)
write_generic_def(mod, typeinfo, generic_file)
located_file.write(auto_gen_msg)
write_located_def(typeinfo, located_file)
mod_file.write(auto_gen_msg)
write_ast_mod(mod, mod_file)
with module_filename.open("w") as module_file:
module_file.write(auto_gen_msg)
write_ast_mod(mod, typeinfo, module_file)
print(f"{ast_def_filename}, {ast_mod_filename} regenerated.")
print(f"{generic_filename}, {located_filename}, {module_filename} regenerated.")
if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("input_file", type=Path)
parser.add_argument("-M", "--mod-file", type=Path, required=True)
parser.add_argument("-D", "--def-file", type=Path, required=True)
parser.add_argument("-G", "--generic-file", type=Path, required=True)
parser.add_argument("-L", "--located-file", type=Path, required=True)
parser.add_argument("-M", "--module-file", type=Path, required=True)
parser.add_argument("-d", "--dump-module", action="store_true")
args = parser.parse_args()
main(args.input_file, args.mod_file, args.def_file, args.dump_module)
main(
args.input_file,
args.generic_file,
args.located_file,
args.module_file,
args.dump_module,
)

76
ast/src/attributed.rs Normal file
View file

@ -0,0 +1,76 @@
use rustpython_compiler_core::{
text_size::{TextRange, TextSize},
Location, LocationRange,
};
#[derive(Clone, Debug, PartialEq)]
pub struct Attributed<T, U = ()> {
pub range: TextRange,
pub custom: U,
pub node: T,
}
impl<T, U> Attributed<T, U> {
/// Returns the node
#[inline]
pub fn node(&self) -> &T {
&self.node
}
/// Returns the `range` of the node. The range offsets are absolute to the start of the document.
#[inline]
pub const fn range(&self) -> TextRange {
self.range
}
/// Returns the absolute start position of the node from the beginning of the document.
#[inline]
pub const fn start(&self) -> TextSize {
self.range.start()
}
/// Returns the absolute position at which the node ends in the source document.
#[inline]
pub const fn end(&self) -> TextSize {
self.range.end()
}
}
impl<T> Attributed<T, ()> {
/// Creates a new node that spans the position specified by `range`.
pub fn new(range: impl Into<TextRange>, node: T) -> Self {
Self {
range: range.into(),
custom: (),
node,
}
}
/// Consumes self and returns the node.
#[inline]
pub fn into_node(self) -> T {
self.node
}
}
impl<T> Attributed<T, LocationRange> {
/// Returns the absolute start position of the node from the beginning of the document.
#[inline]
pub const fn location(&self) -> Location {
self.custom.start
}
/// Returns the absolute position at which the node ends in the source document.
#[inline]
pub const fn end_location(&self) -> Location {
self.custom.end
}
}
impl<T, U> std::ops::Deref for Attributed<T, U> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.node
}
}

17
ast/src/constant.rs
View file

@ -137,6 +137,7 @@ impl<U> crate::fold::Fold<U> for ConstantOptimizer {
#[cfg(test)]
mod tests {
use super::*;
use rustpython_compiler_core::text_size::TextRange;
#[cfg(feature = "constant-optimization")]
#[test]
@ -146,13 +147,13 @@ mod tests {
let range = TextRange::default();
#[allow(clippy::let_unit_value)]
let custom = ();
let ast = Located {
let ast = Attributed {
range,
custom,
node: ExprTuple {
ctx: ExprContext::Load,
elts: vec![
Located {
Attributed {
range,
custom,
node: ExprConstant {
@ -161,7 +162,7 @@ mod tests {
}
.into(),
},
Located {
Attributed {
range,
custom,
node: ExprConstant {
@ -170,13 +171,13 @@ mod tests {
}
.into(),
},
Located {
Attributed {
range,
custom,
node: ExprTuple {
ctx: ExprContext::Load,
elts: vec![
Located {
Attributed {
range,
custom,
node: ExprConstant {
@ -185,7 +186,7 @@ mod tests {
}
.into(),
},
Located {
Attributed {
range,
custom,
node: ExprConstant {
@ -194,7 +195,7 @@ mod tests {
}
.into(),
},
Located {
Attributed {
range,
custom,
node: ExprConstant {
@ -216,7 +217,7 @@ mod tests {
.unwrap_or_else(|e| match e {});
assert_eq!(
new_ast,
Located {
Attributed {
range,
custom,
node: ExprConstant {

2
ast/src/fold_helpers.rs
View file

@ -1,6 +1,6 @@
use crate::{constant, fold::Fold};
pub(crate) trait Foldable<T, U> {
pub trait Foldable<T, U> {
type Mapped;
fn fold<F: Fold<T, TargetU = U> + ?Sized>(
self,

102
ast/src/ast_gen.rs → ast/src/generic.rs
View file

@ -2,75 +2,11 @@
#![allow(clippy::derive_partial_eq_without_eq)]
pub use crate::constant::*;
pub use rustpython_compiler_core::text_size::{TextRange, TextSize};
pub use crate::{constant::*, Attributed};
use rustpython_compiler_core::text_size::{TextRange, TextSize};
type Ident = String;
#[derive(Clone, Debug, PartialEq)]
pub struct Located<T, U = ()> {
pub range: TextRange,
pub custom: U,
pub node: T,
}
impl<T> Located<T> {
pub fn new(start: TextSize, end: TextSize, node: T) -> Self {
Self {
range: TextRange::new(start, end),
custom: (),
node,
}
}
/// Creates a new node that spans the position specified by `range`.
pub fn with_range(node: T, range: TextRange) -> Self {
Self {
range,
custom: (),
node,
}
}
/// Returns the absolute start position of the node from the beginning of the document.
#[inline]
pub const fn start(&self) -> TextSize {
self.range.start()
}
/// Returns the node
#[inline]
pub fn node(&self) -> &T {
&self.node
}
/// Consumes self and returns the node.
#[inline]
pub fn into_node(self) -> T {
self.node
}
/// Returns the `range` of the node. The range offsets are absolute to the start of the document.
#[inline]
pub const fn range(&self) -> TextRange {
self.range
}
/// Returns the absolute position at which the node ends in the source document.
#[inline]
pub const fn end(&self) -> TextSize {
self.range.end()
}
}
impl<T, U> std::ops::Deref for Located<T, U> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.node
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct ModModule<U = ()> {
pub body: Vec<Stmt<U>>,
@ -467,7 +403,7 @@ pub enum StmtKind<U = ()> {
Break,
Continue,
}
pub type Stmt<U = ()> = Located<StmtKind<U>, U>;
pub type Stmt<U = ()> = Attributed<StmtKind<U>, U>;
#[derive(Clone, Debug, PartialEq)]
pub struct ExprBoolOp<U = ()> {
@ -827,7 +763,7 @@ pub enum ExprKind<U = ()> {
Tuple(ExprTuple<U>),
Slice(ExprSlice<U>),
}
pub type Expr<U = ()> = Located<ExprKind<U>, U>;
pub type Expr<U = ()> = Attributed<ExprKind<U>, U>;
#[derive(Clone, Debug, PartialEq)]
pub enum ExprContext {
@ -906,7 +842,7 @@ impl<U> From<ExcepthandlerExceptHandler<U>> for ExcepthandlerKind<U> {
pub enum ExcepthandlerKind<U = ()> {
ExceptHandler(ExcepthandlerExceptHandler<U>),
}
pub type Excepthandler<U = ()> = Located<ExcepthandlerKind<U>, U>;
pub type Excepthandler<U = ()> = Attributed<ExcepthandlerKind<U>, U>;
#[derive(Clone, Debug, PartialEq)]
pub struct Arguments<U = ()> {
@ -925,21 +861,21 @@ pub struct ArgData<U = ()> {
pub annotation: Option<Box<Expr<U>>>,
pub type_comment: Option<String>,
}
pub type Arg<U = ()> = Located<ArgData<U>, U>;
pub type Arg<U = ()> = Attributed<ArgData<U>, U>;
#[derive(Clone, Debug, PartialEq)]
pub struct KeywordData<U = ()> {
pub arg: Option<Ident>,
pub value: Expr<U>,
}
pub type Keyword<U = ()> = Located<KeywordData<U>, U>;
pub type Keyword<U = ()> = Attributed<KeywordData<U>, U>;
#[derive(Clone, Debug, PartialEq)]
pub struct AliasData {
pub name: Ident,
pub asname: Option<Ident>,
}
pub type Alias<U = ()> = Located<AliasData, U>;
pub type Alias<U = ()> = Attributed<AliasData, U>;
#[derive(Clone, Debug, PartialEq)]
pub struct Withitem<U = ()> {
@ -1059,7 +995,7 @@ pub enum PatternKind<U = ()> {
MatchAs(PatternMatchAs<U>),
MatchOr(PatternMatchOr<U>),
}
pub type Pattern<U = ()> = Located<PatternKind<U>, U>;
pub type Pattern<U = ()> = Attributed<PatternKind<U>, U>;
#[derive(Clone, Debug, PartialEq)]
pub struct TypeIgnoreTypeIgnore {
@ -1086,6 +1022,17 @@ pub mod fold {
type TargetU;
type Error;
fn map_user(&mut self, user: U) -> Result<Self::TargetU, Self::Error>;
fn map_located<T>(
&mut self,
located: Attributed<T, U>,
) -> Result<Attributed<T, Self::TargetU>, Self::Error> {
let custom = self.map_user(located.custom)?;
Ok(Attributed {
range: located.range,
custom,
node: located.node,
})
}
fn fold_mod(&mut self, node: Mod<U>) -> Result<Mod<Self::TargetU>, Self::Error> {
fold_mod(self, node)
}
@ -1164,11 +1111,12 @@ pub mod fold {
}
fn fold_located<U, F: Fold<U> + ?Sized, T, MT>(
folder: &mut F,
node: Located<T, U>,
node: Attributed<T, U>,
f: impl FnOnce(&mut F, T) -> Result<MT, F::Error>,
) -> Result<Located<MT, F::TargetU>, F::Error> {
Ok(Located {
custom: folder.map_user(node.custom)?,
) -> Result<Attributed<MT, F::TargetU>, F::Error> {
let node = folder.map_located(node)?;
Ok(Attributed {
custom: node.custom,
range: node.range,
node: f(folder, node.node)?,
})

14
ast/src/lib.rs
View file

@ -1,11 +1,21 @@
mod ast_gen;
mod attributed;
mod constant;
#[cfg(feature = "fold")]
mod fold_helpers;
mod generic;
mod impls;
#[cfg(feature = "location")]
pub mod located;
#[cfg(feature = "location")]
mod locator;
#[cfg(feature = "unparse")]
mod unparse;
pub use ast_gen::*;
pub use attributed::Attributed;
pub use constant::{Constant, ConversionFlag};
pub use generic::*;
#[cfg(feature = "location")]
pub use locator::Locator;
pub type Suite<U = ()> = Vec<Stmt<U>>;

95
ast/src/located.rs Normal file
View file

@ -0,0 +1,95 @@
// File automatically generated by ast/asdl_rs.py.
use rustpython_compiler_core::LocationRange;
pub type Located<T> = super::generic::Attributed<T, LocationRange>;
pub type Mod = super::generic::Mod<LocationRange>;
pub type ModModule = super::generic::ModModule<LocationRange>;
pub type ModInteractive = super::generic::ModInteractive<LocationRange>;
pub type ModExpression = super::generic::ModExpression<LocationRange>;
pub type ModFunctionType = super::generic::ModFunctionType<LocationRange>;
pub type Stmt = super::generic::Stmt<LocationRange>;
pub type StmtKind = super::generic::StmtKind<LocationRange>;
pub type StmtFunctionDef = super::generic::StmtFunctionDef<LocationRange>;
pub type StmtAsyncFunctionDef = super::generic::StmtAsyncFunctionDef<LocationRange>;
pub type StmtClassDef = super::generic::StmtClassDef<LocationRange>;
pub type StmtReturn = super::generic::StmtReturn<LocationRange>;
pub type StmtDelete = super::generic::StmtDelete<LocationRange>;
pub type StmtAssign = super::generic::StmtAssign<LocationRange>;
pub type StmtAugAssign = super::generic::StmtAugAssign<LocationRange>;
pub type StmtAnnAssign = super::generic::StmtAnnAssign<LocationRange>;
pub type StmtFor = super::generic::StmtFor<LocationRange>;
pub type StmtAsyncFor = super::generic::StmtAsyncFor<LocationRange>;
pub type StmtWhile = super::generic::StmtWhile<LocationRange>;
pub type StmtIf = super::generic::StmtIf<LocationRange>;
pub type StmtWith = super::generic::StmtWith<LocationRange>;
pub type StmtAsyncWith = super::generic::StmtAsyncWith<LocationRange>;
pub type StmtMatch = super::generic::StmtMatch<LocationRange>;
pub type StmtRaise = super::generic::StmtRaise<LocationRange>;
pub type StmtTry = super::generic::StmtTry<LocationRange>;
pub type StmtTryStar = super::generic::StmtTryStar<LocationRange>;
pub type StmtAssert = super::generic::StmtAssert<LocationRange>;
pub type StmtImport = super::generic::StmtImport<LocationRange>;
pub type StmtImportFrom = super::generic::StmtImportFrom<LocationRange>;
pub type StmtGlobal = super::generic::StmtGlobal;
pub type StmtNonlocal = super::generic::StmtNonlocal;
pub type StmtExpr = super::generic::StmtExpr<LocationRange>;
pub type Expr = super::generic::Expr<LocationRange>;
pub type ExprKind = super::generic::ExprKind<LocationRange>;
pub type ExprBoolOp = super::generic::ExprBoolOp<LocationRange>;
pub type ExprNamedExpr = super::generic::ExprNamedExpr<LocationRange>;
pub type ExprBinOp = super::generic::ExprBinOp<LocationRange>;
pub type ExprUnaryOp = super::generic::ExprUnaryOp<LocationRange>;
pub type ExprLambda = super::generic::ExprLambda<LocationRange>;
pub type ExprIfExp = super::generic::ExprIfExp<LocationRange>;
pub type ExprDict = super::generic::ExprDict<LocationRange>;
pub type ExprSet = super::generic::ExprSet<LocationRange>;
pub type ExprListComp = super::generic::ExprListComp<LocationRange>;
pub type ExprSetComp = super::generic::ExprSetComp<LocationRange>;
pub type ExprDictComp = super::generic::ExprDictComp<LocationRange>;
pub type ExprGeneratorExp = super::generic::ExprGeneratorExp<LocationRange>;
pub type ExprAwait = super::generic::ExprAwait<LocationRange>;
pub type ExprYield = super::generic::ExprYield<LocationRange>;
pub type ExprYieldFrom = super::generic::ExprYieldFrom<LocationRange>;
pub type ExprCompare = super::generic::ExprCompare<LocationRange>;
pub type ExprCall = super::generic::ExprCall<LocationRange>;
pub type ExprFormattedValue = super::generic::ExprFormattedValue<LocationRange>;
pub type ExprJoinedStr = super::generic::ExprJoinedStr<LocationRange>;
pub type ExprConstant = super::generic::ExprConstant;
pub type ExprAttribute = super::generic::ExprAttribute<LocationRange>;
pub type ExprSubscript = super::generic::ExprSubscript<LocationRange>;
pub type ExprStarred = super::generic::ExprStarred<LocationRange>;
pub type ExprName = super::generic::ExprName;
pub type ExprList = super::generic::ExprList<LocationRange>;
pub type ExprTuple = super::generic::ExprTuple<LocationRange>;
pub type ExprSlice = super::generic::ExprSlice<LocationRange>;
pub type ExprContext = super::generic::ExprContext;
pub type Boolop = super::generic::Boolop;
pub type Operator = super::generic::Operator;
pub type Unaryop = super::generic::Unaryop;
pub type Cmpop = super::generic::Cmpop;
pub type Comprehension = super::generic::Comprehension<LocationRange>;
pub type Excepthandler = super::generic::Excepthandler<LocationRange>;
pub type ExcepthandlerKind = super::generic::ExcepthandlerKind<LocationRange>;
pub type ExcepthandlerExceptHandler = super::generic::ExcepthandlerExceptHandler<LocationRange>;
pub type Arguments = super::generic::Arguments<LocationRange>;
pub type Arg = super::generic::Arg<LocationRange>;
pub type ArgData = super::generic::ArgData<LocationRange>;
pub type Keyword = super::generic::Keyword<LocationRange>;
pub type KeywordData = super::generic::KeywordData<LocationRange>;
pub type Alias = super::generic::Alias<LocationRange>;
pub type AliasData = super::generic::AliasData;
pub type Withitem = super::generic::Withitem<LocationRange>;
pub type MatchCase = super::generic::MatchCase<LocationRange>;
pub type Pattern = super::generic::Pattern<LocationRange>;
pub type PatternKind = super::generic::PatternKind<LocationRange>;
pub type PatternMatchValue = super::generic::PatternMatchValue<LocationRange>;
pub type PatternMatchSingleton = super::generic::PatternMatchSingleton;
pub type PatternMatchSequence = super::generic::PatternMatchSequence<LocationRange>;
pub type PatternMatchMapping = super::generic::PatternMatchMapping<LocationRange>;
pub type PatternMatchClass = super::generic::PatternMatchClass<LocationRange>;
pub type PatternMatchStar = super::generic::PatternMatchStar;
pub type PatternMatchAs = super::generic::PatternMatchAs<LocationRange>;
pub type PatternMatchOr = super::generic::PatternMatchOr<LocationRange>;
pub type TypeIgnore = super::generic::TypeIgnore;
pub type TypeIgnoreTypeIgnore = super::generic::TypeIgnoreTypeIgnore;

57
ast/src/locator.rs Normal file
View file

@ -0,0 +1,57 @@
use crate::attributed::Attributed;
use crate::fold_helpers::Foldable;
use rustpython_compiler_core::{
text_size::{TextRange, TextSize},
Location, LocationRange,
};
/// Converts source code byte-offset to Python convention line and column numbers.
#[derive(Default)]
pub struct Locator<'a> {
source: &'a str,
}
impl<'a> Locator<'a> {
#[inline]
pub fn new(source: &'a str) -> Self {
Self { source }
}
pub fn source(&'a self) -> &'a str {
self.source
}
pub fn locate(&mut self, offset: TextSize) -> Location {
todo!()
}
pub fn locate_range(&mut self, range: TextRange) -> LocationRange {
self.locate(range.start())..self.locate(range.end())
}
pub fn locate_ast<X: Foldable<(), LocationRange>>(&mut self, ast: X) -> X::Mapped {
ast.fold(self).unwrap()
}
}
impl crate::fold::Fold<()> for Locator<'_> {
type TargetU = LocationRange;
type Error = std::convert::Infallible;
#[cold]
fn map_user(&mut self, _user: ()) -> Result<Self::TargetU, Self::Error> {
unreachable!("implemented map_located");
}
fn map_located<T>(
&mut self,
node: Attributed<T, ()>,
) -> Result<Attributed<T, Self::TargetU>, Self::Error> {
let location = self.locate_range(node.range);
Ok(Attributed {
range: node.range,
custom: location,
node: node.node,
})
}
}

75
core/src/error.rs
View file

@ -1,11 +1,10 @@
use ruff_text_size::TextSize;
use std::error::Error as StdError;
use crate::{text_size::TextSize, Location};
use std::fmt::Display;
#[derive(Debug, PartialEq, Eq)]
pub struct BaseError<T> {
pub error: T,
pub location: TextSize,
pub offset: TextSize,
pub source_path: String,
}
@ -17,11 +16,11 @@ impl<T> std::ops::Deref for BaseError<T> {
}
}
impl<T> StdError for BaseError<T>
impl<T> std::error::Error for BaseError<T>
where
T: StdError + 'static,
T: std::error::Error + 'static,
{
fn source(&self) -> Option<&(dyn StdError + 'static)> {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.error)
}
}
@ -35,7 +34,7 @@ where
f,
"{} at byte offset {}",
&self.error,
u32::from(self.location)
u32::from(self.offset)
)
}
}
@ -51,7 +50,7 @@ impl<T> BaseError<T> {
{
Self {
error: obj.error.into(),
location: obj.location,
offset: obj.offset,
source_path: obj.source_path,
}
}
@ -62,4 +61,64 @@ impl<T> BaseError<T> {
{
BaseError::from(self)
}
pub fn into_located<U>(self, locator: &str) -> LocatedError<U>
where
T: Into<U>,
{
todo!()
}
}
#[derive(Debug, PartialEq, Eq)]
pub struct LocatedError<T> {
pub error: T,
pub location: Location,
pub source_path: String,
}
impl<T> LocatedError<T> {
pub fn error(self) -> T {
self.error
}
pub fn from<U>(obj: LocatedError<U>) -> Self
where
U: Into<T>,
{
Self {
error: obj.error.into(),
location: obj.location,
source_path: obj.source_path,
}
}
pub fn into<U>(self) -> LocatedError<U>
where
T: Into<U>,
{
LocatedError::from(self)
}
}
impl<T> Display for LocatedError<T>
where
T: std::fmt::Display,
{
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
write!(
f,
"{} at row {} col {}",
&self.error, self.location.row, self.location.column,
)
}
}
impl<T> std::error::Error for LocatedError<T>
where
T: std::error::Error + 'static,
{
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
Some(&self.error)
}
}

9
core/src/lib.rs
View file

@ -8,8 +8,13 @@ pub mod marshal;
mod mode;
pub use bytecode::*;
pub use error::BaseError;
pub use location::Location;
pub use error::{BaseError, LocatedError};
pub use location::{Location, LocationRange};
pub use mode::Mode;
pub use ruff_text_size as text_size; // re-export mandatory and frequently accessed dependency
// FIXME: temp code
pub fn to_location(offset: &text_size::TextSize, source: &str) -> Location {
todo!()
}

7
core/src/location.rs
View file

@ -1,9 +1,6 @@
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
/// Source code location.
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
pub struct Location {
pub(super) row: u32,
pub(super) column: u32,
@ -96,6 +93,8 @@ impl Location {
}
}
pub type LocationRange = std::ops::Range<Location>;
#[cfg(test)]
mod tests {
use super::*;

3
parser/src/function.rs
View file

@ -114,8 +114,7 @@ pub(crate) fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentLis
}
keywords.push(ast::Keyword::new(
start,
end,
start..end,
ast::KeywordData { arg: name, value },
));
}

4
parser/src/lexer.rs
View file

@ -211,9 +211,9 @@ pub fn lex(source: &str, mode: Mode) -> impl Iterator<Item = LexResult> + '_ {
pub fn lex_located(
source: &str,
mode: Mode,
start_location: TextSize,
start_offset: TextSize,
) -> impl Iterator<Item = LexResult> + '_ {
SoftKeywordTransformer::new(Lexer::new(source.chars(), start_location), mode)
SoftKeywordTransformer::new(Lexer::new(source.chars(), start_offset), mode)
}
impl<T> Lexer<T>

20
parser/src/parser.rs
View file

@ -92,9 +92,9 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
pub fn parse_expression_located(
source: &str,
path: &str,
location: TextSize,
offset: TextSize,
) -> Result<ast::Expr, ParseError> {
parse_located(source, Mode::Expression, path, location).map(|top| match top {
parse_located(source, Mode::Expression, path, offset).map(|top| match top {
ast::Mod::Expression(ast::ModExpression { body }) => *body,
_ => unreachable!(),
})
@ -161,9 +161,9 @@ pub fn parse_located(
source: &str,
mode: Mode,
source_path: &str,
location: TextSize,
offset: TextSize,
) -> Result<ast::Mod, ParseError> {
let lxr = lexer::lex_located(source, mode, location);
let lxr = lexer::lex_located(source, mode, offset);
parse_tokens(lxr, mode, source_path)
}
@ -233,17 +233,17 @@ fn parse_error_from_lalrpop(
// TODO: Are there cases where this isn't an EOF?
LalrpopError::InvalidToken { location } => ParseError {
error: ParseErrorType::Eof,
location,
offset: location,
source_path,
},
LalrpopError::ExtraToken { token } => ParseError {
error: ParseErrorType::ExtraToken(token.1),
location: token.0,
offset: token.0,
source_path,
},
LalrpopError::User { error } => ParseError {
error: ParseErrorType::Lexical(error.error),
location: error.location,
offset: error.location,
source_path,
},
LalrpopError::UnrecognizedToken { token, expected } => {
@ -252,7 +252,7 @@ fn parse_error_from_lalrpop(
let expected = (expected.len() == 1).then(|| expected[0].clone());
ParseError {
error: ParseErrorType::UnrecognizedToken(token.1, expected),
location: token.0,
offset: token.0,
source_path,
}
}
@ -262,13 +262,13 @@ fn parse_error_from_lalrpop(
if indent_error {
ParseError {
error: ParseErrorType::Lexical(LexicalErrorType::IndentationError),
location,
offset: location,
source_path,
}
} else {
ParseError {
error: ParseErrorType::Eof,
location,
offset: location,
source_path,
}
}

209
parser/src/python.lalrpop
View file

@ -68,7 +68,7 @@ SmallStatement: ast::Stmt = {
PassStatement: ast::Stmt = {
<location:@L> "pass" <end_location:@R> => {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtKind::Pass,
)
@ -78,7 +78,7 @@ PassStatement: ast::Stmt = {
DelStatement: ast::Stmt = {
<location:@L> "del" <targets:ExpressionList2> <end_location:@R> => {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtDelete { targets: targets.into_iter().map(|expr| set_context(expr, ast::ExprContext::Del)).collect() }.into()
)
@ -90,7 +90,7 @@ ExpressionStatement: ast::Stmt = {
// Just an expression, no assignment:
if suffix.is_empty() {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtExpr { value: Box::new(expression) }.into()
)
@ -105,7 +105,7 @@ ExpressionStatement: ast::Stmt = {
let value = Box::new(values.into_iter().next().unwrap());
ast::Stmt::new(
location,
location..
end_location,
ast::StmtAssign { targets, value, type_comment: None }.into()
)
@ -113,7 +113,7 @@ ExpressionStatement: ast::Stmt = {
},
<location:@L> <target:TestOrStarExprList> <op:AugAssign> <rhs:TestListOrYieldExpr> <end_location:@R> => {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtAugAssign {
target: Box::new(set_context(target, ast::ExprContext::Store)),
@ -125,7 +125,7 @@ ExpressionStatement: ast::Stmt = {
<location:@L> <target:Test<"all">> ":" <annotation:Test<"all">> <rhs:AssignSuffix?> <end_location:@R> => {
let simple = matches!(target.node, ast::ExprKind::Name { .. });
ast::Stmt::new(
location,
location..
end_location,
ast::StmtAnnAssign {
target: Box::new(set_context(target, ast::ExprContext::Store)),
@ -186,28 +186,28 @@ AugAssign: ast::Operator = {
FlowStatement: ast::Stmt = {
<location:@L> "break" <end_location:@R> => {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtKind::Break,
)
},
<location:@L> "continue" <end_location:@R> => {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtKind::Continue,
)
},
<location:@L> "return" <value:TestList?> <end_location:@R> => {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtReturn { value: value.map(Box::new) }.into()
)
},
<location:@L> <expression:YieldExpr> <end_location:@R> => {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtExpr { value: Box::new(expression) }.into()
)
@ -218,14 +218,14 @@ FlowStatement: ast::Stmt = {
RaiseStatement: ast::Stmt = {
<location:@L> "raise" <end_location:@R> => {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtRaise { exc: None, cause: None }.into()
)
},
<location:@L> "raise" <t:Test<"all">> <c:("from" Test<"all">)?> <end_location:@R> => {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtRaise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)) }.into()
)
@ -235,7 +235,7 @@ RaiseStatement: ast::Stmt = {
ImportStatement: ast::Stmt = {
<location:@L> "import" <names: OneOrMore<ImportAsAlias<DottedName>>> <end_location:@R> => {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtImport { names }.into()
)
@ -243,7 +243,7 @@ ImportStatement: ast::Stmt = {
<location:@L> "from" <source:ImportFromLocation> "import" <names: ImportAsNames> <end_location:@R> => {
let (level, module) = source;
ast::Stmt::new(
location,
location..
end_location,
ast::StmtImportFrom {
level,
@ -273,14 +273,14 @@ ImportAsNames: Vec<ast::Alias> = {
<location:@L> "(" <i:OneOrMore<ImportAsAlias<Identifier>>> ","? ")" <end_location:@R> => i,
<location:@L> "*" <end_location:@R> => {
// Star import all
vec![ast::Alias::new(location, end_location, ast::AliasData { name: "*".to_string(), asname: None })]
vec![ast::Alias::new(location..end_location, ast::AliasData { name: "*".to_string(), asname: None })]
},
};
#[inline]
ImportAsAlias<I>: ast::Alias = {
<location:@L> <name:I> <a: ("as" Identifier)?> <end_location:@R> => ast::Alias::new(location, end_location, ast::AliasData { name, asname: a.map(|a| a.1) }),
<location:@L> <name:I> <a: ("as" Identifier)?> <end_location:@R> => ast::Alias::new(location..end_location, ast::AliasData { name, asname: a.map(|a| a.1) }),
}
// A name like abc or abc.def.ghi
@ -299,7 +299,7 @@ DottedName: String = {
GlobalStatement: ast::Stmt = {
<location:@L> "global" <names:OneOrMore<Identifier>> <end_location:@R> => {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtGlobal { names }.into()
)
@ -309,7 +309,7 @@ GlobalStatement: ast::Stmt = {
NonlocalStatement: ast::Stmt = {
<location:@L> "nonlocal" <names:OneOrMore<Identifier>> <end_location:@R> => {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtNonlocal { names }.into()
)
@ -319,7 +319,7 @@ NonlocalStatement: ast::Stmt = {
AssertStatement: ast::Stmt = {
<location:@L> "assert" <test:Test<"all">> <msg: ("," Test<"all">)?> <end_location:@R> => {
ast::Stmt::new(
location,
location..
end_location,
ast::StmtAssert {
test: Box::new(test),
@ -350,7 +350,7 @@ MatchStatement: ast::Stmt = {
.unwrap()
.end();
ast::Stmt::new(
location,
location..
end_location,
ast::StmtMatch {
subject: Box::new(subject),
@ -367,7 +367,7 @@ MatchStatement: ast::Stmt = {
.unwrap()
.end();
ast::Stmt::new(
location,
location..
end_location,
ast::StmtMatch {
subject: Box::new(subject),
@ -386,11 +386,11 @@ MatchStatement: ast::Stmt = {
let mut subjects = subjects;
subjects.insert(0, subject);
ast::Stmt::new(
location,
location..
end_location,
ast::StmtMatch {
subject: Box::new(ast::Expr::new(
location,
location..
end_location,
ast::ExprTuple {
elts: subjects,
@ -421,7 +421,7 @@ Guard: ast::Expr = {
Patterns: ast::Pattern = {
<location:@L> <pattern:Pattern> "," <end_location:@R> => ast::Pattern::new(
location,
location..
end_location,
ast::PatternMatchSequence {
patterns: vec![pattern]
@ -431,7 +431,7 @@ Patterns: ast::Pattern = {
let mut patterns = patterns;
patterns.insert(0, pattern);
ast::Pattern::new(
location,
location..
end_location,
ast::PatternMatchSequence {
patterns
@ -455,7 +455,7 @@ AsPattern: ast::Pattern = {
})?
} else {
Ok(ast::Pattern::new(
location,
location..
end_location,
ast::PatternMatchAs {
pattern: Some(Box::new(pattern)),
@ -472,7 +472,7 @@ OrPattern: ast::Pattern = {
let mut patterns = patterns;
patterns.insert(0, pattern);
ast::Pattern::new(
location,
location..
end_location,
ast::PatternMatchOr { patterns }.into()
)
@ -481,37 +481,37 @@ OrPattern: ast::Pattern = {
ClosedPattern: ast::Pattern = {
<location:@L> <node:LiteralPattern> <end_location:@R> => ast::Pattern::new(
location,
location..
end_location,
node,
),
<location:@L> <node:CapturePattern> <end_location:@R> => ast::Pattern::new(
location,
location..
end_location,
node,
),
<location:@L> <node:StarPattern> <end_location:@R> => ast::Pattern::new(
location,
location..
end_location,
node,
),
<location:@L> <node:ValuePattern> <end_location:@R> => ast::Pattern::new(
location,
location..
end_location,
node,
),
<location:@L> <node:SequencePattern> <end_location:@R> => ast::Pattern::new(
location,
location..
end_location,
node,
),
<location:@L> <node:MappingPattern> <end_location:@R> => ast::Pattern::new(
location,
location..
end_location,
node,
),
<location:@L> <node:ClassPattern> <end_location:@R> => ast::Pattern::new(
location,
location..
end_location,
node,
),
@ -543,7 +543,7 @@ StarPattern: ast::PatternKind = {
ConstantAtom: ast::Expr = {
<location:@L> <value:Constant> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprConstant { value, kind: None }.into()
),
@ -552,7 +552,7 @@ ConstantAtom: ast::Expr = {
ConstantExpr: ast::Expr = {
ConstantAtom,
<location:@L> "-" <operand:ConstantAtom> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprUnaryOp {
op: ast::Unaryop::USub,
@ -563,7 +563,7 @@ ConstantExpr: ast::Expr = {
AddOpExpr: ast::Expr = {
<location:@L> <left:ConstantExpr> <op:AddOp> <right:ConstantAtom> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprBinOp {
left: Box::new(left),
@ -603,7 +603,7 @@ CapturePattern: ast::PatternKind = {
MatchName: ast::Expr = {
<location:@L> <name:Identifier> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprName { id: name, ctx: ast::ExprContext::Load }.into(),
),
@ -611,7 +611,7 @@ MatchName: ast::Expr = {
MatchNameOrAttr: ast::Expr = {
<location:@L> <name:MatchName> "." <attr:Identifier> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprAttribute {
value: Box::new(name),
@ -620,7 +620,7 @@ MatchNameOrAttr: ast::Expr = {
}.into(),
),
<location:@L> <e:MatchNameOrAttr> "." <attr:Identifier> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprAttribute {
value: Box::new(e),
@ -641,7 +641,7 @@ MappingKey: ast::Expr = {
AddOpExpr,
MatchNameOrAttr,
<location:@L> "None" <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprConstant {
value: ast::Constant::None,
@ -649,7 +649,7 @@ MappingKey: ast::Expr = {
}.into(),
),
<location:@L> "True" <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprConstant {
value: true.into(),
@ -657,7 +657,7 @@ MappingKey: ast::Expr = {
}.into(),
),
<location:@L> "False" <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprConstant {
value: false.into(),
@ -804,7 +804,7 @@ IfStatement: ast::Stmt = {
// handle elif:
for i in s2.into_iter().rev() {
let x = ast::Stmt::new(
i.0,
i.0..
end_location,
ast::StmtIf { test: Box::new(i.2), body: i.4, orelse: last }.into()
);
@ -812,7 +812,7 @@ IfStatement: ast::Stmt = {
}
ast::Stmt::new(
location,
location..
end_location,
ast::StmtIf { test: Box::new(test), body, orelse: last }.into()
)
@ -828,7 +828,7 @@ WhileStatement: ast::Stmt = {
.unwrap()
.end();
ast::Stmt::new(
location,
location..
end_location,
ast::StmtWhile {
test: Box::new(test),
@ -855,7 +855,7 @@ ForStatement: ast::Stmt = {
} else {
ast::StmtFor { target, iter, body, orelse, type_comment }.into()
};
ast::Stmt::new(location, end_location, node)
ast::Stmt::new(location..end_location, node)
},
};
@ -870,7 +870,7 @@ TryStatement: ast::Stmt = {
.or_else(|| handlers.last().map(|last| last.end()))
.unwrap();
ast::Stmt::new(
location,
location..
end_location,
ast::StmtTry {
body,
@ -890,7 +890,7 @@ TryStatement: ast::Stmt = {
.or_else(|| handlers.last().map(|last| last.end()))
.unwrap();
ast::Stmt::new(
location,
location..
end_location,
ast::StmtTryStar {
body,
@ -906,7 +906,7 @@ TryStatement: ast::Stmt = {
let finalbody = finally.2;
let end_location = finalbody.last().unwrap().end();
ast::Stmt::new(
location,
location..
end_location,
ast::StmtTry {
body,
@ -922,7 +922,7 @@ ExceptStarClause: ast::Excepthandler = {
<location:@L> "except" "*" <typ:Test<"all">> ":" <body:Suite> => {
let end_location = body.last().unwrap().end();
ast::Excepthandler::new(
location,
location..
end_location,
ast::ExcepthandlerExceptHandler {
type_: Some(Box::new(typ)),
@ -934,7 +934,7 @@ ExceptStarClause: ast::Excepthandler = {
<location:@L> "except" "*" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
let end_location = body.last().unwrap().end();
ast::Excepthandler::new(
location,
location..
end_location,
ast::ExcepthandlerExceptHandler {
type_: Some(Box::new(x.0)),
@ -950,7 +950,7 @@ ExceptClause: ast::Excepthandler = {
<location:@L> "except" <typ:Test<"all">?> ":" <body:Suite> => {
let end_location = body.last().unwrap().end();
ast::Excepthandler::new(
location,
location..
end_location,
ast::ExcepthandlerExceptHandler {
type_: typ.map(Box::new),
@ -962,7 +962,7 @@ ExceptClause: ast::Excepthandler = {
<location:@L> "except" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
let end_location = body.last().unwrap().end();
ast::Excepthandler::new(
location,
location..
end_location,
ast::ExcepthandlerExceptHandler {
type_: Some(Box::new(x.0)),
@ -982,7 +982,7 @@ WithStatement: ast::Stmt = {
} else {
ast::StmtWith { items, body, type_comment }.into()
};
ast::Stmt::new(location, end_location, node)
ast::Stmt::new(location..end_location, node)
},
};
@ -1023,7 +1023,7 @@ FuncDef: ast::Stmt = {
} else {
ast::StmtFunctionDef { name, args, body, decorator_list, returns, type_comment }.into()
};
ast::Stmt::new(location, end_location, node)
ast::Stmt::new(location..end_location, node)
},
};
@ -1126,7 +1126,7 @@ ParameterDef<ArgType>: (ast::Arg, Option<ast::Expr>) = {
UntypedParameter: ast::Arg = {
<location:@L> <arg:Identifier> <end_location:@R> => ast::Arg::new(
location,
location..
end_location,
ast::ArgData { arg, annotation: None, type_comment: None },
),
@ -1135,14 +1135,14 @@ UntypedParameter: ast::Arg = {
TypedParameter: ast::Arg = {
<location:@L> <arg:Identifier> <a:(":" Test<"all">)?> <end_location:@R> => {
let annotation = a.map(|x| Box::new(x.1));
ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None })
ast::Arg::new(location..end_location, ast::ArgData { arg, annotation, type_comment: None })
},
};
StarTypedParameter: ast::Arg = {
<location:@L> <arg:Identifier> <a:(":" TestOrStarExpr)?> <end_location:@R> => {
let annotation = a.map(|x| Box::new(x.1));
ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None })
ast::Arg::new(location..end_location, ast::ArgData { arg, annotation, type_comment: None })
},
};
@ -1193,7 +1193,7 @@ ClassDef: ast::Stmt = {
};
let end_location = body.last().unwrap().end();
ast::Stmt::new(
location,
location..
end_location,
ast::StmtClassDef {
name,
@ -1215,12 +1215,12 @@ Decorator: ast::Expr = {
YieldExpr: ast::Expr = {
<location:@L> "yield" <value:TestList?> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprYield { value: value.map(Box::new) }.into()
),
<location:@L> "yield" "from" <e:Test<"all">> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprYieldFrom { value: Box::new(e) }.into()
),
@ -1228,7 +1228,7 @@ YieldExpr: ast::Expr = {
Test<Goal>: ast::Expr = {
<location:@L> <body:OrTest<"all">> "if" <test:OrTest<"all">> "else" <orelse:Test<"all">> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprIfExp {
test: Box::new(test),
@ -1248,11 +1248,11 @@ NamedExpressionTest: ast::Expr = {
NamedExpression: ast::Expr = {
<location:@L> <id:Identifier> <end_location:@R> ":=" <value:Test<"all">> => {
ast::Expr::new(
location,
location..
value.end(),
ast::ExprNamedExpr {
target: Box::new(ast::Expr::new(
location,
location..
end_location,
ast::ExprName { id, ctx: ast::ExprContext::Store }.into(),
)),
@ -1279,7 +1279,7 @@ LambdaDef: ast::Expr = {
))?;
Ok(ast::Expr::new(
location,
location..
end_location,
ast::ExprLambda {
args: Box::new(p),
@ -1294,7 +1294,7 @@ OrTest<Goal>: ast::Expr = {
let mut values = vec![e1];
values.extend(e2.into_iter().map(|e| e.1));
ast::Expr::new(
location,
location..
end_location,
ast::ExprBoolOp { op: ast::Boolop::Or, values }.into()
)
@ -1307,7 +1307,7 @@ AndTest<Goal>: ast::Expr = {
let mut values = vec![e1];
values.extend(e2.into_iter().map(|e| e.1));
ast::Expr::new(
location,
location..
end_location,
ast::ExprBoolOp { op: ast::Boolop::And, values }.into()
)
@ -1317,7 +1317,7 @@ AndTest<Goal>: ast::Expr = {
NotTest<Goal>: ast::Expr = {
<location:@L> "not" <e:NotTest<"all">> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprUnaryOp { operand: Box::new(e), op: ast::Unaryop::Not }.into()
),
@ -1328,7 +1328,7 @@ Comparison<Goal>: ast::Expr = {
<location:@L> <left:Expression<"all">> <comparisons:(CompOp Expression<"all">)+> <end_location:@R> => {
let (ops, comparators) = comparisons.into_iter().unzip();
ast::Expr::new(
location,
location..
end_location,
ast::ExprCompare { left: Box::new(left), ops, comparators }.into()
)
@ -1351,7 +1351,7 @@ CompOp: ast::Cmpop = {
Expression<Goal>: ast::Expr = {
<location:@L> <e1:Expression<"all">> "|" <e2:XorExpression<"all">> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprBinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) }.into()
),
@ -1360,7 +1360,7 @@ Expression<Goal>: ast::Expr = {
XorExpression<Goal>: ast::Expr = {
<location:@L> <e1:XorExpression<"all">> "^" <e2:AndExpression<"all">> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprBinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) }.into()
),
@ -1369,7 +1369,7 @@ XorExpression<Goal>: ast::Expr = {
AndExpression<Goal>: ast::Expr = {
<location:@L> <e1:AndExpression<"all">> "&" <e2:ShiftExpression<"all">> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprBinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) }.into()
),
@ -1378,7 +1378,7 @@ AndExpression<Goal>: ast::Expr = {
ShiftExpression<Goal>: ast::Expr = {
<location:@L> <e1:ShiftExpression<"all">> <op:ShiftOp> <e2:ArithmeticExpression<"all">> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprBinOp { left: Box::new(e1), op, right: Box::new(e2) }.into()
),
@ -1392,7 +1392,7 @@ ShiftOp: ast::Operator = {
ArithmeticExpression<Goal>: ast::Expr = {
<location:@L> <a:ArithmeticExpression<"all">> <op:AddOp> <b:Term<"all">> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprBinOp { left: Box::new(a), op, right: Box::new(b) }.into()
),
@ -1406,7 +1406,7 @@ AddOp: ast::Operator = {
Term<Goal>: ast::Expr = {
<location:@L> <a:Term<"all">> <op:MulOp> <b:Factor<"all">> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprBinOp { left: Box::new(a), op, right: Box::new(b) }.into()
),
@ -1423,7 +1423,7 @@ MulOp: ast::Operator = {
Factor<Goal>: ast::Expr = {
<location:@L> <op:UnaryOp> <e:Factor<"all">> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprUnaryOp { operand: Box::new(e), op }.into()
),
@ -1438,7 +1438,7 @@ UnaryOp: ast::Unaryop = {
Power<Goal>: ast::Expr = {
<location:@L> <e:AtomExpr<"all">> "**" <b:Factor<"all">> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprBinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) }.into()
),
@ -1448,7 +1448,7 @@ Power<Goal>: ast::Expr = {
AtomExpr<Goal>: ast::Expr = {
<location:@L> "await" <atom:AtomExpr2<"all">> <end_location:@R> => {
ast::Expr::new(
location,
location..
end_location,
ast::ExprAwait { value: Box::new(atom) }.into()
)
@ -1460,18 +1460,18 @@ AtomExpr2<Goal>: ast::Expr = {
Atom<Goal>,
<location:@L> <f:AtomExpr2<"all">> "(" <a:ArgumentList> ")" <end_location:@R> => {
ast::Expr::new(
location,
location..
end_location,
ast::ExprCall { func: Box::new(f), args: a.args, keywords: a.keywords }.into()
)
},
<location:@L> <e:AtomExpr2<"all">> "[" <s:SubscriptList> "]" <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprSubscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load }.into()
),
<location:@L> <e:AtomExpr2<"all">> "." <attr:Identifier> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprAttribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load }.into()
),
@ -1488,7 +1488,7 @@ SubscriptList: ast::Expr = {
}
ast::Expr::new(
location,
location..
end_location,
ast::ExprTuple { elts: dims, ctx: ast::ExprContext::Load }.into(),
)
@ -1503,7 +1503,7 @@ Subscript: ast::Expr = {
let upper = e2.map(Box::new);
let step = e3.flatten().map(Box::new);
ast::Expr::new(
location,
location..
end_location,
ast::ExprSlice { lower, upper, step }.into()
)
@ -1517,26 +1517,26 @@ SliceOp: Option<ast::Expr> = {
Atom<Goal>: ast::Expr = {
<location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?),
<location:@L> <value:Constant> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprConstant { value, kind: None }.into()
),
<location:@L> <name:Identifier> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprName { id: name, ctx: ast::ExprContext::Load }.into()
),
<location:@L> "[" <e:ListLiteralValues?> "]"<end_location:@R> => {
let elts = e.unwrap_or_default();
ast::Expr::new(
location,
location..
end_location,
ast::ExprList { elts, ctx: ast::ExprContext::Load }.into()
)
},
<location:@L> "[" <elt:TestOrStarNamedExpr> <generators:CompFor> "]" <end_location:@R> => {
ast::Expr::new(
location,
location..
end_location,
ast::ExprListComp { elt: Box::new(elt), generators }.into()
)
@ -1546,7 +1546,7 @@ Atom<Goal>: ast::Expr = {
elts.into_iter().next().unwrap()
} else {
ast::Expr::new(
location,
location..
end_location,
ast::ExprTuple { elts, ctx: ast::ExprContext::Load }.into()
)
@ -1564,21 +1564,21 @@ Atom<Goal>: ast::Expr = {
} else {
let elts = left.into_iter().flatten().chain([mid]).chain(right).collect();
Ok(ast::Expr::new(
location,
location..
end_location,
ast::ExprTuple { elts, ctx: ast::ExprContext::Load }.into(),
))
}
},
<location:@L> "(" ")" <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprTuple { elts: Vec::new(), ctx: ast::ExprContext::Load }.into()
),
"(" <e:YieldExpr> ")" => e,
<location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" <end_location:@R> => {
ast::Expr::new(
location,
location..
end_location,
ast::ExprGeneratorExp { elt: Box::new(elt), generators }.into()
)
@ -1596,14 +1596,14 @@ Atom<Goal>: ast::Expr = {
.map(|(k, v)| (k.map(|x| *x), v))
.unzip();
ast::Expr::new(
location,
location..
end_location,
ast::ExprDict { keys, values }.into()
)
},
<location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" <end_location:@R> => {
ast::Expr::new(
location,
location..
end_location,
ast::ExprDictComp {
key: Box::new(e1.0),
@ -1613,21 +1613,21 @@ Atom<Goal>: ast::Expr = {
)
},
<location:@L> "{" <elts:SetLiteralValues> "}" <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprSet { elts }.into()
),
<location:@L> "{" <elt:NamedExpressionTest> <generators:CompFor> "}" <end_location:@R> => {
ast::Expr::new(
location,
location..
end_location,
ast::ExprSetComp { elt: Box::new(elt), generators }.into()
)
},
<location:@L> "True" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprConstant { value: true.into(), kind: None }.into()),
<location:@L> "False" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprConstant { value: false.into(), kind: None }.into()),
<location:@L> "None" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprConstant { value: ast::Constant::None, kind: None }.into()),
<location:@L> "..." <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprConstant { value: ast::Constant::Ellipsis, kind: None }.into()),
<location:@L> "True" <end_location:@R> => ast::Expr::new(location..end_location, ast::ExprConstant { value: true.into(), kind: None }.into()),
<location:@L> "False" <end_location:@R> => ast::Expr::new(location..end_location, ast::ExprConstant { value: false.into(), kind: None }.into()),
<location:@L> "None" <end_location:@R> => ast::Expr::new(location..end_location, ast::ExprConstant { value: ast::Constant::None, kind: None }.into()),
<location:@L> "..." <end_location:@R> => ast::Expr::new(location..end_location, ast::ExprConstant { value: ast::Constant::Ellipsis, kind: None }.into()),
};
ListLiteralValues: Vec<ast::Expr> = {
@ -1679,7 +1679,7 @@ GenericList<Element>: ast::Expr = {
elts.into_iter().next().unwrap()
} else {
ast::Expr::new(
location,
location..
end_location,
ast::ExprTuple { elts, ctx: ast::ExprContext::Load }.into()
)
@ -1690,7 +1690,7 @@ GenericList<Element>: ast::Expr = {
// Test
StarExpr: ast::Expr = {
<location:@L> "*" <e:Expression<"all">> <end_location:@R> => ast::Expr::new(
location,
location..
end_location,
ast::ExprStarred { value: Box::new(e), ctx: ast::ExprContext::Load }.into(),
)
@ -1725,8 +1725,7 @@ FunctionArgument: (Option<(crate::text_size::TextSize, crate::text_size::TextSiz
<location:@L> <e:NamedExpressionTest> <c:CompFor?> <end_location:@R> => {
let expr = match c {
Some(c) => ast::Expr::new(
location,
end_location,
location..end_location,
ast::ExprGeneratorExp {
elt: Box::new(e),
generators: c,
@ -1739,7 +1738,7 @@ FunctionArgument: (Option<(crate::text_size::TextSize, crate::text_size::TextSiz
<location:@L> <i:Identifier> "=" <e:Test<"all">> <end_location:@R> => (Some((location, end_location, Some(i))), e),
<location:@L> "*" <e:Test<"all">> <end_location:@R> => {
let expr = ast::Expr::new(
location,
location..
end_location,
ast::ExprStarred { value: Box::new(e), ctx: ast::ExprContext::Load }.into(),
);

279
parser/src/python.rs generated
View file

File diff suppressed because it is too large Load diff

18
parser/src/string.rs
View file

@ -10,9 +10,7 @@ use crate::{
token::{StringKind, Tok},
};
use itertools::Itertools;
use rustpython_compiler_core::{
text_size::{TextLen, TextSize},
};
use rustpython_compiler_core::text_size::{TextLen, TextSize};
// unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
const MAX_UNICODE_NAME: usize = 88;
@ -67,7 +65,7 @@ impl<'a> StringParser<'a> {
#[inline]
fn expr(&self, node: ExprKind) -> Expr {
Expr::new(self.start, self.end, node)
Expr::new(self.start..self.end, node)
}
fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
@ -624,8 +622,7 @@ pub(crate) fn parse_strings(
}
}
return Ok(Expr::new(
initial_start,
last_end,
initial_start..last_end,
ast::ExprConstant {
value: Constant::Bytes(content),
kind: None,
@ -648,8 +645,7 @@ pub(crate) fn parse_strings(
}
}
return Ok(Expr::new(
initial_start,
last_end,
initial_start..last_end,
ast::ExprConstant {
value: Constant::Str(content.join("")),
kind: initial_kind,
@ -664,8 +660,7 @@ pub(crate) fn parse_strings(
let take_current = |current: &mut Vec<String>| -> Expr {
Expr::new(
initial_start,
last_end,
initial_start..last_end,
ast::ExprConstant {
value: Constant::Str(current.drain(..).join("")),
kind: initial_kind.clone(),
@ -696,8 +691,7 @@ pub(crate) fn parse_strings(
}
Ok(Expr::new(
initial_start,
last_end,
initial_start..last_end,
ast::ExprJoinedStr { values: deduped }.into(),
))
}

7
ruff_text_size/src/range.rs
View file

@ -463,6 +463,13 @@ impl RangeBounds<TextSize> for TextRange {
}
}
impl From<Range<TextSize>> for TextRange {
#[inline]
fn from(r: Range<TextSize>) -> Self {
TextRange::new(r.start, r.end)
}
}
impl<T> From<TextRange> for Range<T>
where
T: From<TextSize>,

28
ruff_text_size/src/size.rs
View file

@ -63,6 +63,30 @@ impl TextSize {
pub fn of<T: TextLen>(text: T) -> TextSize {
text.text_len()
}
/// Returns current raw `offset` as u32.
///
/// # Examples
///
/// ```rust
/// # use ruff_text_size::*;
/// assert_eq!(TextSize::from(4).to_u32(), 4);
/// ```
pub fn to_u32(&self) -> u32 {
self.raw
}
/// Returns current raw `offset` as usize.
///
/// # Examples
///
/// ```rust
/// # use ruff_text_size::*;
/// assert_eq!(TextSize::from(4).to_usize(), 4);
/// ```
pub fn to_usize(&self) -> usize {
self.raw as usize
}
}
/// Methods to act like a primitive integer type, where reasonably applicable.
@ -91,7 +115,7 @@ impl From<u32> for TextSize {
impl From<TextSize> for u32 {
#[inline]
fn from(value: TextSize) -> Self {
value.raw
value.to_u32()
}
}
@ -106,7 +130,7 @@ impl TryFrom<usize> for TextSize {
impl From<TextSize> for usize {
#[inline]
fn from(value: TextSize) -> Self {
value.raw as usize
value.to_usize()
}
}

7
scripts/update_asdl.sh Executable file
View file

@ -0,0 +1,7 @@
#!/bin/bash
set -e
cd "$(dirname "$(dirname "$0")")"
python ast/asdl_rs.py --generic-file ast/src/generic.rs --located-file ast/src/located.rs --module-file ../RustPython/vm/src/stdlib/ast/gen.rs ast/Python.asdl
rustfmt ast/src/generic.rs ast/src/located.rs ../RustPython/vm/src/stdlib/ast/gen.rs