Separate byteoffset ast and located ast

ast/Cargo.toml

@@ -8,8 +8,9 @@ repository = "https://github.com/RustPython/RustPython"
 license = "MIT"
 
 [features]
-default = ["constant-optimization", "fold"]
+default = ["constant-optimization", "fold", "location"]
 constant-optimization = ["fold"]
+location = []
 fold = []
 unparse = ["rustpython-literal"]
 

ast/asdl_rs.py

@@ -7,6 +7,8 @@ import textwrap
 
 from argparse import ArgumentParser
 from pathlib import Path
+from typing import Optional, Dict
+from attr import dataclass
 
 import asdl
 
@@ -62,38 +64,62 @@ def asdl_of(name, obj):
         return "{} = {}".format(name, types)
 
 
-class EmitVisitor(asdl.VisitorBase):
-    """Visit that emits lines"""
-
-    def __init__(self, file):
-        self.file = file
-        self.identifiers = set()
-        super(EmitVisitor, self).__init__()
-
-    def emit_identifier(self, name):
-        name = str(name)
-        if name in self.identifiers:
-            return
-        self.emit("_Py_IDENTIFIER(%s);" % name, 0)
-        self.identifiers.add(name)
-
-    def emit(self, line, depth):
-        if line:
-            line = (" " * TABSIZE * depth) + line
-        self.file.write(line + "\n")
-
-
 class TypeInfo:
+    name: str
+    enum_name: Optional[str]
+    has_userdata: Optional[bool]
+    has_attributes: bool
+    children: set
+    boxed: bool
+    product: bool
+    has_expr: bool = False
+
     def __init__(self, name):
         self.name = name
+        self.enum_name = None
         self.has_userdata = None
+        self.has_attributes = False
         self.children = set()
         self.boxed = False
         self.product = False
+        self.product_has_expr = False
 
     def __repr__(self):
         return f"<TypeInfo: {self.name}>"
 
+    @property
+    def rust_name(self):
+        return get_rust_type(self.name)
+
+    @property
+    def sum_name(self):
+        if self.enum_name is None:
+            return self.name
+        else:
+            return f"{self.enum_name}_{self.name}"
+
+    @property
+    def rust_sum_name(self):
+        rust_name = get_rust_type(self.name)
+        if self.enum_name is None:
+            return rust_name
+        else:
+            name = get_rust_type(self.enum_name) + rust_name
+            return name
+
+    @property
+    def rust_suffix(self):
+        if self.product:
+            if self.has_attributes:
+                return "Data"
+            else:
+                return ""
+        else:
+            if self.has_attributes:
+                return "Kind"
+            else:
+                return ""
+
     def determine_userdata(self, typeinfo, stack):
         if self.name in stack:
             return None
@@ -110,6 +136,41 @@ class TypeInfo:
         return self.has_userdata
 
 
+class TypeInfoMixin:
+    typeinfo: Dict[str, TypeInfo]
+
+    def has_userdata(self, typ):
+        return self.typeinfo[typ].has_userdata
+
+    def get_generics(self, typ, *generics):
+        if self.has_userdata(typ):
+            return [f"<{g}>" for g in generics]
+        else:
+            return ["" for g in generics]
+
+
+class EmitVisitor(asdl.VisitorBase, TypeInfoMixin):
+    """Visit that emits lines"""
+
+    def __init__(self, file, typeinfo):
+        self.file = file
+        self.typeinfo = typeinfo
+        self.identifiers = set()
+        super(EmitVisitor, self).__init__()
+
+    def emit_identifier(self, name):
+        name = str(name)
+        if name in self.identifiers:
+            return
+        self.emit("_Py_IDENTIFIER(%s);" % name, 0)
+        self.identifiers.add(name)
+
+    def emit(self, line, depth):
+        if line:
+            line = (" " * TABSIZE * depth) + line
+        self.file.write(line + "\n")
+
+
 class FindUserdataTypesVisitor(asdl.VisitorBase):
     def __init__(self, typeinfo):
         self.typeinfo = typeinfo
@@ -132,21 +193,29 @@ class FindUserdataTypesVisitor(asdl.VisitorBase):
             info.has_userdata = False
         else:
             for t in sum.types:
-                self.typeinfo[t.name] = TypeInfo(t.name)
+                if not t.fields:
+                    continue
+                t_info = TypeInfo(t.name)
+                t_info.enum_name = name
+                self.typeinfo[t.name] = t_info
                 self.add_children(t.name, t.fields)
             if len(sum.types) > 1:
                 info.boxed = True
             if sum.attributes:
-                # attributes means Located, which has the `custom: U` field
+                # attributes means located, which has the `custom: U` field
                 info.has_userdata = True
+                info.has_attributes = True
+
         for variant in sum.types:
             self.add_children(name, variant.fields)
 
     def visitProduct(self, product, name):
         info = self.typeinfo[name]
         if product.attributes:
-            # attributes means Located, which has the `custom: U` field
+            # attributes means located, which has the `custom: U` field
             info.has_userdata = True
+            info.has_attributes = True
+        info.has_expr = product_has_expr(product)
         if len(product.fields) > 2:
             info.boxed = True
         info.product = True
@@ -163,24 +232,17 @@ def rust_field(field_name):
         return field_name
 
 
-class TypeInfoEmitVisitor(EmitVisitor):
+def product_has_expr(product):
-    def __init__(self, file, typeinfo):
+    return any(f.type != "identifier" for f in product.fields)
-        self.typeinfo = typeinfo
-        super().__init__(file)
-
-    def has_userdata(self, typ):
-        return self.typeinfo[typ].has_userdata
-
-    def get_generics(self, typ, *generics):
-        if self.has_userdata(typ):
-            return [f"<{g}>" for g in generics]
-        else:
-            return ["" for g in generics]
 
 
-class StructVisitor(TypeInfoEmitVisitor):
+class StructVisitor(EmitVisitor):
     """Visitor to generate typedefs for AST."""
 
+    def __init__(self, *args, **kw):
+        super().__init__(*args, **kw)
+        self.rust_type_defs = []
+
     def visitModule(self, mod):
         for dfn in mod.dfns:
             self.visit(dfn)
@@ -208,57 +270,56 @@ class StructVisitor(TypeInfoEmitVisitor):
 
     def sum_with_constructors(self, sum, name, depth):
         typeinfo = self.typeinfo[name]
-        enumname = rustname = get_rust_type(name)
+        suffix = typeinfo.rust_suffix
+        rustname = get_rust_type(name)
         # all the attributes right now are for location, so if it has attrs we
-        # can just wrap it in Located<>
+        # can just wrap it in Attributed<>
-        if sum.attributes:
-            enumname = rustname + "Kind"
 
         for t in sum.types:
             if not t.fields:
                 continue
-            self.emit_attrs(depth)
+            self.sum_subtype_struct(typeinfo, t, rustname, depth)
-            self.typeinfo[t] = TypeInfo(t)
-            t_generics, t_generics_applied = self.get_generics(t.name, "U = ()", "U")
-            payload_name = f"{rustname}{t.name}"
-            self.emit(f"pub struct {payload_name}{t_generics} {{", depth)
-            for f in t.fields:
-                self.visit(f, typeinfo, "pub ", depth + 1, t.name)
-            self.emit("}", depth)
-            self.emit(
-                textwrap.dedent(
-                    f"""
-            impl{t_generics_applied} From<{payload_name}{t_generics_applied}> for {enumname}{t_generics_applied} {{
-                fn from(payload: {payload_name}{t_generics_applied}) -> Self {{
-                    {enumname}::{t.name}(payload)
-                }}
-            }}
-            """
-                ),
-                depth,
-            )
 
         generics, generics_applied = self.get_generics(name, "U = ()", "U")
         self.emit_attrs(depth)
-        self.emit(f"pub enum {enumname}{generics} {{", depth)
+        self.emit(f"pub enum {rustname}{suffix}{generics} {{", depth)
         for t in sum.types:
             if t.fields:
-                t_generics, t_generics_applied = self.get_generics(
+                (t_generics_applied,) = self.get_generics(t.name, "U")
-                    t.name, "U = ()", "U"
-                )
                 self.emit(
                     f"{t.name}({rustname}{t.name}{t_generics_applied}),", depth + 1
                 )
             else:
                 self.emit(f"{t.name},", depth + 1)
         self.emit("}", depth)
-        if sum.attributes:
+        if typeinfo.has_attributes:
             self.emit(
-                f"pub type {rustname}<U = ()> = Located<{enumname}{generics_applied}, U>;",
+                f"pub type {rustname}<U = ()> = Attributed<{rustname}{suffix}{generics_applied}, U>;",
                 depth,
             )
         self.emit("", depth)
 
+    def sum_subtype_struct(self, sum_typeinfo, t, rustname, depth):
+        self.emit_attrs(depth)
+        generics, generics_applied = self.get_generics(t.name, "U = ()", "U")
+        payload_name = f"{rustname}{t.name}"
+        self.emit(f"pub struct {payload_name}{generics} {{", depth)
+        for f in t.fields:
+            self.visit(f, sum_typeinfo, "pub ", depth + 1, t.name)
+        self.emit("}", depth)
+        self.emit(
+            textwrap.dedent(
+                f"""
+                impl{generics_applied} From<{payload_name}{generics_applied}> for {rustname}{sum_typeinfo.rust_suffix}{generics_applied} {{
+                    fn from(payload: {payload_name}{generics_applied}) -> Self {{
+                        {rustname}{sum_typeinfo.rust_suffix}::{t.name}(payload)
+                    }}
+                }}
+                """
+            ),
+            depth,
+        )
+
     def visitConstructor(self, cons, parent, depth):
         if cons.fields:
             self.emit(f"{cons.name} {{", depth)
@@ -300,7 +361,7 @@ class StructVisitor(TypeInfoEmitVisitor):
         if product.attributes:
             dataname = rustname + "Data"
         self.emit_attrs(depth)
-        has_expr = any(f.type != "identifier" for f in product.fields)
+        has_expr = product_has_expr(product)
         if has_expr:
             datadef = f"{dataname}{generics}"
         else:
@@ -314,20 +375,24 @@ class StructVisitor(TypeInfoEmitVisitor):
             if not has_expr:
                 generics_applied = ""
             self.emit(
-                f"pub type {rustname}<U = ()> = Located<{dataname}{generics_applied}, U>;",
+                f"pub type {rustname}<U = ()> = Attributed<{dataname}{generics_applied}, U>;",
                 depth,
             )
         self.emit("", depth)
 
 
-class FoldTraitDefVisitor(TypeInfoEmitVisitor):
+class FoldTraitDefVisitor(EmitVisitor):
     def visitModule(self, mod, depth):
         self.emit("pub trait Fold<U> {", depth)
         self.emit("type TargetU;", depth + 1)
         self.emit("type Error;", depth + 1)
         self.emit(
             "fn map_user(&mut self, user: U) -> Result<Self::TargetU, Self::Error>;",
-            depth + 2,
+            depth + 1,
+        )
+        self.emit(
+            "fn map_located<T>(&mut self, located: Attributed<T, U>) -> Result<Attributed<T, Self::TargetU>, Self::Error> { let custom = self.map_user(located.custom)?; Ok(Attributed { range: located.range, custom, node: located.node }) }",
+            depth + 1,
         )
         for dfn in mod.dfns:
             self.visit(dfn, depth + 2)
@@ -345,14 +410,14 @@ class FoldTraitDefVisitor(TypeInfoEmitVisitor):
         self.emit("}", depth)
 
 
-class FoldImplVisitor(TypeInfoEmitVisitor):
+class FoldImplVisitor(EmitVisitor):
     def visitModule(self, mod, depth):
         self.emit(
-            "fn fold_located<U, F: Fold<U> + ?Sized, T, MT>(folder: &mut F, node: Located<T, U>, f: impl FnOnce(&mut F, T) -> Result<MT, F::Error>) -> Result<Located<MT, F::TargetU>, F::Error> {",
+            "fn fold_located<U, F: Fold<U> + ?Sized, T, MT>(folder: &mut F, node: Attributed<T, U>, f: impl FnOnce(&mut F, T) -> Result<MT, F::Error>) -> Result<Attributed<MT, F::TargetU>, F::Error> {",
             depth,
         )
         self.emit(
-            "Ok(Located { custom: folder.map_user(node.custom)?, range: node.range, node: f(folder, node.node)? })",
+            "let node = folder.map_located(node)?; Ok(Attributed { custom: node.custom, range: node.range, node: f(folder, node.node)? })",
             depth + 1,
         )
         self.emit("}", depth)
@@ -363,11 +428,11 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
         self.visit(type.value, type.name, depth)
 
     def visitSum(self, sum, name, depth):
+        typeinfo = self.typeinfo[name]
         apply_t, apply_u, apply_target_u = self.get_generics(
             name, "T", "U", "F::TargetU"
         )
         enumname = get_rust_type(name)
-        is_located = bool(sum.attributes)
 
         self.emit(f"impl<T, U> Foldable<T, U> for {enumname}{apply_t} {{", depth)
         self.emit(f"type Mapped = {enumname}{apply_u};", depth + 1)
@@ -383,15 +448,13 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
             f"pub fn fold_{name}<U, F: Fold<U> + ?Sized>(#[allow(unused)] folder: &mut F, node: {enumname}{apply_u}) -> Result<{enumname}{apply_target_u}, F::Error> {{",
             depth,
         )
-        if is_located:
+        if typeinfo.has_attributes:
             self.emit("fold_located(folder, node, |folder, node| {", depth)
-            rustname = enumname + "Kind"
+
-        else:
-            rustname = enumname
         self.emit("match node {", depth + 1)
         for cons in sum.types:
             fields_pattern = self.make_pattern(
-                enumname, rustname, cons.name, cons.fields
+                enumname, typeinfo.rust_suffix, cons.name, cons.fields
             )
             self.emit(
                 f"{fields_pattern[0]} {{ {fields_pattern[1]} }} {fields_pattern[2]} => {{",
@@ -402,7 +465,7 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
             )
             self.emit("}", depth + 2)
         self.emit("}", depth + 1)
-        if is_located:
+        if typeinfo.has_attributes:
             self.emit("})", depth)
         self.emit("}", depth)
 
@@ -411,7 +474,7 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
             name, "T", "U", "F::TargetU"
         )
         structname = get_rust_type(name)
-        is_located = bool(product.attributes)
+        has_attributes = bool(product.attributes)
 
         self.emit(f"impl<T, U> Foldable<T, U> for {structname}{apply_t} {{", depth)
         self.emit(f"type Mapped = {structname}{apply_u};", depth + 1)
@@ -427,7 +490,7 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
             f"pub fn fold_{name}<U, F: Fold<U> + ?Sized>(#[allow(unused)] folder: &mut F, node: {structname}{apply_u}) -> Result<{structname}{apply_target_u}, F::Error> {{",
             depth,
         )
-        if is_located:
+        if has_attributes:
             self.emit("fold_located(folder, node, |folder, node| {", depth)
             rustname = structname + "Data"
         else:
@@ -435,16 +498,16 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
         fields_pattern = self.make_pattern(rustname, structname, None, product.fields)
         self.emit(f"let {rustname} {{ {fields_pattern[1]} }} = node;", depth + 1)
         self.gen_construction(rustname, product.fields, "", depth + 1)
-        if is_located:
+        if has_attributes:
             self.emit("})", depth)
         self.emit("}", depth)
 
-    def make_pattern(self, rustname, pyname, fieldname, fields):
+    def make_pattern(self, rustname, suffix, fieldname, fields):
         if fields:
-            header = f"{pyname}::{fieldname}({rustname}{fieldname}"
+            header = f"{rustname}{suffix}::{fieldname}({rustname}{fieldname}"
             footer = ")"
         else:
-            header = f"{pyname}::{fieldname}"
+            header = f"{rustname}{suffix}::{fieldname}"
             footer = ""
 
         body = ",".join(rust_field(f.name) for f in fields)
@@ -458,7 +521,7 @@ class FoldImplVisitor(TypeInfoEmitVisitor):
         self.emit(f"}}{footer})", depth)
 
 
-class FoldModuleVisitor(TypeInfoEmitVisitor):
+class FoldModuleVisitor(EmitVisitor):
     def visitModule(self, mod):
         depth = 0
         self.emit('#[cfg(feature = "fold")]', depth)
@@ -576,10 +639,10 @@ class TraitImplVisitor(EmitVisitor):
         if sum.attributes:
             rustname = enumname + "Kind"
 
-        self.emit(f"impl NamedNode for ast::{rustname} {{", depth)
+        self.emit(f"impl NamedNode for ast::located::{rustname} {{", depth)
         self.emit(f"const NAME: &'static str = {json.dumps(name)};", depth + 1)
         self.emit("}", depth)
-        self.emit(f"impl Node for ast::{rustname} {{", depth)
+        self.emit(f"impl Node for ast::located::{rustname} {{", depth)
         self.emit(
             "fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef {", depth + 1
         )
@@ -597,10 +660,12 @@ class TraitImplVisitor(EmitVisitor):
         self.emit("}", depth)
 
     def constructor_to_object(self, cons, enumname, rustname, depth):
-        self.emit(f"ast::{rustname}::{cons.name}", depth)
+        self.emit(f"ast::located::{rustname}::{cons.name}", depth)
         if cons.fields:
             fields_pattern = self.make_pattern(cons.fields)
-            self.emit(f"( ast::{enumname}{cons.name} {{ {fields_pattern} }} )", depth)
+            self.emit(
+                f"( ast::located::{enumname}{cons.name} {{ {fields_pattern} }} )", depth
+            )
         self.emit(" => {", depth)
         self.make_node(cons.name, cons.fields, depth + 1)
         self.emit("}", depth)
@@ -610,15 +675,17 @@ class TraitImplVisitor(EmitVisitor):
         if product.attributes:
             structname += "Data"
 
-        self.emit(f"impl NamedNode for ast::{structname} {{", depth)
+        self.emit(f"impl NamedNode for ast::located::{structname} {{", depth)
         self.emit(f"const NAME: &'static str = {json.dumps(name)};", depth + 1)
         self.emit("}", depth)
-        self.emit(f"impl Node for ast::{structname} {{", depth)
+        self.emit(f"impl Node for ast::located::{structname} {{", depth)
         self.emit(
             "fn ast_to_object(self, _vm: &VirtualMachine) -> PyObjectRef {", depth + 1
         )
         fields_pattern = self.make_pattern(product.fields)
-        self.emit(f"let ast::{structname} {{ {fields_pattern} }} = self;", depth + 2)
+        self.emit(
+            f"let ast::located::{structname} {{ {fields_pattern} }} = self;", depth + 2
+        )
         self.make_node(name, product.fields, depth + 2)
         self.emit("}", depth + 1)
         self.emit(
@@ -656,11 +723,14 @@ class TraitImplVisitor(EmitVisitor):
         for cons in sum.types:
             self.emit(f"if _cls.is(Node{cons.name}::static_type()) {{", depth)
             if cons.fields:
-                self.emit(f"ast::{rustname}::{cons.name} (ast::{enumname}{cons.name} {{", depth + 1)
+                self.emit(
+                    f"ast::located::{rustname}::{cons.name} (ast::located::{enumname}{cons.name} {{",
+                    depth + 1,
+                )
                 self.gen_construction_fields(cons, sumname, depth + 1)
                 self.emit("})", depth + 1)
             else:
-                self.emit(f"ast::{rustname}::{cons.name}", depth + 1)
+                self.emit(f"ast::located::{rustname}::{cons.name}", depth + 1)
             self.emit("} else", depth)
 
         self.emit("{", depth)
@@ -684,14 +754,14 @@ class TraitImplVisitor(EmitVisitor):
             )
 
     def gen_construction(self, cons_path, cons, name, depth):
-        self.emit(f"ast::{cons_path} {{", depth)
+        self.emit(f"ast::located::{cons_path} {{", depth)
         self.gen_construction_fields(cons, name, depth + 1)
         self.emit("}", depth)
 
     def extract_location(self, typename, depth):
         row = self.decode_field(asdl.Field("int", "lineno"), typename)
         column = self.decode_field(asdl.Field("int", "col_offset"), typename)
-        self.emit(f"let _location = ast::Location::new({row}, {column});", depth)
+        self.emit(f"let _location = Location::new({row}, {column});", depth)
 
     def decode_field(self, field, typename):
         name = json.dumps(field.name)
@@ -711,84 +781,18 @@ class ChainOfVisitors:
             v.emit("", 0)
 
 
-def write_ast_def(mod, typeinfo, f):
+def write_generic_def(mod, typeinfo, f):
     f.write(
         textwrap.dedent(
             """
         #![allow(clippy::derive_partial_eq_without_eq)]
         
-        pub use crate::constant::*;
+        pub use crate::{Attributed, constant::*};
-        pub use rustpython_compiler_core::text_size::{TextSize, TextRange};
+        use rustpython_compiler_core::{text_size::{TextSize, TextRange}};
 
         type Ident = String;
         \n
-    """
+        """
-        )
-    )
-    StructVisitor(f, typeinfo).emit_attrs(0)
-    f.write(
-        textwrap.dedent(
-            """
-        pub struct Located<T, U = ()> {
-            pub range: TextRange,
-            pub custom: U,
-            pub node: T,
-        }
-    
-        impl<T> Located<T> {
-            pub fn new(start: TextSize, end: TextSize, node: T) -> Self {
-                Self { range: TextRange::new(start, end), custom: (), node }
-            }
-            
-            /// Creates a new node that spans the position specified by `range`.
-            pub fn with_range(node: T, range: TextRange) -> Self {
-                Self {
-                    range,
-                    custom: (),
-                    node,
-                }
-            }
-        
-            /// Returns the absolute start position of the node from the beginning of the document.
-            #[inline]
-            pub const fn start(&self) -> TextSize {
-                self.range.start()
-            }
-        
-            /// Returns the node
-            #[inline]
-            pub fn node(&self) -> &T {
-                &self.node
-            }
-        
-            /// Consumes self and returns the node.
-            #[inline]
-            pub fn into_node(self) -> T {
-                self.node
-            }
-        
-            /// Returns the `range` of the node. The range offsets are absolute to the start of the document.
-            #[inline]
-            pub const fn range(&self) -> TextRange {
-                self.range
-            }
-        
-            /// Returns the absolute position at which the node ends in the source document.
-            #[inline]
-            pub const fn end(&self) -> TextSize {
-                self.range.end()
-            }
-        }
-        
-        impl<T, U> std::ops::Deref for Located<T, U> {
-            type Target = T;
-            
-            fn deref(&self) -> &Self::Target {
-                &self.node
-            }
-        }
-        \n
-    """.lstrip()
         )
     )
 
@@ -796,24 +800,59 @@ def write_ast_def(mod, typeinfo, f):
     c.visit(mod)
 
 
-def write_ast_mod(mod, f):
+def write_located_def(typeinfo, f):
     f.write(
         textwrap.dedent(
             """
-        #![allow(clippy::all)]
+            use rustpython_compiler_core::LocationRange;
 
-        use super::*;
+            pub type Located<T> = super::generic::Attributed<T, LocationRange>;
-        use crate::common::ascii;
+            """
+        )
+    )
+    for info in typeinfo.values():
+        if info.has_userdata:
+            generics = "::<LocationRange>"
+        else:
+            generics = ""
+        f.write(
+            f"pub type {info.rust_sum_name} = super::generic::{info.rust_sum_name}{generics};\n"
+        )
+        if info.rust_suffix:
+            if info.rust_suffix == "Data" and not info.has_expr:
+                generics = ""
+            f.write(
+                f"pub type {info.rust_sum_name}{info.rust_suffix} = super::generic::{info.rust_sum_name}{info.rust_suffix}{generics};\n"
+            )
 
-    """
+
+def write_ast_mod(mod, typeinfo, f):
+    f.write(
+        textwrap.dedent(
+            """
+            #![allow(clippy::all)]
+
+            use super::*;
+            use crate::common::ascii;
+            """
         )
     )
 
-    c = ChainOfVisitors(ClassDefVisitor(f), TraitImplVisitor(f), ExtendModuleVisitor(f))
+    c = ChainOfVisitors(
+        ClassDefVisitor(f, typeinfo),
+        TraitImplVisitor(f, typeinfo),
+        ExtendModuleVisitor(f, typeinfo),
+    )
     c.visit(mod)
 
 
-def main(input_filename, ast_mod_filename, ast_def_filename, dump_module=False):
+def main(
+    input_filename,
+    generic_filename,
+    located_filename,
+    module_filename,
+    dump_module=False,
+):
     auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:]))
     mod = asdl.parse(input_filename)
     if dump_module:
@@ -825,22 +864,34 @@ def main(input_filename, ast_mod_filename, ast_def_filename, dump_module=False):
     typeinfo = {}
     FindUserdataTypesVisitor(typeinfo).visit(mod)
 
-    with ast_def_filename.open("w") as def_file, ast_mod_filename.open("w") as mod_file:
+    with generic_filename.open("w") as generic_file, located_filename.open(
-        def_file.write(auto_gen_msg)
+        "w"
-        write_ast_def(mod, typeinfo, def_file)
+    ) as located_file:
+        generic_file.write(auto_gen_msg)
+        write_generic_def(mod, typeinfo, generic_file)
+        located_file.write(auto_gen_msg)
+        write_located_def(typeinfo, located_file)
 
-        mod_file.write(auto_gen_msg)
+    with module_filename.open("w") as module_file:
-        write_ast_mod(mod, mod_file)
+        module_file.write(auto_gen_msg)
+        write_ast_mod(mod, typeinfo, module_file)
 
-    print(f"{ast_def_filename}, {ast_mod_filename} regenerated.")
+    print(f"{generic_filename}, {located_filename}, {module_filename} regenerated.")
 
 
 if __name__ == "__main__":
     parser = ArgumentParser()
     parser.add_argument("input_file", type=Path)
-    parser.add_argument("-M", "--mod-file", type=Path, required=True)
+    parser.add_argument("-G", "--generic-file", type=Path, required=True)
-    parser.add_argument("-D", "--def-file", type=Path, required=True)
+    parser.add_argument("-L", "--located-file", type=Path, required=True)
+    parser.add_argument("-M", "--module-file", type=Path, required=True)
     parser.add_argument("-d", "--dump-module", action="store_true")
 
     args = parser.parse_args()
-    main(args.input_file, args.mod_file, args.def_file, args.dump_module)
+    main(
+        args.input_file,
+        args.generic_file,
+        args.located_file,
+        args.module_file,
+        args.dump_module,
+    )

ast/src/attributed.rs

@@ -0,0 +1,76 @@
+use rustpython_compiler_core::{
+    text_size::{TextRange, TextSize},
+    Location, LocationRange,
+};
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct Attributed<T, U = ()> {
+    pub range: TextRange,
+    pub custom: U,
+    pub node: T,
+}
+
+impl<T, U> Attributed<T, U> {
+    /// Returns the node
+    #[inline]
+    pub fn node(&self) -> &T {
+        &self.node
+    }
+
+    /// Returns the `range` of the node. The range offsets are absolute to the start of the document.
+    #[inline]
+    pub const fn range(&self) -> TextRange {
+        self.range
+    }
+
+    /// Returns the absolute start position of the node from the beginning of the document.
+    #[inline]
+    pub const fn start(&self) -> TextSize {
+        self.range.start()
+    }
+
+    /// Returns the absolute position at which the node ends in the source document.
+    #[inline]
+    pub const fn end(&self) -> TextSize {
+        self.range.end()
+    }
+}
+
+impl<T> Attributed<T, ()> {
+    /// Creates a new node that spans the position specified by `range`.
+    pub fn new(range: impl Into<TextRange>, node: T) -> Self {
+        Self {
+            range: range.into(),
+            custom: (),
+            node,
+        }
+    }
+
+    /// Consumes self and returns the node.
+    #[inline]
+    pub fn into_node(self) -> T {
+        self.node
+    }
+}
+
+impl<T> Attributed<T, LocationRange> {
+    /// Returns the absolute start position of the node from the beginning of the document.
+    #[inline]
+    pub const fn location(&self) -> Location {
+        self.custom.start
+    }
+
+    /// Returns the absolute position at which the node ends in the source document.
+    #[inline]
+    pub const fn end_location(&self) -> Location {
+        self.custom.end
+    }
+}
+
+impl<T, U> std::ops::Deref for Attributed<T, U> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        &self.node
+    }
+}

ast/src/constant.rs

@@ -137,6 +137,7 @@ impl<U> crate::fold::Fold<U> for ConstantOptimizer {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use rustpython_compiler_core::text_size::TextRange;
 
     #[cfg(feature = "constant-optimization")]
     #[test]
@@ -146,13 +147,13 @@ mod tests {
         let range = TextRange::default();
         #[allow(clippy::let_unit_value)]
         let custom = ();
-        let ast = Located {
+        let ast = Attributed {
             range,
             custom,
             node: ExprTuple {
                 ctx: ExprContext::Load,
                 elts: vec![
-                    Located {
+                    Attributed {
                         range,
                         custom,
                         node: ExprConstant {
@@ -161,7 +162,7 @@ mod tests {
                         }
                         .into(),
                     },
-                    Located {
+                    Attributed {
                         range,
                         custom,
                         node: ExprConstant {
@@ -170,13 +171,13 @@ mod tests {
                         }
                         .into(),
                     },
-                    Located {
+                    Attributed {
                         range,
                         custom,
                         node: ExprTuple {
                             ctx: ExprContext::Load,
                             elts: vec![
-                                Located {
+                                Attributed {
                                     range,
                                     custom,
                                     node: ExprConstant {
@@ -185,7 +186,7 @@ mod tests {
                                     }
                                     .into(),
                                 },
-                                Located {
+                                Attributed {
                                     range,
                                     custom,
                                     node: ExprConstant {
@@ -194,7 +195,7 @@ mod tests {
                                     }
                                     .into(),
                                 },
-                                Located {
+                                Attributed {
                                     range,
                                     custom,
                                     node: ExprConstant {
@@ -216,7 +217,7 @@ mod tests {
             .unwrap_or_else(|e| match e {});
         assert_eq!(
             new_ast,
-            Located {
+            Attributed {
                 range,
                 custom,
                 node: ExprConstant {

ast/src/fold_helpers.rs

@@ -1,6 +1,6 @@
 use crate::{constant, fold::Fold};
 
-pub(crate) trait Foldable<T, U> {
+pub trait Foldable<T, U> {
     type Mapped;
     fn fold<F: Fold<T, TargetU = U> + ?Sized>(
         self,

ast/src/generic.rs

@@ -2,75 +2,11 @@
 
 #![allow(clippy::derive_partial_eq_without_eq)]
 
-pub use crate::constant::*;
+pub use crate::{constant::*, Attributed};
-pub use rustpython_compiler_core::text_size::{TextRange, TextSize};
+use rustpython_compiler_core::text_size::{TextRange, TextSize};
 
 type Ident = String;
 
-#[derive(Clone, Debug, PartialEq)]
-pub struct Located<T, U = ()> {
-    pub range: TextRange,
-    pub custom: U,
-    pub node: T,
-}
-
-impl<T> Located<T> {
-    pub fn new(start: TextSize, end: TextSize, node: T) -> Self {
-        Self {
-            range: TextRange::new(start, end),
-            custom: (),
-            node,
-        }
-    }
-
-    /// Creates a new node that spans the position specified by `range`.
-    pub fn with_range(node: T, range: TextRange) -> Self {
-        Self {
-            range,
-            custom: (),
-            node,
-        }
-    }
-
-    /// Returns the absolute start position of the node from the beginning of the document.
-    #[inline]
-    pub const fn start(&self) -> TextSize {
-        self.range.start()
-    }
-
-    /// Returns the node
-    #[inline]
-    pub fn node(&self) -> &T {
-        &self.node
-    }
-
-    /// Consumes self and returns the node.
-    #[inline]
-    pub fn into_node(self) -> T {
-        self.node
-    }
-
-    /// Returns the `range` of the node. The range offsets are absolute to the start of the document.
-    #[inline]
-    pub const fn range(&self) -> TextRange {
-        self.range
-    }
-
-    /// Returns the absolute position at which the node ends in the source document.
-    #[inline]
-    pub const fn end(&self) -> TextSize {
-        self.range.end()
-    }
-}
-
-impl<T, U> std::ops::Deref for Located<T, U> {
-    type Target = T;
-
-    fn deref(&self) -> &Self::Target {
-        &self.node
-    }
-}
-
 #[derive(Clone, Debug, PartialEq)]
 pub struct ModModule<U = ()> {
     pub body: Vec<Stmt<U>>,
@@ -467,7 +403,7 @@ pub enum StmtKind<U = ()> {
     Break,
     Continue,
 }
-pub type Stmt<U = ()> = Located<StmtKind<U>, U>;
+pub type Stmt<U = ()> = Attributed<StmtKind<U>, U>;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct ExprBoolOp<U = ()> {
@@ -827,7 +763,7 @@ pub enum ExprKind<U = ()> {
     Tuple(ExprTuple<U>),
     Slice(ExprSlice<U>),
 }
-pub type Expr<U = ()> = Located<ExprKind<U>, U>;
+pub type Expr<U = ()> = Attributed<ExprKind<U>, U>;
 
 #[derive(Clone, Debug, PartialEq)]
 pub enum ExprContext {
@@ -906,7 +842,7 @@ impl<U> From<ExcepthandlerExceptHandler<U>> for ExcepthandlerKind<U> {
 pub enum ExcepthandlerKind<U = ()> {
     ExceptHandler(ExcepthandlerExceptHandler<U>),
 }
-pub type Excepthandler<U = ()> = Located<ExcepthandlerKind<U>, U>;
+pub type Excepthandler<U = ()> = Attributed<ExcepthandlerKind<U>, U>;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct Arguments<U = ()> {
@@ -925,21 +861,21 @@ pub struct ArgData<U = ()> {
     pub annotation: Option<Box<Expr<U>>>,
     pub type_comment: Option<String>,
 }
-pub type Arg<U = ()> = Located<ArgData<U>, U>;
+pub type Arg<U = ()> = Attributed<ArgData<U>, U>;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct KeywordData<U = ()> {
     pub arg: Option<Ident>,
     pub value: Expr<U>,
 }
-pub type Keyword<U = ()> = Located<KeywordData<U>, U>;
+pub type Keyword<U = ()> = Attributed<KeywordData<U>, U>;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct AliasData {
     pub name: Ident,
     pub asname: Option<Ident>,
 }
-pub type Alias<U = ()> = Located<AliasData, U>;
+pub type Alias<U = ()> = Attributed<AliasData, U>;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct Withitem<U = ()> {
@@ -1059,7 +995,7 @@ pub enum PatternKind<U = ()> {
     MatchAs(PatternMatchAs<U>),
     MatchOr(PatternMatchOr<U>),
 }
-pub type Pattern<U = ()> = Located<PatternKind<U>, U>;
+pub type Pattern<U = ()> = Attributed<PatternKind<U>, U>;
 
 #[derive(Clone, Debug, PartialEq)]
 pub struct TypeIgnoreTypeIgnore {
@@ -1086,6 +1022,17 @@ pub mod fold {
         type TargetU;
         type Error;
         fn map_user(&mut self, user: U) -> Result<Self::TargetU, Self::Error>;
+        fn map_located<T>(
+            &mut self,
+            located: Attributed<T, U>,
+        ) -> Result<Attributed<T, Self::TargetU>, Self::Error> {
+            let custom = self.map_user(located.custom)?;
+            Ok(Attributed {
+                range: located.range,
+                custom,
+                node: located.node,
+            })
+        }
         fn fold_mod(&mut self, node: Mod<U>) -> Result<Mod<Self::TargetU>, Self::Error> {
             fold_mod(self, node)
         }
@@ -1164,11 +1111,12 @@ pub mod fold {
     }
     fn fold_located<U, F: Fold<U> + ?Sized, T, MT>(
         folder: &mut F,
-        node: Located<T, U>,
+        node: Attributed<T, U>,
         f: impl FnOnce(&mut F, T) -> Result<MT, F::Error>,
-    ) -> Result<Located<MT, F::TargetU>, F::Error> {
+    ) -> Result<Attributed<MT, F::TargetU>, F::Error> {
-        Ok(Located {
+        let node = folder.map_located(node)?;
-            custom: folder.map_user(node.custom)?,
+        Ok(Attributed {
+            custom: node.custom,
             range: node.range,
             node: f(folder, node.node)?,
         })

ast/src/lib.rs

@@ -1,11 +1,21 @@
-mod ast_gen;
+mod attributed;
 mod constant;
 #[cfg(feature = "fold")]
 mod fold_helpers;
+mod generic;
 mod impls;
+#[cfg(feature = "location")]
+pub mod located;
+#[cfg(feature = "location")]
+mod locator;
+
 #[cfg(feature = "unparse")]
 mod unparse;
 
-pub use ast_gen::*;
+pub use attributed::Attributed;
+pub use constant::{Constant, ConversionFlag};
+pub use generic::*;
+#[cfg(feature = "location")]
+pub use locator::Locator;
 
 pub type Suite<U = ()> = Vec<Stmt<U>>;

ast/src/located.rs

@@ -0,0 +1,95 @@
+// File automatically generated by ast/asdl_rs.py.
+
+use rustpython_compiler_core::LocationRange;
+
+pub type Located<T> = super::generic::Attributed<T, LocationRange>;
+pub type Mod = super::generic::Mod<LocationRange>;
+pub type ModModule = super::generic::ModModule<LocationRange>;
+pub type ModInteractive = super::generic::ModInteractive<LocationRange>;
+pub type ModExpression = super::generic::ModExpression<LocationRange>;
+pub type ModFunctionType = super::generic::ModFunctionType<LocationRange>;
+pub type Stmt = super::generic::Stmt<LocationRange>;
+pub type StmtKind = super::generic::StmtKind<LocationRange>;
+pub type StmtFunctionDef = super::generic::StmtFunctionDef<LocationRange>;
+pub type StmtAsyncFunctionDef = super::generic::StmtAsyncFunctionDef<LocationRange>;
+pub type StmtClassDef = super::generic::StmtClassDef<LocationRange>;
+pub type StmtReturn = super::generic::StmtReturn<LocationRange>;
+pub type StmtDelete = super::generic::StmtDelete<LocationRange>;
+pub type StmtAssign = super::generic::StmtAssign<LocationRange>;
+pub type StmtAugAssign = super::generic::StmtAugAssign<LocationRange>;
+pub type StmtAnnAssign = super::generic::StmtAnnAssign<LocationRange>;
+pub type StmtFor = super::generic::StmtFor<LocationRange>;
+pub type StmtAsyncFor = super::generic::StmtAsyncFor<LocationRange>;
+pub type StmtWhile = super::generic::StmtWhile<LocationRange>;
+pub type StmtIf = super::generic::StmtIf<LocationRange>;
+pub type StmtWith = super::generic::StmtWith<LocationRange>;
+pub type StmtAsyncWith = super::generic::StmtAsyncWith<LocationRange>;
+pub type StmtMatch = super::generic::StmtMatch<LocationRange>;
+pub type StmtRaise = super::generic::StmtRaise<LocationRange>;
+pub type StmtTry = super::generic::StmtTry<LocationRange>;
+pub type StmtTryStar = super::generic::StmtTryStar<LocationRange>;
+pub type StmtAssert = super::generic::StmtAssert<LocationRange>;
+pub type StmtImport = super::generic::StmtImport<LocationRange>;
+pub type StmtImportFrom = super::generic::StmtImportFrom<LocationRange>;
+pub type StmtGlobal = super::generic::StmtGlobal;
+pub type StmtNonlocal = super::generic::StmtNonlocal;
+pub type StmtExpr = super::generic::StmtExpr<LocationRange>;
+pub type Expr = super::generic::Expr<LocationRange>;
+pub type ExprKind = super::generic::ExprKind<LocationRange>;
+pub type ExprBoolOp = super::generic::ExprBoolOp<LocationRange>;
+pub type ExprNamedExpr = super::generic::ExprNamedExpr<LocationRange>;
+pub type ExprBinOp = super::generic::ExprBinOp<LocationRange>;
+pub type ExprUnaryOp = super::generic::ExprUnaryOp<LocationRange>;
+pub type ExprLambda = super::generic::ExprLambda<LocationRange>;
+pub type ExprIfExp = super::generic::ExprIfExp<LocationRange>;
+pub type ExprDict = super::generic::ExprDict<LocationRange>;
+pub type ExprSet = super::generic::ExprSet<LocationRange>;
+pub type ExprListComp = super::generic::ExprListComp<LocationRange>;
+pub type ExprSetComp = super::generic::ExprSetComp<LocationRange>;
+pub type ExprDictComp = super::generic::ExprDictComp<LocationRange>;
+pub type ExprGeneratorExp = super::generic::ExprGeneratorExp<LocationRange>;
+pub type ExprAwait = super::generic::ExprAwait<LocationRange>;
+pub type ExprYield = super::generic::ExprYield<LocationRange>;
+pub type ExprYieldFrom = super::generic::ExprYieldFrom<LocationRange>;
+pub type ExprCompare = super::generic::ExprCompare<LocationRange>;
+pub type ExprCall = super::generic::ExprCall<LocationRange>;
+pub type ExprFormattedValue = super::generic::ExprFormattedValue<LocationRange>;
+pub type ExprJoinedStr = super::generic::ExprJoinedStr<LocationRange>;
+pub type ExprConstant = super::generic::ExprConstant;
+pub type ExprAttribute = super::generic::ExprAttribute<LocationRange>;
+pub type ExprSubscript = super::generic::ExprSubscript<LocationRange>;
+pub type ExprStarred = super::generic::ExprStarred<LocationRange>;
+pub type ExprName = super::generic::ExprName;
+pub type ExprList = super::generic::ExprList<LocationRange>;
+pub type ExprTuple = super::generic::ExprTuple<LocationRange>;
+pub type ExprSlice = super::generic::ExprSlice<LocationRange>;
+pub type ExprContext = super::generic::ExprContext;
+pub type Boolop = super::generic::Boolop;
+pub type Operator = super::generic::Operator;
+pub type Unaryop = super::generic::Unaryop;
+pub type Cmpop = super::generic::Cmpop;
+pub type Comprehension = super::generic::Comprehension<LocationRange>;
+pub type Excepthandler = super::generic::Excepthandler<LocationRange>;
+pub type ExcepthandlerKind = super::generic::ExcepthandlerKind<LocationRange>;
+pub type ExcepthandlerExceptHandler = super::generic::ExcepthandlerExceptHandler<LocationRange>;
+pub type Arguments = super::generic::Arguments<LocationRange>;
+pub type Arg = super::generic::Arg<LocationRange>;
+pub type ArgData = super::generic::ArgData<LocationRange>;
+pub type Keyword = super::generic::Keyword<LocationRange>;
+pub type KeywordData = super::generic::KeywordData<LocationRange>;
+pub type Alias = super::generic::Alias<LocationRange>;
+pub type AliasData = super::generic::AliasData;
+pub type Withitem = super::generic::Withitem<LocationRange>;
+pub type MatchCase = super::generic::MatchCase<LocationRange>;
+pub type Pattern = super::generic::Pattern<LocationRange>;
+pub type PatternKind = super::generic::PatternKind<LocationRange>;
+pub type PatternMatchValue = super::generic::PatternMatchValue<LocationRange>;
+pub type PatternMatchSingleton = super::generic::PatternMatchSingleton;
+pub type PatternMatchSequence = super::generic::PatternMatchSequence<LocationRange>;
+pub type PatternMatchMapping = super::generic::PatternMatchMapping<LocationRange>;
+pub type PatternMatchClass = super::generic::PatternMatchClass<LocationRange>;
+pub type PatternMatchStar = super::generic::PatternMatchStar;
+pub type PatternMatchAs = super::generic::PatternMatchAs<LocationRange>;
+pub type PatternMatchOr = super::generic::PatternMatchOr<LocationRange>;
+pub type TypeIgnore = super::generic::TypeIgnore;
+pub type TypeIgnoreTypeIgnore = super::generic::TypeIgnoreTypeIgnore;

ast/src/locator.rs

@@ -0,0 +1,57 @@
+use crate::attributed::Attributed;
+use crate::fold_helpers::Foldable;
+use rustpython_compiler_core::{
+    text_size::{TextRange, TextSize},
+    Location, LocationRange,
+};
+
+/// Converts source code byte-offset to Python convention line and column numbers.
+#[derive(Default)]
+pub struct Locator<'a> {
+    source: &'a str,
+}
+
+impl<'a> Locator<'a> {
+    #[inline]
+    pub fn new(source: &'a str) -> Self {
+        Self { source }
+    }
+
+    pub fn source(&'a self) -> &'a str {
+        self.source
+    }
+
+    pub fn locate(&mut self, offset: TextSize) -> Location {
+        todo!()
+    }
+
+    pub fn locate_range(&mut self, range: TextRange) -> LocationRange {
+        self.locate(range.start())..self.locate(range.end())
+    }
+
+    pub fn locate_ast<X: Foldable<(), LocationRange>>(&mut self, ast: X) -> X::Mapped {
+        ast.fold(self).unwrap()
+    }
+}
+
+impl crate::fold::Fold<()> for Locator<'_> {
+    type TargetU = LocationRange;
+    type Error = std::convert::Infallible;
+
+    #[cold]
+    fn map_user(&mut self, _user: ()) -> Result<Self::TargetU, Self::Error> {
+        unreachable!("implemented map_located");
+    }
+
+    fn map_located<T>(
+        &mut self,
+        node: Attributed<T, ()>,
+    ) -> Result<Attributed<T, Self::TargetU>, Self::Error> {
+        let location = self.locate_range(node.range);
+        Ok(Attributed {
+            range: node.range,
+            custom: location,
+            node: node.node,
+        })
+    }
+}

core/src/error.rs

@@ -1,11 +1,10 @@
-use ruff_text_size::TextSize;
+use crate::{text_size::TextSize, Location};
-use std::error::Error as StdError;
 use std::fmt::Display;
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct BaseError<T> {
     pub error: T,
-    pub location: TextSize,
+    pub offset: TextSize,
     pub source_path: String,
 }
 
@@ -17,11 +16,11 @@ impl<T> std::ops::Deref for BaseError<T> {
     }
 }
 
-impl<T> StdError for BaseError<T>
+impl<T> std::error::Error for BaseError<T>
 where
-    T: StdError + 'static,
+    T: std::error::Error + 'static,
 {
-    fn source(&self) -> Option<&(dyn StdError + 'static)> {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
         Some(&self.error)
     }
 }
@@ -35,7 +34,7 @@ where
             f,
             "{} at byte offset {}",
             &self.error,
-            u32::from(self.location)
+            u32::from(self.offset)
         )
     }
 }
@@ -51,7 +50,7 @@ impl<T> BaseError<T> {
     {
         Self {
             error: obj.error.into(),
-            location: obj.location,
+            offset: obj.offset,
             source_path: obj.source_path,
         }
     }
@@ -62,4 +61,64 @@ impl<T> BaseError<T> {
     {
         BaseError::from(self)
     }
+
+    pub fn into_located<U>(self, locator: &str) -> LocatedError<U>
+    where
+        T: Into<U>,
+    {
+        todo!()
+    }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct LocatedError<T> {
+    pub error: T,
+    pub location: Location,
+    pub source_path: String,
+}
+
+impl<T> LocatedError<T> {
+    pub fn error(self) -> T {
+        self.error
+    }
+
+    pub fn from<U>(obj: LocatedError<U>) -> Self
+    where
+        U: Into<T>,
+    {
+        Self {
+            error: obj.error.into(),
+            location: obj.location,
+            source_path: obj.source_path,
+        }
+    }
+
+    pub fn into<U>(self) -> LocatedError<U>
+    where
+        T: Into<U>,
+    {
+        LocatedError::from(self)
+    }
+}
+
+impl<T> Display for LocatedError<T>
+where
+    T: std::fmt::Display,
+{
+    fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(
+            f,
+            "{} at row {} col {}",
+            &self.error, self.location.row, self.location.column,
+        )
+    }
+}
+
+impl<T> std::error::Error for LocatedError<T>
+where
+    T: std::error::Error + 'static,
+{
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        Some(&self.error)
+    }
 }

core/src/lib.rs

@@ -8,8 +8,13 @@ pub mod marshal;
 mod mode;
 
 pub use bytecode::*;
-pub use error::BaseError;
+pub use error::{BaseError, LocatedError};
-pub use location::Location;
+pub use location::{Location, LocationRange};
 pub use mode::Mode;
 
 pub use ruff_text_size as text_size; // re-export mandatory and frequently accessed dependency
+
+// FIXME: temp code
+pub fn to_location(offset: &text_size::TextSize, source: &str) -> Location {
+    todo!()
+}

core/src/location.rs

@@ -1,9 +1,6 @@
-#[cfg(feature = "serde")]
-use serde::{Deserialize, Serialize};
-
 /// Source code location.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
-#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
 pub struct Location {
     pub(super) row: u32,
     pub(super) column: u32,
@@ -96,6 +93,8 @@ impl Location {
     }
 }
 
+pub type LocationRange = std::ops::Range<Location>;
+
 #[cfg(test)]
 mod tests {
     use super::*;

parser/src/function.rs

@@ -114,8 +114,7 @@ pub(crate) fn parse_args(func_args: Vec<FunctionArgument>) -> Result<ArgumentLis
                 }
 
                 keywords.push(ast::Keyword::new(
-                    start,
+                    start..end,
-                    end,
                     ast::KeywordData { arg: name, value },
                 ));
             }

parser/src/lexer.rs

@@ -211,9 +211,9 @@ pub fn lex(source: &str, mode: Mode) -> impl Iterator<Item = LexResult> + '_ {
 pub fn lex_located(
     source: &str,
     mode: Mode,
-    start_location: TextSize,
+    start_offset: TextSize,
 ) -> impl Iterator<Item = LexResult> + '_ {
-    SoftKeywordTransformer::new(Lexer::new(source.chars(), start_location), mode)
+    SoftKeywordTransformer::new(Lexer::new(source.chars(), start_offset), mode)
 }
 
 impl<T> Lexer<T>

parser/src/parser.rs

@@ -92,9 +92,9 @@ pub fn parse_expression(source: &str, path: &str) -> Result<ast::Expr, ParseErro
 pub fn parse_expression_located(
     source: &str,
     path: &str,
-    location: TextSize,
+    offset: TextSize,
 ) -> Result<ast::Expr, ParseError> {
-    parse_located(source, Mode::Expression, path, location).map(|top| match top {
+    parse_located(source, Mode::Expression, path, offset).map(|top| match top {
         ast::Mod::Expression(ast::ModExpression { body }) => *body,
         _ => unreachable!(),
     })
@@ -161,9 +161,9 @@ pub fn parse_located(
     source: &str,
     mode: Mode,
     source_path: &str,
-    location: TextSize,
+    offset: TextSize,
 ) -> Result<ast::Mod, ParseError> {
-    let lxr = lexer::lex_located(source, mode, location);
+    let lxr = lexer::lex_located(source, mode, offset);
     parse_tokens(lxr, mode, source_path)
 }
 
@@ -233,17 +233,17 @@ fn parse_error_from_lalrpop(
         // TODO: Are there cases where this isn't an EOF?
         LalrpopError::InvalidToken { location } => ParseError {
             error: ParseErrorType::Eof,
-            location,
+            offset: location,
             source_path,
         },
         LalrpopError::ExtraToken { token } => ParseError {
             error: ParseErrorType::ExtraToken(token.1),
-            location: token.0,
+            offset: token.0,
             source_path,
         },
         LalrpopError::User { error } => ParseError {
             error: ParseErrorType::Lexical(error.error),
-            location: error.location,
+            offset: error.location,
             source_path,
         },
         LalrpopError::UnrecognizedToken { token, expected } => {
@@ -252,7 +252,7 @@ fn parse_error_from_lalrpop(
             let expected = (expected.len() == 1).then(|| expected[0].clone());
             ParseError {
                 error: ParseErrorType::UnrecognizedToken(token.1, expected),
-                location: token.0,
+                offset: token.0,
                 source_path,
             }
         }
@@ -262,13 +262,13 @@ fn parse_error_from_lalrpop(
             if indent_error {
                 ParseError {
                     error: ParseErrorType::Lexical(LexicalErrorType::IndentationError),
-                    location,
+                    offset: location,
                     source_path,
                 }
             } else {
                 ParseError {
                     error: ParseErrorType::Eof,
-                    location,
+                    offset: location,
                     source_path,
                 }
             }

parser/src/python.lalrpop

@@ -68,7 +68,7 @@ SmallStatement: ast::Stmt = {
 PassStatement: ast::Stmt = {
     <location:@L> "pass" <end_location:@R> => {
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtKind::Pass,
         )
@@ -78,7 +78,7 @@ PassStatement: ast::Stmt = {
 DelStatement: ast::Stmt = {
     <location:@L> "del" <targets:ExpressionList2> <end_location:@R> => {
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtDelete { targets: targets.into_iter().map(|expr| set_context(expr, ast::ExprContext::Del)).collect() }.into()
         )
@@ -90,7 +90,7 @@ ExpressionStatement: ast::Stmt = {
         // Just an expression, no assignment:
         if suffix.is_empty() {
             ast::Stmt::new(
-                location,
+                location..
                 end_location,
                 ast::StmtExpr { value: Box::new(expression) }.into()
             )
@@ -105,7 +105,7 @@ ExpressionStatement: ast::Stmt = {
             let value = Box::new(values.into_iter().next().unwrap());
 
             ast::Stmt::new(
-                location,
+                location..
                 end_location,
                 ast::StmtAssign { targets, value, type_comment: None }.into()
             )
@@ -113,7 +113,7 @@ ExpressionStatement: ast::Stmt = {
     },
     <location:@L> <target:TestOrStarExprList> <op:AugAssign> <rhs:TestListOrYieldExpr> <end_location:@R> => {
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtAugAssign {
                 target: Box::new(set_context(target, ast::ExprContext::Store)),
@@ -125,7 +125,7 @@ ExpressionStatement: ast::Stmt = {
     <location:@L> <target:Test<"all">> ":" <annotation:Test<"all">> <rhs:AssignSuffix?> <end_location:@R> => {
         let simple = matches!(target.node, ast::ExprKind::Name { .. });
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtAnnAssign {
                 target: Box::new(set_context(target, ast::ExprContext::Store)),
@@ -186,28 +186,28 @@ AugAssign: ast::Operator = {
 FlowStatement: ast::Stmt = {
     <location:@L> "break" <end_location:@R> => {
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtKind::Break,
         )
     },
     <location:@L> "continue" <end_location:@R> => {
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtKind::Continue,
         )
     },
     <location:@L> "return" <value:TestList?> <end_location:@R> => {
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtReturn { value: value.map(Box::new) }.into()
         )
     },
     <location:@L> <expression:YieldExpr> <end_location:@R> => {
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtExpr { value: Box::new(expression) }.into()
         )
@@ -218,14 +218,14 @@ FlowStatement: ast::Stmt = {
 RaiseStatement: ast::Stmt = {
     <location:@L> "raise" <end_location:@R> => {
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtRaise { exc: None, cause: None }.into()
         )
     },
     <location:@L> "raise" <t:Test<"all">> <c:("from" Test<"all">)?> <end_location:@R> => {
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtRaise { exc: Some(Box::new(t)), cause: c.map(|x| Box::new(x.1)) }.into()
         )
@@ -235,7 +235,7 @@ RaiseStatement: ast::Stmt = {
 ImportStatement: ast::Stmt = {
     <location:@L> "import" <names: OneOrMore<ImportAsAlias<DottedName>>> <end_location:@R> => {
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtImport { names }.into()
         )
@@ -243,7 +243,7 @@ ImportStatement: ast::Stmt = {
     <location:@L> "from" <source:ImportFromLocation> "import" <names: ImportAsNames> <end_location:@R> => {
         let (level, module) = source;
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtImportFrom {
                 level,
@@ -273,14 +273,14 @@ ImportAsNames: Vec<ast::Alias> = {
     <location:@L> "(" <i:OneOrMore<ImportAsAlias<Identifier>>> ","? ")" <end_location:@R> => i,
     <location:@L> "*" <end_location:@R> => {
         // Star import all
-        vec![ast::Alias::new(location, end_location, ast::AliasData { name: "*".to_string(), asname: None })]
+        vec![ast::Alias::new(location..end_location, ast::AliasData { name: "*".to_string(), asname: None })]
     },
 };
 
 
 #[inline]
 ImportAsAlias<I>: ast::Alias = {
-    <location:@L> <name:I> <a: ("as" Identifier)?> <end_location:@R> => ast::Alias::new(location, end_location, ast::AliasData { name, asname: a.map(|a| a.1) }),
+    <location:@L> <name:I> <a: ("as" Identifier)?> <end_location:@R> => ast::Alias::new(location..end_location, ast::AliasData { name, asname: a.map(|a| a.1) }),
 }
 
 // A name like abc or abc.def.ghi
@@ -299,7 +299,7 @@ DottedName: String = {
 GlobalStatement: ast::Stmt = {
     <location:@L> "global" <names:OneOrMore<Identifier>> <end_location:@R> => {
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtGlobal { names }.into()
         )
@@ -309,7 +309,7 @@ GlobalStatement: ast::Stmt = {
 NonlocalStatement: ast::Stmt = {
     <location:@L> "nonlocal" <names:OneOrMore<Identifier>> <end_location:@R> => {
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtNonlocal { names }.into()
         )
@@ -319,7 +319,7 @@ NonlocalStatement: ast::Stmt = {
 AssertStatement: ast::Stmt = {
     <location:@L> "assert" <test:Test<"all">> <msg: ("," Test<"all">)?> <end_location:@R> => {
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtAssert {
                 test: Box::new(test),
@@ -350,7 +350,7 @@ MatchStatement: ast::Stmt = {
             .unwrap()
             .end();
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtMatch {
                 subject: Box::new(subject),
@@ -367,7 +367,7 @@ MatchStatement: ast::Stmt = {
             .unwrap()
             .end();
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtMatch {
                 subject: Box::new(subject),
@@ -386,11 +386,11 @@ MatchStatement: ast::Stmt = {
         let mut subjects = subjects;
         subjects.insert(0, subject);
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtMatch {
                 subject: Box::new(ast::Expr::new(
-                    location,
+                    location..
                     end_location,
                     ast::ExprTuple {
                         elts: subjects,
@@ -421,7 +421,7 @@ Guard: ast::Expr = {
 
 Patterns: ast::Pattern = {
     <location:@L> <pattern:Pattern> "," <end_location:@R> => ast::Pattern::new(
-        location,
+        location..
         end_location,
         ast::PatternMatchSequence {
             patterns: vec![pattern]
@@ -431,7 +431,7 @@ Patterns: ast::Pattern = {
         let mut patterns = patterns;
         patterns.insert(0, pattern);
         ast::Pattern::new(
-            location,
+            location..
             end_location,
             ast::PatternMatchSequence {
                 patterns
@@ -455,7 +455,7 @@ AsPattern: ast::Pattern = {
             })?
         } else {
             Ok(ast::Pattern::new(
-                location,
+                location..
                 end_location,
                 ast::PatternMatchAs {
                    pattern: Some(Box::new(pattern)),
@@ -472,7 +472,7 @@ OrPattern: ast::Pattern = {
         let mut patterns = patterns;
         patterns.insert(0, pattern);
         ast::Pattern::new(
-            location,
+            location..
             end_location,
             ast::PatternMatchOr { patterns }.into()
         )
@@ -481,37 +481,37 @@ OrPattern: ast::Pattern = {
 
 ClosedPattern: ast::Pattern = {
     <location:@L> <node:LiteralPattern> <end_location:@R> => ast::Pattern::new(
-        location,
+        location..
         end_location,
         node,
     ),
     <location:@L> <node:CapturePattern> <end_location:@R> => ast::Pattern::new(
-        location,
+        location..
         end_location,
         node,
     ),
     <location:@L> <node:StarPattern> <end_location:@R> => ast::Pattern::new(
-        location,
+        location..
         end_location,
         node,
     ),
     <location:@L> <node:ValuePattern> <end_location:@R> => ast::Pattern::new(
-        location,
+        location..
         end_location,
         node,
     ),
     <location:@L> <node:SequencePattern> <end_location:@R> => ast::Pattern::new(
-        location,
+        location..
         end_location,
         node,
     ),
     <location:@L> <node:MappingPattern> <end_location:@R> => ast::Pattern::new(
-        location,
+        location..
         end_location,
         node,
     ),
     <location:@L> <node:ClassPattern> <end_location:@R> => ast::Pattern::new(
-        location,
+        location..
         end_location,
         node,
     ),
@@ -543,7 +543,7 @@ StarPattern: ast::PatternKind = {
 
 ConstantAtom: ast::Expr = {
     <location:@L> <value:Constant> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprConstant { value, kind: None }.into()
     ),
@@ -552,7 +552,7 @@ ConstantAtom: ast::Expr = {
 ConstantExpr: ast::Expr = {
     ConstantAtom,
     <location:@L> "-" <operand:ConstantAtom> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprUnaryOp {
             op: ast::Unaryop::USub,
@@ -563,7 +563,7 @@ ConstantExpr: ast::Expr = {
 
 AddOpExpr: ast::Expr = {
     <location:@L> <left:ConstantExpr> <op:AddOp> <right:ConstantAtom> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprBinOp {
             left: Box::new(left),
@@ -603,7 +603,7 @@ CapturePattern: ast::PatternKind = {
 
 MatchName: ast::Expr = {
     <location:@L> <name:Identifier> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprName { id: name, ctx: ast::ExprContext::Load }.into(),
     ),
@@ -611,7 +611,7 @@ MatchName: ast::Expr = {
 
 MatchNameOrAttr: ast::Expr = {
     <location:@L> <name:MatchName> "." <attr:Identifier> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprAttribute {
             value: Box::new(name),
@@ -620,7 +620,7 @@ MatchNameOrAttr: ast::Expr = {
         }.into(),
     ),
     <location:@L> <e:MatchNameOrAttr> "." <attr:Identifier> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprAttribute {
             value: Box::new(e),
@@ -641,7 +641,7 @@ MappingKey: ast::Expr = {
     AddOpExpr,
     MatchNameOrAttr,
     <location:@L> "None" <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprConstant {
             value: ast::Constant::None,
@@ -649,7 +649,7 @@ MappingKey: ast::Expr = {
         }.into(),
     ),
     <location:@L> "True" <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprConstant {
             value: true.into(),
@@ -657,7 +657,7 @@ MappingKey: ast::Expr = {
         }.into(),
     ),
     <location:@L> "False" <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprConstant {
             value: false.into(),
@@ -804,7 +804,7 @@ IfStatement: ast::Stmt = {
         // handle elif:
         for i in s2.into_iter().rev() {
             let x = ast::Stmt::new(
-                i.0,
+                i.0..
                 end_location,
                 ast::StmtIf { test: Box::new(i.2), body: i.4, orelse: last }.into()
             );
@@ -812,7 +812,7 @@ IfStatement: ast::Stmt = {
         }
 
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtIf { test: Box::new(test), body, orelse: last }.into()
         )
@@ -828,7 +828,7 @@ WhileStatement: ast::Stmt = {
             .unwrap()
             .end();
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtWhile {
                 test: Box::new(test),
@@ -855,7 +855,7 @@ ForStatement: ast::Stmt = {
         } else {
             ast::StmtFor { target, iter, body, orelse, type_comment }.into()
         };
-        ast::Stmt::new(location, end_location, node)
+        ast::Stmt::new(location..end_location, node)
     },
 };
 
@@ -870,7 +870,7 @@ TryStatement: ast::Stmt = {
             .or_else(|| handlers.last().map(|last| last.end()))
             .unwrap();
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtTry {
                 body,
@@ -890,7 +890,7 @@ TryStatement: ast::Stmt = {
             .or_else(|| handlers.last().map(|last| last.end()))
             .unwrap();
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtTryStar {
                 body,
@@ -906,7 +906,7 @@ TryStatement: ast::Stmt = {
         let finalbody = finally.2;
         let end_location = finalbody.last().unwrap().end();
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtTry {
                 body,
@@ -922,7 +922,7 @@ ExceptStarClause: ast::Excepthandler = {
     <location:@L> "except" "*" <typ:Test<"all">> ":" <body:Suite> => {
         let end_location = body.last().unwrap().end();
         ast::Excepthandler::new(
-            location,
+            location..
             end_location,
             ast::ExcepthandlerExceptHandler {
                 type_: Some(Box::new(typ)),
@@ -934,7 +934,7 @@ ExceptStarClause: ast::Excepthandler = {
     <location:@L> "except" "*" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
         let end_location = body.last().unwrap().end();
         ast::Excepthandler::new(
-            location,
+            location..
             end_location,
             ast::ExcepthandlerExceptHandler {
                 type_: Some(Box::new(x.0)),
@@ -950,7 +950,7 @@ ExceptClause: ast::Excepthandler = {
     <location:@L> "except" <typ:Test<"all">?> ":" <body:Suite> => {
         let end_location = body.last().unwrap().end();
         ast::Excepthandler::new(
-            location,
+            location..
             end_location,
             ast::ExcepthandlerExceptHandler {
                 type_: typ.map(Box::new),
@@ -962,7 +962,7 @@ ExceptClause: ast::Excepthandler = {
     <location:@L> "except" <x:(Test<"all"> "as" Identifier)> ":" <body:Suite> => {
         let end_location = body.last().unwrap().end();
         ast::Excepthandler::new(
-            location,
+            location..
             end_location,
             ast::ExcepthandlerExceptHandler {
                 type_: Some(Box::new(x.0)),
@@ -982,7 +982,7 @@ WithStatement: ast::Stmt = {
         } else {
             ast::StmtWith { items, body, type_comment }.into()
         };
-        ast::Stmt::new(location, end_location, node)
+        ast::Stmt::new(location..end_location, node)
     },
 };
 
@@ -1023,7 +1023,7 @@ FuncDef: ast::Stmt = {
         } else {
             ast::StmtFunctionDef { name, args, body, decorator_list, returns, type_comment }.into()
         };
-        ast::Stmt::new(location, end_location, node)
+        ast::Stmt::new(location..end_location, node)
     },
 };
 
@@ -1126,7 +1126,7 @@ ParameterDef<ArgType>: (ast::Arg, Option<ast::Expr>) = {
 
 UntypedParameter: ast::Arg = {
     <location:@L> <arg:Identifier> <end_location:@R> => ast::Arg::new(
-        location,
+        location..
         end_location,
         ast::ArgData { arg, annotation: None, type_comment: None },
     ),
@@ -1135,14 +1135,14 @@ UntypedParameter: ast::Arg = {
 TypedParameter: ast::Arg = {
     <location:@L> <arg:Identifier> <a:(":" Test<"all">)?> <end_location:@R> => {
         let annotation = a.map(|x| Box::new(x.1));
-        ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None })
+        ast::Arg::new(location..end_location, ast::ArgData { arg, annotation, type_comment: None })
     },
 };
 
 StarTypedParameter: ast::Arg = {
     <location:@L> <arg:Identifier> <a:(":" TestOrStarExpr)?> <end_location:@R> => {
         let annotation = a.map(|x| Box::new(x.1));
-        ast::Arg::new(location, end_location, ast::ArgData { arg, annotation, type_comment: None })
+        ast::Arg::new(location..end_location, ast::ArgData { arg, annotation, type_comment: None })
     },
 };
 
@@ -1193,7 +1193,7 @@ ClassDef: ast::Stmt = {
         };
         let end_location = body.last().unwrap().end();
         ast::Stmt::new(
-            location,
+            location..
             end_location,
             ast::StmtClassDef {
                 name,
@@ -1215,12 +1215,12 @@ Decorator: ast::Expr = {
 
 YieldExpr: ast::Expr = {
     <location:@L> "yield" <value:TestList?> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprYield { value: value.map(Box::new) }.into()
     ),
     <location:@L> "yield" "from" <e:Test<"all">> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprYieldFrom { value: Box::new(e) }.into()
     ),
@@ -1228,7 +1228,7 @@ YieldExpr: ast::Expr = {
 
 Test<Goal>: ast::Expr = {
     <location:@L> <body:OrTest<"all">> "if" <test:OrTest<"all">> "else" <orelse:Test<"all">> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprIfExp {
             test: Box::new(test),
@@ -1248,11 +1248,11 @@ NamedExpressionTest: ast::Expr = {
 NamedExpression: ast::Expr = {
     <location:@L> <id:Identifier> <end_location:@R> ":=" <value:Test<"all">> => {
         ast::Expr::new(
-            location,
+            location..
             value.end(),
             ast::ExprNamedExpr {
                 target: Box::new(ast::Expr::new(
-                    location,
+                    location..
                     end_location,
                     ast::ExprName { id, ctx: ast::ExprContext::Store }.into(),
                 )),
@@ -1279,7 +1279,7 @@ LambdaDef: ast::Expr = {
         ))?;
 
         Ok(ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprLambda {
                 args: Box::new(p),
@@ -1294,7 +1294,7 @@ OrTest<Goal>: ast::Expr = {
         let mut values = vec![e1];
         values.extend(e2.into_iter().map(|e| e.1));
         ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprBoolOp { op: ast::Boolop::Or, values }.into()
         )
@@ -1307,7 +1307,7 @@ AndTest<Goal>: ast::Expr = {
         let mut values = vec![e1];
         values.extend(e2.into_iter().map(|e| e.1));
         ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprBoolOp { op: ast::Boolop::And, values }.into()
         )
@@ -1317,7 +1317,7 @@ AndTest<Goal>: ast::Expr = {
 
 NotTest<Goal>: ast::Expr = {
     <location:@L> "not" <e:NotTest<"all">> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprUnaryOp { operand: Box::new(e), op: ast::Unaryop::Not }.into()
     ),
@@ -1328,7 +1328,7 @@ Comparison<Goal>: ast::Expr = {
     <location:@L> <left:Expression<"all">> <comparisons:(CompOp Expression<"all">)+> <end_location:@R> => {
         let (ops, comparators) = comparisons.into_iter().unzip();
         ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprCompare { left: Box::new(left), ops, comparators }.into()
         )
@@ -1351,7 +1351,7 @@ CompOp: ast::Cmpop = {
 
 Expression<Goal>: ast::Expr = {
     <location:@L> <e1:Expression<"all">> "|" <e2:XorExpression<"all">> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprBinOp { left: Box::new(e1), op: ast::Operator::BitOr, right: Box::new(e2) }.into()
     ),
@@ -1360,7 +1360,7 @@ Expression<Goal>: ast::Expr = {
 
 XorExpression<Goal>: ast::Expr = {
     <location:@L> <e1:XorExpression<"all">> "^" <e2:AndExpression<"all">> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprBinOp { left: Box::new(e1), op: ast::Operator::BitXor, right: Box::new(e2) }.into()
     ),
@@ -1369,7 +1369,7 @@ XorExpression<Goal>: ast::Expr = {
 
 AndExpression<Goal>: ast::Expr = {
     <location:@L> <e1:AndExpression<"all">> "&" <e2:ShiftExpression<"all">> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprBinOp { left: Box::new(e1), op: ast::Operator::BitAnd, right: Box::new(e2) }.into()
     ),
@@ -1378,7 +1378,7 @@ AndExpression<Goal>: ast::Expr = {
 
 ShiftExpression<Goal>: ast::Expr = {
     <location:@L> <e1:ShiftExpression<"all">> <op:ShiftOp> <e2:ArithmeticExpression<"all">> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprBinOp { left: Box::new(e1), op, right: Box::new(e2) }.into()
     ),
@@ -1392,7 +1392,7 @@ ShiftOp: ast::Operator = {
 
 ArithmeticExpression<Goal>: ast::Expr = {
     <location:@L> <a:ArithmeticExpression<"all">> <op:AddOp> <b:Term<"all">> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprBinOp { left: Box::new(a), op, right: Box::new(b) }.into()
     ),
@@ -1406,7 +1406,7 @@ AddOp: ast::Operator = {
 
 Term<Goal>: ast::Expr = {
     <location:@L> <a:Term<"all">> <op:MulOp> <b:Factor<"all">> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprBinOp { left: Box::new(a), op, right: Box::new(b) }.into()
     ),
@@ -1423,7 +1423,7 @@ MulOp: ast::Operator = {
 
 Factor<Goal>: ast::Expr = {
     <location:@L> <op:UnaryOp> <e:Factor<"all">> <end_location:@R>  => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprUnaryOp { operand: Box::new(e), op }.into()
     ),
@@ -1438,7 +1438,7 @@ UnaryOp: ast::Unaryop = {
 
 Power<Goal>: ast::Expr = {
     <location:@L> <e:AtomExpr<"all">> "**" <b:Factor<"all">> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprBinOp { left: Box::new(e), op: ast::Operator::Pow, right: Box::new(b) }.into()
     ),
@@ -1448,7 +1448,7 @@ Power<Goal>: ast::Expr = {
 AtomExpr<Goal>: ast::Expr = {
     <location:@L> "await" <atom:AtomExpr2<"all">> <end_location:@R> => {
         ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprAwait { value: Box::new(atom) }.into()
         )
@@ -1460,18 +1460,18 @@ AtomExpr2<Goal>: ast::Expr = {
     Atom<Goal>,
     <location:@L> <f:AtomExpr2<"all">> "(" <a:ArgumentList> ")" <end_location:@R> => {
         ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprCall { func: Box::new(f), args: a.args, keywords: a.keywords }.into()
         )
     },
     <location:@L> <e:AtomExpr2<"all">> "[" <s:SubscriptList> "]" <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprSubscript { value: Box::new(e), slice: Box::new(s), ctx: ast::ExprContext::Load }.into()
     ),
     <location:@L> <e:AtomExpr2<"all">> "." <attr:Identifier> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprAttribute { value: Box::new(e), attr, ctx: ast::ExprContext::Load }.into()
     ),
@@ -1488,7 +1488,7 @@ SubscriptList: ast::Expr = {
             }
 
             ast::Expr::new(
-                location,
+                location..
                 end_location,
                 ast::ExprTuple { elts: dims, ctx: ast::ExprContext::Load }.into(),
             )
@@ -1503,7 +1503,7 @@ Subscript: ast::Expr = {
         let upper = e2.map(Box::new);
         let step = e3.flatten().map(Box::new);
         ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprSlice { lower, upper, step }.into()
         )
@@ -1517,26 +1517,26 @@ SliceOp: Option<ast::Expr> = {
 Atom<Goal>: ast::Expr = {
     <location:@L> <s:(@L string @R)+> =>? Ok(parse_strings(s)?),
     <location:@L> <value:Constant> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprConstant { value, kind: None }.into()
     ),
     <location:@L> <name:Identifier> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprName { id: name, ctx: ast::ExprContext::Load }.into()
     ),
     <location:@L> "[" <e:ListLiteralValues?> "]"<end_location:@R>  => {
         let elts = e.unwrap_or_default();
         ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprList { elts, ctx: ast::ExprContext::Load }.into()
         )
     },
     <location:@L> "[" <elt:TestOrStarNamedExpr> <generators:CompFor> "]" <end_location:@R> => {
         ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprListComp { elt: Box::new(elt), generators }.into()
         )
@@ -1546,7 +1546,7 @@ Atom<Goal>: ast::Expr = {
             elts.into_iter().next().unwrap()
         } else {
             ast::Expr::new(
-                location,
+                location..
                 end_location,
                 ast::ExprTuple { elts, ctx: ast::ExprContext::Load }.into()
             )
@@ -1564,21 +1564,21 @@ Atom<Goal>: ast::Expr = {
         } else {
             let elts = left.into_iter().flatten().chain([mid]).chain(right).collect();
             Ok(ast::Expr::new(
-                location,
+                location..
                 end_location,
                 ast::ExprTuple { elts, ctx: ast::ExprContext::Load }.into(),
             ))
         }
     },
     <location:@L> "(" ")" <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprTuple { elts: Vec::new(), ctx: ast::ExprContext::Load }.into()
     ),
     "(" <e:YieldExpr> ")" => e,
     <location:@L> "(" <elt:NamedExpressionTest> <generators:CompFor> ")" <end_location:@R> => {
         ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprGeneratorExp { elt: Box::new(elt), generators }.into()
         )
@@ -1596,14 +1596,14 @@ Atom<Goal>: ast::Expr = {
             .map(|(k, v)| (k.map(|x| *x), v))
             .unzip();
         ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprDict { keys, values }.into()
         )
     },
     <location:@L> "{" <e1:DictEntry> <generators:CompFor> "}" <end_location:@R> => {
         ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprDictComp {
                 key: Box::new(e1.0),
@@ -1613,21 +1613,21 @@ Atom<Goal>: ast::Expr = {
         )
     },
     <location:@L> "{" <elts:SetLiteralValues> "}" <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprSet { elts }.into()
     ),
     <location:@L> "{" <elt:NamedExpressionTest> <generators:CompFor> "}" <end_location:@R> => {
         ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprSetComp { elt: Box::new(elt), generators }.into()
         )
     },
-    <location:@L> "True" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprConstant { value: true.into(), kind: None }.into()),
+    <location:@L> "True" <end_location:@R> => ast::Expr::new(location..end_location, ast::ExprConstant { value: true.into(), kind: None }.into()),
-    <location:@L> "False" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprConstant { value: false.into(), kind: None }.into()),
+    <location:@L> "False" <end_location:@R> => ast::Expr::new(location..end_location, ast::ExprConstant { value: false.into(), kind: None }.into()),
-    <location:@L> "None" <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprConstant { value: ast::Constant::None, kind: None }.into()),
+    <location:@L> "None" <end_location:@R> => ast::Expr::new(location..end_location, ast::ExprConstant { value: ast::Constant::None, kind: None }.into()),
-    <location:@L> "..." <end_location:@R> => ast::Expr::new(location, end_location, ast::ExprConstant { value: ast::Constant::Ellipsis, kind: None }.into()),
+    <location:@L> "..." <end_location:@R> => ast::Expr::new(location..end_location, ast::ExprConstant { value: ast::Constant::Ellipsis, kind: None }.into()),
 };
 
 ListLiteralValues: Vec<ast::Expr> = {
@@ -1679,7 +1679,7 @@ GenericList<Element>: ast::Expr = {
             elts.into_iter().next().unwrap()
         } else {
             ast::Expr::new(
-                location,
+                location..
                 end_location,
                 ast::ExprTuple { elts, ctx: ast::ExprContext::Load }.into()
             )
@@ -1690,7 +1690,7 @@ GenericList<Element>: ast::Expr = {
 // Test
 StarExpr: ast::Expr = {
     <location:@L> "*" <e:Expression<"all">> <end_location:@R> => ast::Expr::new(
-        location,
+        location..
         end_location,
         ast::ExprStarred { value: Box::new(e), ctx: ast::ExprContext::Load }.into(),
     )
@@ -1725,8 +1725,7 @@ FunctionArgument: (Option<(crate::text_size::TextSize, crate::text_size::TextSiz
     <location:@L> <e:NamedExpressionTest> <c:CompFor?> <end_location:@R> => {
         let expr = match c {
             Some(c) => ast::Expr::new(
-                location,
+                location..end_location,
-                end_location,
                 ast::ExprGeneratorExp {
                     elt: Box::new(e),
                     generators: c,
@@ -1739,7 +1738,7 @@ FunctionArgument: (Option<(crate::text_size::TextSize, crate::text_size::TextSiz
     <location:@L> <i:Identifier> "=" <e:Test<"all">> <end_location:@R> => (Some((location, end_location, Some(i))), e),
     <location:@L> "*" <e:Test<"all">> <end_location:@R> => {
         let expr = ast::Expr::new(
-            location,
+            location..
             end_location,
             ast::ExprStarred { value: Box::new(e), ctx: ast::ExprContext::Load }.into(),
         );

parser/src/string.rs

@@ -10,9 +10,7 @@ use crate::{
     token::{StringKind, Tok},
 };
 use itertools::Itertools;
-use rustpython_compiler_core::{
+use rustpython_compiler_core::text_size::{TextLen, TextSize};
-    text_size::{TextLen, TextSize},
-};
 
 // unicode_name2 does not expose `MAX_NAME_LENGTH`, so we replicate that constant here, fix #3798
 const MAX_UNICODE_NAME: usize = 88;
@@ -67,7 +65,7 @@ impl<'a> StringParser<'a> {
 
     #[inline]
     fn expr(&self, node: ExprKind) -> Expr {
-        Expr::new(self.start, self.end, node)
+        Expr::new(self.start..self.end, node)
     }
 
     fn parse_unicode_literal(&mut self, literal_number: usize) -> Result<char, LexicalError> {
@@ -624,8 +622,7 @@ pub(crate) fn parse_strings(
             }
         }
         return Ok(Expr::new(
-            initial_start,
+            initial_start..last_end,
-            last_end,
             ast::ExprConstant {
                 value: Constant::Bytes(content),
                 kind: None,
@@ -648,8 +645,7 @@ pub(crate) fn parse_strings(
             }
         }
         return Ok(Expr::new(
-            initial_start,
+            initial_start..last_end,
-            last_end,
             ast::ExprConstant {
                 value: Constant::Str(content.join("")),
                 kind: initial_kind,
@@ -664,8 +660,7 @@ pub(crate) fn parse_strings(
 
     let take_current = |current: &mut Vec<String>| -> Expr {
         Expr::new(
-            initial_start,
+            initial_start..last_end,
-            last_end,
             ast::ExprConstant {
                 value: Constant::Str(current.drain(..).join("")),
                 kind: initial_kind.clone(),
@@ -696,8 +691,7 @@ pub(crate) fn parse_strings(
     }
 
     Ok(Expr::new(
-        initial_start,
+        initial_start..last_end,
-        last_end,
         ast::ExprJoinedStr { values: deduped }.into(),
     ))
 }

ruff_text_size/src/range.rs

@@ -463,6 +463,13 @@ impl RangeBounds<TextSize> for TextRange {
     }
 }
 
+impl From<Range<TextSize>> for TextRange {
+    #[inline]
+    fn from(r: Range<TextSize>) -> Self {
+        TextRange::new(r.start, r.end)
+    }
+}
+
 impl<T> From<TextRange> for Range<T>
 where
     T: From<TextSize>,

ruff_text_size/src/size.rs

@@ -63,6 +63,30 @@ impl TextSize {
     pub fn of<T: TextLen>(text: T) -> TextSize {
         text.text_len()
     }
+
+    /// Returns current raw `offset` as u32.
+    ///
+    /// # Examples
+    ///
+    /// ```rust
+    /// # use ruff_text_size::*;
+    /// assert_eq!(TextSize::from(4).to_u32(), 4);
+    /// ```
+    pub fn to_u32(&self) -> u32 {
+        self.raw
+    }
+
+    /// Returns current raw `offset` as usize.
+    ///
+    /// # Examples
+    ///
+    /// ```rust
+    /// # use ruff_text_size::*;
+    /// assert_eq!(TextSize::from(4).to_usize(), 4);
+    /// ```
+    pub fn to_usize(&self) -> usize {
+        self.raw as usize
+    }
 }
 
 /// Methods to act like a primitive integer type, where reasonably applicable.
@@ -91,7 +115,7 @@ impl From<u32> for TextSize {
 impl From<TextSize> for u32 {
     #[inline]
     fn from(value: TextSize) -> Self {
-        value.raw
+        value.to_u32()
     }
 }
 
@@ -106,7 +130,7 @@ impl TryFrom<usize> for TextSize {
 impl From<TextSize> for usize {
     #[inline]
     fn from(value: TextSize) -> Self {
-        value.raw as usize
+        value.to_usize()
     }
 }
 

scripts/update_asdl.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+cd "$(dirname "$(dirname "$0")")"
+
+python ast/asdl_rs.py --generic-file ast/src/generic.rs --located-file ast/src/located.rs --module-file ../RustPython/vm/src/stdlib/ast/gen.rs ast/Python.asdl
+rustfmt ast/src/generic.rs ast/src/located.rs ../RustPython/vm/src/stdlib/ast/gen.rs