Separate rustpython_ast_pyo3

Cargo.toml

@@ -12,6 +12,7 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
 resolver = "2"
 members = [
     "ast", "core", "format", "literal", "parser",
+    "ast-pyo3",
     "ruff_text_size", "ruff_source_location",
 ]
 

ast-pyo3/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "rustpython-ast-pyo3"
+version = "0.1.0"
+edition = "2021"
+
+[features]
+# This feature is experimental
+# It reimplements AST types, but currently both slower than python AST types and limited to use in other API
+wrapper = []
+
+[dependencies]
+rustpython-ast = { workspace = true, features = ["location"] }
+num-complex = { workspace = true }
+once_cell = { workspace = true }
+
+pyo3 = { workspace = true, features = ["num-bigint", "num-complex"] }

ast-pyo3/src/lib.rs

@@ -0,0 +1,5 @@
+mod py_ast;
+#[cfg(feature = "wrapper")]
+pub mod wrapper;
+
+pub use py_ast::{init, PyNode, ToPyAst};

ast-pyo3/src/py_ast.rs

@@ -1,4 +1,3 @@
-use crate::{source_code::SourceRange, text_size::TextRange, ConversionFlag, Node};
 use num_complex::Complex64;
 use once_cell::sync::OnceCell;
 use pyo3::{
@@ -6,8 +5,11 @@ use pyo3::{
     types::{PyBool, PyBytes, PyList, PyString, PyTuple},
     ToPyObject,
 };
+use rustpython_ast::{
+    self as ast, source_code::SourceRange, text_size::TextRange, ConversionFlag, Node,
+};
 
-pub trait Pyo3Node {
+pub trait PyNode {
     fn py_type_cache() -> &'static OnceCell<(Py<PyAny>, Py<PyAny>)> {
         {
             static PY_TYPE: OnceCell<(Py<PyAny>, Py<PyAny>)> = OnceCell::new();
@@ -16,120 +18,96 @@ pub trait Pyo3Node {
     }
 }
 
-pub trait ToPyo3Ast {
+pub trait ToPyAst {
-    fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny>;
+    fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny>;
 }
 
-impl<T: ToPyo3Ast> ToPyo3Ast for Box<T> {
+impl<T: ToPyAst> ToPyAst for Box<T> {
     #[inline]
-    fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
+    fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
-        (**self).to_pyo3_ast(py)
+        (**self).to_py_ast(py)
     }
 }
 
-impl<T: ToPyo3Ast> ToPyo3Ast for Option<T> {
+impl<T: ToPyAst> ToPyAst for Option<T> {
     #[inline]
-    fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
+    fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
         match self {
-            Some(ast) => ast.to_pyo3_ast(py),
+            Some(ast) => ast.to_py_ast(py),
             None => Ok(ast_cache().none_ref(py)),
         }
     }
 }
 
-impl<T: ToPyo3Ast> ToPyo3Ast for Vec<T> {
+impl<T: ToPyAst> ToPyAst for Vec<T> {
-    fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
+    fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
         let elts = self
             .iter()
-            .map(|item| item.to_pyo3_ast(py))
+            .map(|item| item.to_py_ast(py))
             .collect::<Result<Vec<_>, _>>()?;
         let list = PyList::new(py, elts);
         Ok(list.into())
     }
 }
 
-impl ToPyo3Ast for crate::Identifier {
+impl ToPyAst for ast::Identifier {
     #[inline]
-    fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
+    fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
         Ok(PyString::new(py, self.as_str()).into())
     }
 }
 
-impl ToPyo3Ast for crate::String {
+impl ToPyAst for ast::String {
     #[inline]
-    fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
+    fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
         Ok(PyString::new(py, self.as_str()).into())
     }
 }
 
-impl ToPyo3Ast for bool {
+impl ToPyAst for bool {
     #[inline]
-    fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
+    fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
         Ok(ast_cache().bool_int(py, *self))
     }
 }
 
-impl ToPyo3Ast for ConversionFlag {
+impl ToPyAst for ConversionFlag {
     #[inline]
-    fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
+    fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
         Ok(ast_cache().conversion_flag(py, *self))
     }
 }
 
-impl ToPyObject for crate::Constant {
+fn constant_to_object(constant: &ast::Constant, py: Python) -> PyObject {
-    fn to_object(&self, py: Python) -> PyObject {
     let cache = ast_cache();
-        match self {
+    match constant {
-            crate::Constant::None => cache.none.clone_ref(py),
+        ast::Constant::None => cache.none.clone_ref(py),
-            crate::Constant::Bool(bool) => cache.bool(py, *bool).into(),
+        ast::Constant::Bool(bool) => cache.bool(py, *bool).into(),
-            crate::Constant::Str(string) => string.to_object(py),
+        ast::Constant::Str(string) => string.to_object(py),
-            crate::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
+        ast::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
-            crate::Constant::Int(int) => int.to_object(py),
+        ast::Constant::Int(int) => int.to_object(py),
-            crate::Constant::Tuple(elts) => {
+        ast::Constant::Tuple(elts) => {
-                let elts: Vec<_> = elts.iter().map(|c| c.to_object(py)).collect();
+            let elts: Vec<_> = elts.iter().map(|c| constant_to_object(c, py)).collect();
             PyTuple::new(py, elts).into()
         }
-            crate::Constant::Float(f64) => f64.to_object(py),
+        ast::Constant::Float(f64) => f64.to_object(py),
-            crate::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
+        ast::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
-            crate::Constant::Ellipsis => py.Ellipsis(),
+        ast::Constant::Ellipsis => py.Ellipsis(),
-        }
     }
 }
 
-// impl ToPyo3Ast for crate::Constant {
-//     #[inline]
-//     fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
-//         let cache = ast_cache();
-//         let value = match self {
-//             crate::Constant::None => cache.none_ref(py),
-//             crate::Constant::Bool(bool) => cache.bool_ref(py),
-//             crate::Constant::Str(string) => string.to_object(py),
-//             crate::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
-//             crate::Constant::Int(int) => int.to_object(py),
-//             crate::Constant::Tuple(elts) => {
-//                 let elts: PyResult<Vec<_>> = elts.iter().map(|c| c.to_pyo3_ast(py)).collect();
-//                 PyTuple::new(py, elts?).into()
-//             }
-//             crate::Constant::Float(f64) => f64.to_object(py),
-//             crate::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
-//             crate::Constant::Ellipsis => py.Ellipsis(),
-//         };
-//         Ok(value)
-//     }
-// }
-
 #[pyclass(module = "rustpython_ast", subclass)]
-pub struct AST;
+pub struct Ast;
 
 #[pymethods]
-impl AST {
+impl Ast {
     #[new]
     fn new() -> Self {
         Self
     }
 }
 
-fn cache_py_type<N: Pyo3Node + Node>(ast_module: &PyAny) -> PyResult<()> {
+fn cache_py_type<N: PyNode + Node>(ast_module: &PyAny) -> PyResult<()> {
     let class = ast_module.getattr(N::NAME)?;
     let base = if std::mem::size_of::<N>() == 0 {
         class.call0()?
@@ -209,4 +187,4 @@ pub fn init(py: Python) -> PyResult<()> {
     init_types(py)
 }
 
-include!("gen/to_pyo3.rs");
+include!("gen/to_py_ast.rs");

ast-pyo3/src/wrapper.rs

@@ -0,0 +1,141 @@
+use crate::PyNode;
+use num_complex::Complex64;
+use pyo3::prelude::*;
+use pyo3::types::{PyBytes, PyList, PyTuple};
+use rustpython_ast::{
+    self as ast, source_code::SourceRange, text_size::TextRange, ConversionFlag, Node,
+};
+
+pub trait ToPyWrapper {
+    fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>>;
+}
+
+impl<T: ToPyWrapper> ToPyWrapper for Box<T> {
+    #[inline]
+    fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
+        (**self).to_py_wrapper(py)
+    }
+}
+
+impl<T: ToPyWrapper> ToPyWrapper for Option<T> {
+    #[inline]
+    fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
+        match self {
+            Some(ast) => ast.to_py_wrapper(py),
+            None => Ok(py.None()),
+        }
+    }
+}
+
+impl ToPyWrapper for ast::Identifier {
+    #[inline]
+    fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
+        Ok(self.as_str().to_object(py))
+    }
+}
+
+impl ToPyWrapper for ast::String {
+    #[inline]
+    fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
+        Ok(self.as_str().to_object(py))
+    }
+}
+
+impl ToPyWrapper for ast::Int {
+    #[inline]
+    fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
+        Ok((self.to_u32()).to_object(py))
+    }
+}
+
+impl ToPyWrapper for bool {
+    #[inline]
+    fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
+        Ok((*self as u32).to_object(py))
+    }
+}
+
+impl ToPyWrapper for ConversionFlag {
+    #[inline]
+    fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
+        Ok((*self as i8).to_object(py))
+    }
+}
+
+impl ToPyWrapper for ast::Constant {
+    fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
+        let value = match self {
+            ast::Constant::None => py.None(),
+            ast::Constant::Bool(bool) => bool.to_object(py),
+            ast::Constant::Str(string) => string.to_object(py),
+            ast::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
+            ast::Constant::Int(int) => int.to_object(py),
+            ast::Constant::Tuple(elts) => {
+                let elts: PyResult<Vec<_>> = elts.iter().map(|c| c.to_py_wrapper(py)).collect();
+                PyTuple::new(py, elts?).into()
+            }
+            ast::Constant::Float(f64) => f64.to_object(py),
+            ast::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
+            ast::Constant::Ellipsis => py.Ellipsis(),
+        };
+        Ok(value)
+    }
+}
+
+impl<T: ToPyWrapper> ToPyWrapper for Vec<T> {
+    fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
+        let list = PyList::empty(py);
+        for item in self {
+            let py_item = item.to_py_wrapper(py)?;
+            list.append(py_item)?;
+        }
+        Ok(list.into())
+    }
+}
+
+#[pyclass(module = "rustpython_ast", subclass)]
+pub struct Ast;
+
+#[pymethods]
+impl Ast {
+    #[new]
+    fn new() -> Self {
+        Self
+    }
+}
+
+pub mod located {
+    pub use super::Ast;
+    use super::*;
+    include!("gen/wrapper_located.rs");
+}
+
+pub mod ranged {
+    pub use super::Ast;
+    use super::*;
+    include!("gen/wrapper_ranged.rs");
+}
+
+fn init_type<P: pyo3::PyClass, N: PyNode + Node>(py: Python, m: &PyModule) -> PyResult<()> {
+    m.add_class::<P>()?;
+    let node = m.getattr(P::NAME)?;
+    if P::NAME != N::NAME {
+        // TODO: no idea how to escape rust keyword on #[pyclass]
+        m.setattr(P::NAME, node)?;
+    }
+    let names: Vec<&'static str> = N::FIELD_NAMES.to_vec();
+    let fields = PyTuple::new(py, names);
+    node.setattr("_fields", fields)?;
+    Ok(())
+}
+
+/// A Python module implemented in Rust.
+fn init_module(py: Python, m: &PyModule) -> PyResult<()> {
+    m.add_class::<Ast>()?;
+
+    let ast = m.getattr("AST")?;
+    let fields = PyTuple::empty(py);
+    ast.setattr("_fields", fields)?;
+
+    Ok(())
+}

ast/Cargo.toml

@@ -15,11 +15,6 @@ fold = []
 unparse = ["rustpython-literal"]
 visitor = []
 all-nodes-with-ranges = []
-pyo3 = ["dep:pyo3", "num-complex", "once_cell"]
-
-# This feature is experimental
-# It reimplements AST types, but currently both slower than python AST types and limited to use in other API
-pyo3-wrapper = ["pyo3"]
 
 [dependencies]
 rustpython-parser-core = { workspace = true }
@@ -28,6 +23,5 @@ rustpython-literal = { workspace = true, optional = true }
 is-macro = { workspace = true }
 num-bigint = { workspace = true }
 static_assertions = "1.1.0"
-num-complex = { workspace = true, optional = true }
+
-once_cell = { workspace = true, optional = true }
 pyo3 = { workspace = true, optional = true, features = ["num-bigint", "num-complex"] }

ast/asdl_rs.py

@@ -1102,16 +1102,16 @@ class ToPyo3AstVisitor(EmitVisitor):
 
         self.emit(
             f"""
-            impl ToPyo3Ast for crate::generic::{rust_name}{self.generics} {{
+            impl ToPyAst for ast::{rust_name}{self.generics} {{
                 #[inline]
-                fn to_pyo3_ast<'py>(&self, {"_" if simple else ""}py: Python<'py>) -> PyResult<&'py PyAny> {{
+                fn to_py_ast<'py>(&self, {"_" if simple else ""}py: Python<'py>) -> PyResult<&'py PyAny> {{
                     let instance = match &self {{
             """,
             0,
         )
         for cons in sum.types:
             self.emit(
-                f"crate::{rust_name}::{cons.name}(cons) => cons.to_pyo3_ast(py)?,",
+                f"ast::{rust_name}::{cons.name}(cons) => cons.to_py_ast(py)?,",
                 1,
             )
         self.emit(
@@ -1135,9 +1135,9 @@ class ToPyo3AstVisitor(EmitVisitor):
         type_info = self.type_info[type.name]
         self.emit(
             f"""
-            impl ToPyo3Ast for crate::{name}{self.generics} {{
+            impl ToPyAst for ast::{name}{self.generics} {{
                 #[inline]
-                    fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {{
+                    fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {{
                         let cache = Self::py_type_cache().get().unwrap();
             """,
             0,
@@ -1159,7 +1159,7 @@ class ToPyo3AstVisitor(EmitVisitor):
             for field in cons.fields:
                 if field.type == "constant":
                     self.emit(
-                        f"{rust_field(field.name)}.to_object(py),",
+                        f"constant_to_object({rust_field(field.name)}, py),",
                         3,
                     )
                     continue
@@ -1183,7 +1183,7 @@ class ToPyo3AstVisitor(EmitVisitor):
                         )
                         continue
                 self.emit(
-                    f"{rust_field(field.name)}.to_pyo3_ast(py)?,",
+                    f"{rust_field(field.name)}.to_py_ast(py)?,",
                     3,
                 )
             self.emit(
@@ -1252,7 +1252,7 @@ class Pyo3StructVisitor(EmitVisitor):
     def ref(self):
         return "&" if self.borrow else ""
 
-    def emit_class(self, name, rust_name, simple, base="super::AST"):
+    def emit_class(self, name, rust_name, simple, base="super::Ast"):
         info = self.type_info[name]
         if simple:
             generics = ""
@@ -1264,7 +1264,7 @@ class Pyo3StructVisitor(EmitVisitor):
             into = f"{rust_name}"
         else:
             subclass = ""
-            body = f"(pub {self.ref_def} crate::{rust_name}{generics})"
+            body = f"(pub {self.ref_def} ast::{rust_name}{generics})"
             into = f"{rust_name}(node)"
 
         self.emit(
@@ -1273,8 +1273,8 @@ class Pyo3StructVisitor(EmitVisitor):
             #[derive(Clone, Debug)]
             pub struct {rust_name} {body};
 
-            impl From<{self.ref_def} crate::{rust_name}{generics}> for {rust_name} {{
+            impl From<{self.ref_def} ast::{rust_name}{generics}> for {rust_name} {{
-                fn from({"" if body else "_"}node: {self.ref_def} crate::{rust_name}{generics}) -> Self {{
+                fn from({"" if body else "_"}node: {self.ref_def} ast::{rust_name}{generics}) -> Self {{
                     {into}
                 }}
             }}
@@ -1288,7 +1288,7 @@ class Pyo3StructVisitor(EmitVisitor):
                 impl {rust_name} {{
                     #[new]
                     fn new() -> PyClassInitializer<Self> {{
-                        PyClassInitializer::from(AST)
+                        PyClassInitializer::from(Ast)
                             .add_subclass(Self)
                     }}
 
@@ -1303,7 +1303,7 @@ class Pyo3StructVisitor(EmitVisitor):
                 0,
             )
         else:
-            if base != "super::AST":
+            if base != "super::Ast":
                 add_subclass = f".add_subclass({base})"
             else:
                 add_subclass = ""
@@ -1311,7 +1311,7 @@ class Pyo3StructVisitor(EmitVisitor):
                 f"""
                 impl ToPyObject for {rust_name} {{
                     fn to_object(&self, py: Python) -> PyObject {{
-                        let initializer = PyClassInitializer::from(AST)
+                        let initializer = PyClassInitializer::from(Ast)
                         {add_subclass}
                         .add_subclass(self.clone());
                         Py::new(py, initializer).unwrap().into_py(py)
@@ -1339,7 +1339,7 @@ class Pyo3StructVisitor(EmitVisitor):
                 #[getter]
                 #[inline]
                 fn get_{field.name}(&self, py: Python) -> PyResult<PyObject> {{
-                    self.0.{rust_field(field.name)}.to_pyo3_wrapper(py)
+                    self.0.{rust_field(field.name)}.to_py_wrapper(py)
                 }}
                 """,
                 3,
@@ -1365,7 +1365,7 @@ class Pyo3StructVisitor(EmitVisitor):
 
         for field in owner.fields:
             self.emit(
-                f'"{field.name}" => self.0.{rust_field(field.name)}.to_pyo3_wrapper(py)?,',
+                f'"{field.name}" => self.0.{rust_field(field.name)}.to_py_wrapper(py)?,',
                 3,
             )
 
@@ -1383,9 +1383,9 @@ class Pyo3StructVisitor(EmitVisitor):
     def emit_wrapper(self, rust_name):
         self.emit(
             f"""
-            impl ToPyo3Wrapper for crate::{rust_name}{self.generics} {{
+            impl ToPyWrapper for ast::{rust_name}{self.generics} {{
                 #[inline]
-                fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
+                fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
                     Ok({rust_name}(self).to_object(py))
                 }}
             }}
@@ -1409,16 +1409,16 @@ class Pyo3StructVisitor(EmitVisitor):
         if not simple:
             self.emit(
                 f"""
-                impl ToPyo3Wrapper for crate::{rust_name}{self.generics} {{
+                impl ToPyWrapper for ast::{rust_name}{self.generics} {{
                     #[inline]
-                    fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
+                    fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
                         match &self {{
                 """,
                 0,
             )
 
             for cons in sum.types:
-                self.emit(f"Self::{cons.name}(cons) => cons.to_pyo3_wrapper(py),", 3)
+                self.emit(f"Self::{cons.name}(cons) => cons.to_py_wrapper(py),", 3)
 
             self.emit(
                 """
@@ -1447,7 +1447,7 @@ class Pyo3StructVisitor(EmitVisitor):
 
                 impl ToPyObject for {parent}{cons.name} {{
                     fn to_object(&self, py: Python) -> PyObject {{
-                        let initializer = PyClassInitializer::from(AST)
+                        let initializer = PyClassInitializer::from(Ast)
                         .add_subclass({parent})
                         .add_subclass(Self);
                         Py::new(py, initializer).unwrap().into_py(py)
@@ -1496,9 +1496,7 @@ class Pyo3PymoduleVisitor(EmitVisitor):
         self.emit_fields(cons.name, rust_name, simple)
 
     def emit_fields(self, name, rust_name, simple):
-        self.emit(
+        self.emit(f"super::init_type::<{rust_name}, ast::{rust_name}>(py, m)?;", 1)
-            f"super::init_type::<{rust_name}, crate::generic::{rust_name}>(py, m)?;", 1
-        )
 
 
 class StdlibClassDefVisitor(EmitVisitor):
@@ -1846,7 +1844,7 @@ def write_pyo3_node(type_info, f):
 
         f.write(
             f"""
-            impl{generics} Pyo3Node for crate::generic::{rust_name}{generics} {{
+            impl{generics} PyNode for ast::{rust_name}{generics} {{
                 #[inline]
                 fn py_type_cache() -> &'static OnceCell<(Py<PyAny>, Py<PyAny>)> {{
                     static PY_TYPE: OnceCell<(Py<PyAny>, Py<PyAny>)> = OnceCell::new();
@@ -1876,7 +1874,7 @@ def write_to_pyo3(mod, type_info, f):
 
     for info in type_info.values():
         rust_name = info.rust_sum_name
-        f.write(f"cache_py_type::<crate::generic::{rust_name}>(ast_module)?;\n")
+        f.write(f"cache_py_type::<ast::{rust_name}>(ast_module)?;\n")
     f.write("Ok(())\n}")
 
 
@@ -1890,15 +1888,15 @@ def write_to_pyo3_simple(type_info, f):
         rust_name = type_info.rust_sum_name
         f.write(
             f"""
-            impl ToPyo3Ast for crate::generic::{rust_name} {{
+            impl ToPyAst for ast::{rust_name} {{
                 #[inline]
-                fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {{
+                fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {{
                     let cell = match &self {{
             """,
         )
         for cons in type_info.type.value.types:
             f.write(
-                f"""crate::{rust_name}::{cons.name} => crate::{rust_name}{cons.name}::py_type_cache(),""",
+                f"""ast::{rust_name}::{cons.name} => ast::{rust_name}{cons.name}::py_type_cache(),""",
             )
         f.write(
             """
@@ -1921,9 +1919,9 @@ def write_pyo3_wrapper(mod, type_info, namespace, f):
             rust_name = type_info.rust_sum_name
             f.write(
                 f"""
-                impl ToPyo3Wrapper for crate::generic::{rust_name} {{
+                impl ToPyWrapper for ast::{rust_name} {{
                     #[inline]
-                    fn to_pyo3_wrapper(&self, py: Python) -> PyResult<Py<PyAny>> {{
+                    fn to_py_wrapper(&self, py: Python) -> PyResult<Py<PyAny>> {{
                         match &self {{
                 """,
             )
@@ -1942,9 +1940,9 @@ def write_pyo3_wrapper(mod, type_info, namespace, f):
             for cons in type_info.type.value.types:
                 f.write(
                     f"""
-                    impl ToPyo3Wrapper for crate::generic::{rust_name}{cons.name} {{
+                    impl ToPyWrapper for ast::{rust_name}{cons.name} {{
                         #[inline]
-                        fn to_pyo3_wrapper(&self, py: Python) -> PyResult<Py<PyAny>> {{
+                        fn to_py_wrapper(&self, py: Python) -> PyResult<Py<PyAny>> {{
                             Ok({rust_name}{cons.name}.to_object(py))
                         }}
                     }}
@@ -1983,6 +1981,7 @@ def write_ast_mod(mod, type_info, f):
 def main(
     input_filename,
     ast_dir,
+    ast_pyo3_dir,
     module_filename,
     dump_module=False,
 ):
@@ -2005,14 +2004,20 @@ def main(
         ("ranged", p(write_ranged_def, mod, type_info)),
         ("located", p(write_located_def, mod, type_info)),
         ("visitor", p(write_visitor_def, mod, type_info)),
-        ("to_pyo3", p(write_to_pyo3, mod, type_info)),
-        ("pyo3_wrapper_located", p(write_pyo3_wrapper, mod, type_info, "located")),
-        ("pyo3_wrapper_ranged", p(write_pyo3_wrapper, mod, type_info, "ranged")),
     ]:
         with (ast_dir / f"{filename}.rs").open("w") as f:
             f.write(auto_gen_msg)
             write(f)
 
+    for filename, write in [
+        ("to_py_ast", p(write_to_pyo3, mod, type_info)),
+        ("wrapper_located", p(write_pyo3_wrapper, mod, type_info, "located")),
+        ("wrapper_ranged", p(write_pyo3_wrapper, mod, type_info, "ranged")),
+    ]:
+        with (ast_pyo3_dir / f"{filename}.rs").open("w") as f:
+            f.write(auto_gen_msg)
+            write(f)
+
     with module_filename.open("w") as module_file:
         module_file.write(auto_gen_msg)
         write_ast_mod(mod, type_info, module_file)
@@ -2024,6 +2029,7 @@ if __name__ == "__main__":
     parser = ArgumentParser()
     parser.add_argument("input_file", type=Path)
     parser.add_argument("-A", "--ast-dir", type=Path, required=True)
+    parser.add_argument("-O", "--ast-pyo3-dir", type=Path, required=True)
     parser.add_argument("-M", "--module-file", type=Path, required=True)
     parser.add_argument("-d", "--dump-module", action="store_true")
 
@@ -2031,6 +2037,7 @@ if __name__ == "__main__":
     main(
         args.input_file,
         args.ast_dir,
+        args.ast_pyo3_dir,
         args.module_file,
         args.dump_module,
     )

ast/src/lib.rs

@@ -41,8 +41,3 @@ pub use visitor::Visitor;
 mod optimizer;
 #[cfg(feature = "constant-optimization")]
 pub use optimizer::ConstantOptimizer;
-
-#[cfg(feature = "pyo3")]
-pub mod pyo3;
-#[cfg(feature = "pyo3-wrapper")]
-pub mod pyo3_wrapper;

ast/src/pyo3_wrapper.rs

@@ -1,128 +0,0 @@
-use crate::pyo3::{Pyo3Node, AST};
-use crate::{source_code::SourceRange, text_size::TextRange, ConversionFlag, Node};
-use num_complex::Complex64;
-use pyo3::prelude::*;
-use pyo3::types::{PyBytes, PyList, PyTuple};
-
-pub trait ToPyo3Wrapper {
-    fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>>;
-}
-
-impl<T: ToPyo3Wrapper> ToPyo3Wrapper for Box<T> {
-    #[inline]
-    fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
-        (**self).to_pyo3_wrapper(py)
-    }
-}
-
-impl<T: ToPyo3Wrapper> ToPyo3Wrapper for Option<T> {
-    #[inline]
-    fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
-        match self {
-            Some(ast) => ast.to_pyo3_wrapper(py),
-            None => Ok(py.None()),
-        }
-    }
-}
-
-impl ToPyo3Wrapper for crate::Identifier {
-    #[inline]
-    fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
-        Ok(self.as_str().to_object(py))
-    }
-}
-
-impl ToPyo3Wrapper for crate::String {
-    #[inline]
-    fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
-        Ok(self.as_str().to_object(py))
-    }
-}
-
-impl ToPyo3Wrapper for crate::Int {
-    #[inline]
-    fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
-        Ok((self.to_u32()).to_object(py))
-    }
-}
-
-impl ToPyo3Wrapper for bool {
-    #[inline]
-    fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
-        Ok((*self as u32).to_object(py))
-    }
-}
-
-impl ToPyo3Wrapper for ConversionFlag {
-    #[inline]
-    fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
-        Ok((*self as i8).to_object(py))
-    }
-}
-
-impl ToPyo3Wrapper for crate::Constant {
-    fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
-        let value = match self {
-            crate::Constant::None => py.None(),
-            crate::Constant::Bool(bool) => bool.to_object(py),
-            crate::Constant::Str(string) => string.to_object(py),
-            crate::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
-            crate::Constant::Int(int) => int.to_object(py),
-            crate::Constant::Tuple(elts) => {
-                let elts: PyResult<Vec<_>> = elts.iter().map(|c| c.to_pyo3_wrapper(py)).collect();
-                PyTuple::new(py, elts?).into()
-            }
-            crate::Constant::Float(f64) => f64.to_object(py),
-            crate::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
-            crate::Constant::Ellipsis => py.Ellipsis(),
-        };
-        Ok(value)
-    }
-}
-
-impl<T: ToPyo3Wrapper> ToPyo3Wrapper for Vec<T> {
-    fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
-        let list = PyList::empty(py);
-        for item in self {
-            let py_item = item.to_pyo3_wrapper(py)?;
-            list.append(py_item)?;
-        }
-        Ok(list.into())
-    }
-}
-
-pub mod located {
-    use super::*;
-    pub use crate::pyo3::AST;
-    include!("gen/pyo3_wrapper_located.rs");
-}
-
-pub mod ranged {
-    use super::*;
-    pub use crate::pyo3::AST;
-    include!("gen/pyo3_wrapper_ranged.rs");
-}
-
-fn init_type<P: pyo3::PyClass, N: Pyo3Node + Node>(py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_class::<P>()?;
-    let node = m.getattr(P::NAME)?;
-    if P::NAME != N::NAME {
-        // TODO: no idea how to escape rust keyword on #[pyclass]
-        m.setattr(P::NAME, node)?;
-    }
-    let names: Vec<&'static str> = N::FIELD_NAMES.to_vec();
-    let fields = PyTuple::new(py, names);
-    node.setattr("_fields", fields)?;
-    Ok(())
-}
-
-/// A Python module implemented in Rust.
-fn init_module(py: Python, m: &PyModule) -> PyResult<()> {
-    m.add_class::<AST>()?;
-
-    let ast = m.getattr("AST")?;
-    let fields = PyTuple::empty(py);
-    ast.setattr("_fields", fields)?;
-
-    Ok(())
-}

scripts/update_asdl.sh

@@ -4,5 +4,5 @@ set -e
 cd "$(dirname "$(dirname "$0")")"
 
 # rm ast/src/gen/*.rs
-python ast/asdl_rs.py --ast-dir ast/src/gen/ --module-file ../RustPython/vm/src/stdlib/ast/gen.rs ast/Python.asdl
+python ast/asdl_rs.py --ast-dir ast/src/gen/ --ast-pyo3-dir ast-pyo3/src/gen/ --module-file ../RustPython/vm/src/stdlib/ast/gen.rs ast/Python.asdl
-rustfmt ast/src/gen/*.rs ../RustPython/vm/src/stdlib/ast/gen.rs
+rustfmt ast/src/gen/*.rs ast-pyo3/src/gen/*.rs ../RustPython/vm/src/stdlib/ast/gen.rs