mirrors/RustPython-Parser
1
0
Fork 0
mirror of https://github.com/RustPython/Parser.git synced 2025-07-07 21:25:31 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Separate rustpython_ast_pyo3

Browse source
This commit is contained in:
Jeong YunWon 2023-05-23 02:24:03 +09:00
parent b81273e9bc
commit 7d384d88d0
13 changed files with 3209 additions and 3208 deletions
Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -12,6 +12,7 @@ include = ["LICENSE", "Cargo.toml", "src/**/*.rs"]
resolver = "2"
members = [
"ast", "core", "format", "literal", "parser",
"ast-pyo3",
"ruff_text_size", "ruff_source_location",
]

16
ast-pyo3/Cargo.toml Normal file
View file

@ -0,0 +1,16 @@
[package]
name = "rustpython-ast-pyo3"
version = "0.1.0"
edition = "2021"
[features]
# This feature is experimental
# It reimplements AST types, but currently both slower than python AST types and limited to use in other API
wrapper = []
[dependencies]
rustpython-ast = { workspace = true, features = ["location"] }
num-complex = { workspace = true }
once_cell = { workspace = true }
pyo3 = { workspace = true, features = ["num-bigint", "num-complex"] }

2136
ast/src/gen/to_pyo3.rs → ast-pyo3/src/gen/to_py_ast.rs
View file

File diff suppressed because it is too large Load diff

1968
ast/src/gen/pyo3_wrapper_located.rs → ast-pyo3/src/gen/wrapper_located.rs
View file

File diff suppressed because it is too large Load diff

1820
ast/src/gen/pyo3_wrapper_ranged.rs → ast-pyo3/src/gen/wrapper_ranged.rs
View file

File diff suppressed because it is too large Load diff

5
ast-pyo3/src/lib.rs Normal file
View file

@ -0,0 +1,5 @@
mod py_ast;
#[cfg(feature = "wrapper")]
pub mod wrapper;
pub use py_ast::{init, PyNode, ToPyAst};

104
ast/src/pyo3.rs → ast-pyo3/src/py_ast.rs
View file

@ -1,4 +1,3 @@
use crate::{source_code::SourceRange, text_size::TextRange, ConversionFlag, Node};
use num_complex::Complex64;
use once_cell::sync::OnceCell;
use pyo3::{
@ -6,8 +5,11 @@ use pyo3::{
types::{PyBool, PyBytes, PyList, PyString, PyTuple},
ToPyObject,
};
use rustpython_ast::{
self as ast, source_code::SourceRange, text_size::TextRange, ConversionFlag, Node,
};
pub trait Pyo3Node {
pub trait PyNode {
fn py_type_cache() -> &'static OnceCell<(Py<PyAny>, Py<PyAny>)> {
{
static PY_TYPE: OnceCell<(Py<PyAny>, Py<PyAny>)> = OnceCell::new();
@ -16,120 +18,96 @@ pub trait Pyo3Node {
}
}
pub trait ToPyo3Ast {
fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny>;
pub trait ToPyAst {
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny>;
}
impl<T: ToPyo3Ast> ToPyo3Ast for Box<T> {
impl<T: ToPyAst> ToPyAst for Box<T> {
#[inline]
fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
(**self).to_pyo3_ast(py)
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
(**self).to_py_ast(py)
}
}
impl<T: ToPyo3Ast> ToPyo3Ast for Option<T> {
impl<T: ToPyAst> ToPyAst for Option<T> {
#[inline]
fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
match self {
Some(ast) => ast.to_pyo3_ast(py),
Some(ast) => ast.to_py_ast(py),
None => Ok(ast_cache().none_ref(py)),
}
}
}
impl<T: ToPyo3Ast> ToPyo3Ast for Vec<T> {
fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
impl<T: ToPyAst> ToPyAst for Vec<T> {
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
let elts = self
.iter()
.map(|item| item.to_pyo3_ast(py))
.map(|item| item.to_py_ast(py))
.collect::<Result<Vec<_>, _>>()?;
let list = PyList::new(py, elts);
Ok(list.into())
}
}
impl ToPyo3Ast for crate::Identifier {
impl ToPyAst for ast::Identifier {
#[inline]
fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
Ok(PyString::new(py, self.as_str()).into())
}
}
impl ToPyo3Ast for crate::String {
impl ToPyAst for ast::String {
#[inline]
fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
Ok(PyString::new(py, self.as_str()).into())
}
}
impl ToPyo3Ast for bool {
impl ToPyAst for bool {
#[inline]
fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
Ok(ast_cache().bool_int(py, *self))
}
}
impl ToPyo3Ast for ConversionFlag {
impl ToPyAst for ConversionFlag {
#[inline]
fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
Ok(ast_cache().conversion_flag(py, *self))
}
}
impl ToPyObject for crate::Constant {
fn to_object(&self, py: Python) -> PyObject {
let cache = ast_cache();
match self {
crate::Constant::None => cache.none.clone_ref(py),
crate::Constant::Bool(bool) => cache.bool(py, *bool).into(),
crate::Constant::Str(string) => string.to_object(py),
crate::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
crate::Constant::Int(int) => int.to_object(py),
crate::Constant::Tuple(elts) => {
let elts: Vec<_> = elts.iter().map(|c| c.to_object(py)).collect();
PyTuple::new(py, elts).into()
}
crate::Constant::Float(f64) => f64.to_object(py),
crate::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
crate::Constant::Ellipsis => py.Ellipsis(),
fn constant_to_object(constant: &ast::Constant, py: Python) -> PyObject {
let cache = ast_cache();
match constant {
ast::Constant::None => cache.none.clone_ref(py),
ast::Constant::Bool(bool) => cache.bool(py, *bool).into(),
ast::Constant::Str(string) => string.to_object(py),
ast::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
ast::Constant::Int(int) => int.to_object(py),
ast::Constant::Tuple(elts) => {
let elts: Vec<_> = elts.iter().map(|c| constant_to_object(c, py)).collect();
PyTuple::new(py, elts).into()
}
ast::Constant::Float(f64) => f64.to_object(py),
ast::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
ast::Constant::Ellipsis => py.Ellipsis(),
}
}
// impl ToPyo3Ast for crate::Constant {
// #[inline]
// fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {
// let cache = ast_cache();
// let value = match self {
// crate::Constant::None => cache.none_ref(py),
// crate::Constant::Bool(bool) => cache.bool_ref(py),
// crate::Constant::Str(string) => string.to_object(py),
// crate::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
// crate::Constant::Int(int) => int.to_object(py),
// crate::Constant::Tuple(elts) => {
// let elts: PyResult<Vec<_>> = elts.iter().map(|c| c.to_pyo3_ast(py)).collect();
// PyTuple::new(py, elts?).into()
// }
// crate::Constant::Float(f64) => f64.to_object(py),
// crate::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
// crate::Constant::Ellipsis => py.Ellipsis(),
// };
// Ok(value)
// }
// }
#[pyclass(module = "rustpython_ast", subclass)]
pub struct AST;
pub struct Ast;
#[pymethods]
impl AST {
impl Ast {
#[new]
fn new() -> Self {
Self
}
}
fn cache_py_type<N: Pyo3Node + Node>(ast_module: &PyAny) -> PyResult<()> {
fn cache_py_type<N: PyNode + Node>(ast_module: &PyAny) -> PyResult<()> {
let class = ast_module.getattr(N::NAME)?;
let base = if std::mem::size_of::<N>() == 0 {
class.call0()?
@ -209,4 +187,4 @@ pub fn init(py: Python) -> PyResult<()> {
init_types(py)
}
include!("gen/to_pyo3.rs");
include!("gen/to_py_ast.rs");

141
ast-pyo3/src/wrapper.rs Normal file
View file

@ -0,0 +1,141 @@
use crate::PyNode;
use num_complex::Complex64;
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyList, PyTuple};
use rustpython_ast::{
self as ast, source_code::SourceRange, text_size::TextRange, ConversionFlag, Node,
};
pub trait ToPyWrapper {
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>>;
}
impl<T: ToPyWrapper> ToPyWrapper for Box<T> {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
(**self).to_py_wrapper(py)
}
}
impl<T: ToPyWrapper> ToPyWrapper for Option<T> {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
match self {
Some(ast) => ast.to_py_wrapper(py),
None => Ok(py.None()),
}
}
}
impl ToPyWrapper for ast::Identifier {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyWrapper for ast::String {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyWrapper for ast::Int {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((self.to_u32()).to_object(py))
}
}
impl ToPyWrapper for bool {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as u32).to_object(py))
}
}
impl ToPyWrapper for ConversionFlag {
#[inline]
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as i8).to_object(py))
}
}
impl ToPyWrapper for ast::Constant {
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let value = match self {
ast::Constant::None => py.None(),
ast::Constant::Bool(bool) => bool.to_object(py),
ast::Constant::Str(string) => string.to_object(py),
ast::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
ast::Constant::Int(int) => int.to_object(py),
ast::Constant::Tuple(elts) => {
let elts: PyResult<Vec<_>> = elts.iter().map(|c| c.to_py_wrapper(py)).collect();
PyTuple::new(py, elts?).into()
}
ast::Constant::Float(f64) => f64.to_object(py),
ast::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
ast::Constant::Ellipsis => py.Ellipsis(),
};
Ok(value)
}
}
impl<T: ToPyWrapper> ToPyWrapper for Vec<T> {
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_py_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
#[pyclass(module = "rustpython_ast", subclass)]
pub struct Ast;
#[pymethods]
impl Ast {
#[new]
fn new() -> Self {
Self
}
}
pub mod located {
pub use super::Ast;
use super::*;
include!("gen/wrapper_located.rs");
}
pub mod ranged {
pub use super::Ast;
use super::*;
include!("gen/wrapper_ranged.rs");
}
fn init_type<P: pyo3::PyClass, N: PyNode + Node>(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<P>()?;
let node = m.getattr(P::NAME)?;
if P::NAME != N::NAME {
// TODO: no idea how to escape rust keyword on #[pyclass]
m.setattr(P::NAME, node)?;
}
let names: Vec<&'static str> = N::FIELD_NAMES.to_vec();
let fields = PyTuple::new(py, names);
node.setattr("_fields", fields)?;
Ok(())
}
/// A Python module implemented in Rust.
fn init_module(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<Ast>()?;
let ast = m.getattr("AST")?;
let fields = PyTuple::empty(py);
ast.setattr("_fields", fields)?;
Ok(())
}

8
ast/Cargo.toml
View file

@ -15,11 +15,6 @@ fold = []
unparse = ["rustpython-literal"]
visitor = []
all-nodes-with-ranges = []
pyo3 = ["dep:pyo3", "num-complex", "once_cell"]
# This feature is experimental
# It reimplements AST types, but currently both slower than python AST types and limited to use in other API
pyo3-wrapper = ["pyo3"]
[dependencies]
rustpython-parser-core = { workspace = true }
@ -28,6 +23,5 @@ rustpython-literal = { workspace = true, optional = true }
is-macro = { workspace = true }
num-bigint = { workspace = true }
static_assertions = "1.1.0"
num-complex = { workspace = true, optional = true }
once_cell = { workspace = true, optional = true }
pyo3 = { workspace = true, optional = true, features = ["num-bigint", "num-complex"] }

81
ast/asdl_rs.py
View file

@ -1102,16 +1102,16 @@ class ToPyo3AstVisitor(EmitVisitor):
self.emit(
f"""
impl ToPyo3Ast for crate::generic::{rust_name}{self.generics} {{
impl ToPyAst for ast::{rust_name}{self.generics} {{
#[inline]
fn to_pyo3_ast<'py>(&self, {"_" if simple else ""}py: Python<'py>) -> PyResult<&'py PyAny> {{
fn to_py_ast<'py>(&self, {"_" if simple else ""}py: Python<'py>) -> PyResult<&'py PyAny> {{
let instance = match &self {{
""",
0,
)
for cons in sum.types:
self.emit(
f"crate::{rust_name}::{cons.name}(cons) => cons.to_pyo3_ast(py)?,",
f"ast::{rust_name}::{cons.name}(cons) => cons.to_py_ast(py)?,",
1,
)
self.emit(
@ -1135,9 +1135,9 @@ class ToPyo3AstVisitor(EmitVisitor):
type_info = self.type_info[type.name]
self.emit(
f"""
impl ToPyo3Ast for crate::{name}{self.generics} {{
impl ToPyAst for ast::{name}{self.generics} {{
#[inline]
fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {{
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {{
let cache = Self::py_type_cache().get().unwrap();
""",
0,
@ -1159,7 +1159,7 @@ class ToPyo3AstVisitor(EmitVisitor):
for field in cons.fields:
if field.type == "constant":
self.emit(
f"{rust_field(field.name)}.to_object(py),",
f"constant_to_object({rust_field(field.name)}, py),",
3,
)
continue
@ -1183,7 +1183,7 @@ class ToPyo3AstVisitor(EmitVisitor):
)
continue
self.emit(
f"{rust_field(field.name)}.to_pyo3_ast(py)?,",
f"{rust_field(field.name)}.to_py_ast(py)?,",
3,
)
self.emit(
@ -1252,7 +1252,7 @@ class Pyo3StructVisitor(EmitVisitor):
def ref(self):
return "&" if self.borrow else ""
def emit_class(self, name, rust_name, simple, base="super::AST"):
def emit_class(self, name, rust_name, simple, base="super::Ast"):
info = self.type_info[name]
if simple:
generics = ""
@ -1264,7 +1264,7 @@ class Pyo3StructVisitor(EmitVisitor):
into = f"{rust_name}"
else:
subclass = ""
body = f"(pub {self.ref_def} crate::{rust_name}{generics})"
body = f"(pub {self.ref_def} ast::{rust_name}{generics})"
into = f"{rust_name}(node)"
self.emit(
@ -1273,8 +1273,8 @@ class Pyo3StructVisitor(EmitVisitor):
#[derive(Clone, Debug)]
pub struct {rust_name} {body};
impl From<{self.ref_def} crate::{rust_name}{generics}> for {rust_name} {{
fn from({"" if body else "_"}node: {self.ref_def} crate::{rust_name}{generics}) -> Self {{
impl From<{self.ref_def} ast::{rust_name}{generics}> for {rust_name} {{
fn from({"" if body else "_"}node: {self.ref_def} ast::{rust_name}{generics}) -> Self {{
{into}
}}
}}
@ -1288,7 +1288,7 @@ class Pyo3StructVisitor(EmitVisitor):
impl {rust_name} {{
#[new]
fn new() -> PyClassInitializer<Self> {{
PyClassInitializer::from(AST)
PyClassInitializer::from(Ast)
.add_subclass(Self)
}}
@ -1303,7 +1303,7 @@ class Pyo3StructVisitor(EmitVisitor):
0,
)
else:
if base != "super::AST":
if base != "super::Ast":
add_subclass = f".add_subclass({base})"
else:
add_subclass = ""
@ -1311,7 +1311,7 @@ class Pyo3StructVisitor(EmitVisitor):
f"""
impl ToPyObject for {rust_name} {{
fn to_object(&self, py: Python) -> PyObject {{
let initializer = PyClassInitializer::from(AST)
let initializer = PyClassInitializer::from(Ast)
{add_subclass}
.add_subclass(self.clone());
Py::new(py, initializer).unwrap().into_py(py)
@ -1339,7 +1339,7 @@ class Pyo3StructVisitor(EmitVisitor):
#[getter]
#[inline]
fn get_{field.name}(&self, py: Python) -> PyResult<PyObject> {{
self.0.{rust_field(field.name)}.to_pyo3_wrapper(py)
self.0.{rust_field(field.name)}.to_py_wrapper(py)
}}
""",
3,
@ -1365,7 +1365,7 @@ class Pyo3StructVisitor(EmitVisitor):
for field in owner.fields:
self.emit(
f'"{field.name}" => self.0.{rust_field(field.name)}.to_pyo3_wrapper(py)?,',
f'"{field.name}" => self.0.{rust_field(field.name)}.to_py_wrapper(py)?,',
3,
)
@ -1383,9 +1383,9 @@ class Pyo3StructVisitor(EmitVisitor):
def emit_wrapper(self, rust_name):
self.emit(
f"""
impl ToPyo3Wrapper for crate::{rust_name}{self.generics} {{
impl ToPyWrapper for ast::{rust_name}{self.generics} {{
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
Ok({rust_name}(self).to_object(py))
}}
}}
@ -1409,16 +1409,16 @@ class Pyo3StructVisitor(EmitVisitor):
if not simple:
self.emit(
f"""
impl ToPyo3Wrapper for crate::{rust_name}{self.generics} {{
impl ToPyWrapper for ast::{rust_name}{self.generics} {{
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
fn to_py_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
match &self {{
""",
0,
)
for cons in sum.types:
self.emit(f"Self::{cons.name}(cons) => cons.to_pyo3_wrapper(py),", 3)
self.emit(f"Self::{cons.name}(cons) => cons.to_py_wrapper(py),", 3)
self.emit(
"""
@ -1447,7 +1447,7 @@ class Pyo3StructVisitor(EmitVisitor):
impl ToPyObject for {parent}{cons.name} {{
fn to_object(&self, py: Python) -> PyObject {{
let initializer = PyClassInitializer::from(AST)
let initializer = PyClassInitializer::from(Ast)
.add_subclass({parent})
.add_subclass(Self);
Py::new(py, initializer).unwrap().into_py(py)
@ -1496,9 +1496,7 @@ class Pyo3PymoduleVisitor(EmitVisitor):
self.emit_fields(cons.name, rust_name, simple)
def emit_fields(self, name, rust_name, simple):
self.emit(
f"super::init_type::<{rust_name}, crate::generic::{rust_name}>(py, m)?;", 1
)
self.emit(f"super::init_type::<{rust_name}, ast::{rust_name}>(py, m)?;", 1)
class StdlibClassDefVisitor(EmitVisitor):
@ -1846,7 +1844,7 @@ def write_pyo3_node(type_info, f):
f.write(
f"""
impl{generics} Pyo3Node for crate::generic::{rust_name}{generics} {{
impl{generics} PyNode for ast::{rust_name}{generics} {{
#[inline]
fn py_type_cache() -> &'static OnceCell<(Py<PyAny>, Py<PyAny>)> {{
static PY_TYPE: OnceCell<(Py<PyAny>, Py<PyAny>)> = OnceCell::new();
@ -1876,7 +1874,7 @@ def write_to_pyo3(mod, type_info, f):
for info in type_info.values():
rust_name = info.rust_sum_name
f.write(f"cache_py_type::<crate::generic::{rust_name}>(ast_module)?;\n")
f.write(f"cache_py_type::<ast::{rust_name}>(ast_module)?;\n")
f.write("Ok(())\n}")
@ -1890,15 +1888,15 @@ def write_to_pyo3_simple(type_info, f):
rust_name = type_info.rust_sum_name
f.write(
f"""
impl ToPyo3Ast for crate::generic::{rust_name} {{
impl ToPyAst for ast::{rust_name} {{
#[inline]
fn to_pyo3_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {{
fn to_py_ast<'py>(&self, py: Python<'py>) -> PyResult<&'py PyAny> {{
let cell = match &self {{
""",
)
for cons in type_info.type.value.types:
f.write(
f"""crate::{rust_name}::{cons.name} => crate::{rust_name}{cons.name}::py_type_cache(),""",
f"""ast::{rust_name}::{cons.name} => ast::{rust_name}{cons.name}::py_type_cache(),""",
)
f.write(
"""
@ -1921,9 +1919,9 @@ def write_pyo3_wrapper(mod, type_info, namespace, f):
rust_name = type_info.rust_sum_name
f.write(
f"""
impl ToPyo3Wrapper for crate::generic::{rust_name} {{
impl ToPyWrapper for ast::{rust_name} {{
#[inline]
fn to_pyo3_wrapper(&self, py: Python) -> PyResult<Py<PyAny>> {{
fn to_py_wrapper(&self, py: Python) -> PyResult<Py<PyAny>> {{
match &self {{
""",
)
@ -1942,9 +1940,9 @@ def write_pyo3_wrapper(mod, type_info, namespace, f):
for cons in type_info.type.value.types:
f.write(
f"""
impl ToPyo3Wrapper for crate::generic::{rust_name}{cons.name} {{
impl ToPyWrapper for ast::{rust_name}{cons.name} {{
#[inline]
fn to_pyo3_wrapper(&self, py: Python) -> PyResult<Py<PyAny>> {{
fn to_py_wrapper(&self, py: Python) -> PyResult<Py<PyAny>> {{
Ok({rust_name}{cons.name}.to_object(py))
}}
}}
@ -1983,6 +1981,7 @@ def write_ast_mod(mod, type_info, f):
def main(
input_filename,
ast_dir,
ast_pyo3_dir,
module_filename,
dump_module=False,
):
@ -2005,14 +2004,20 @@ def main(
("ranged", p(write_ranged_def, mod, type_info)),
("located", p(write_located_def, mod, type_info)),
("visitor", p(write_visitor_def, mod, type_info)),
("to_pyo3", p(write_to_pyo3, mod, type_info)),
("pyo3_wrapper_located", p(write_pyo3_wrapper, mod, type_info, "located")),
("pyo3_wrapper_ranged", p(write_pyo3_wrapper, mod, type_info, "ranged")),
]:
with (ast_dir / f"{filename}.rs").open("w") as f:
f.write(auto_gen_msg)
write(f)
for filename, write in [
("to_py_ast", p(write_to_pyo3, mod, type_info)),
("wrapper_located", p(write_pyo3_wrapper, mod, type_info, "located")),
("wrapper_ranged", p(write_pyo3_wrapper, mod, type_info, "ranged")),
]:
with (ast_pyo3_dir / f"{filename}.rs").open("w") as f:
f.write(auto_gen_msg)
write(f)
with module_filename.open("w") as module_file:
module_file.write(auto_gen_msg)
write_ast_mod(mod, type_info, module_file)
@ -2024,6 +2029,7 @@ if __name__ == "__main__":
parser = ArgumentParser()
parser.add_argument("input_file", type=Path)
parser.add_argument("-A", "--ast-dir", type=Path, required=True)
parser.add_argument("-O", "--ast-pyo3-dir", type=Path, required=True)
parser.add_argument("-M", "--module-file", type=Path, required=True)
parser.add_argument("-d", "--dump-module", action="store_true")
@ -2031,6 +2037,7 @@ if __name__ == "__main__":
main(
args.input_file,
args.ast_dir,
args.ast_pyo3_dir,
args.module_file,
args.dump_module,
)

5
ast/src/lib.rs
View file

@ -41,8 +41,3 @@ pub use visitor::Visitor;
mod optimizer;
#[cfg(feature = "constant-optimization")]
pub use optimizer::ConstantOptimizer;
#[cfg(feature = "pyo3")]
pub mod pyo3;
#[cfg(feature = "pyo3-wrapper")]
pub mod pyo3_wrapper;

128
ast/src/pyo3_wrapper.rs
View file

@ -1,128 +0,0 @@
use crate::pyo3::{Pyo3Node, AST};
use crate::{source_code::SourceRange, text_size::TextRange, ConversionFlag, Node};
use num_complex::Complex64;
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyList, PyTuple};
pub trait ToPyo3Wrapper {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>>;
}
impl<T: ToPyo3Wrapper> ToPyo3Wrapper for Box<T> {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
(**self).to_pyo3_wrapper(py)
}
}
impl<T: ToPyo3Wrapper> ToPyo3Wrapper for Option<T> {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
match self {
Some(ast) => ast.to_pyo3_wrapper(py),
None => Ok(py.None()),
}
}
}
impl ToPyo3Wrapper for crate::Identifier {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyo3Wrapper for crate::String {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyo3Wrapper for crate::Int {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((self.to_u32()).to_object(py))
}
}
impl ToPyo3Wrapper for bool {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as u32).to_object(py))
}
}
impl ToPyo3Wrapper for ConversionFlag {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as i8).to_object(py))
}
}
impl ToPyo3Wrapper for crate::Constant {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let value = match self {
crate::Constant::None => py.None(),
crate::Constant::Bool(bool) => bool.to_object(py),
crate::Constant::Str(string) => string.to_object(py),
crate::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
crate::Constant::Int(int) => int.to_object(py),
crate::Constant::Tuple(elts) => {
let elts: PyResult<Vec<_>> = elts.iter().map(|c| c.to_pyo3_wrapper(py)).collect();
PyTuple::new(py, elts?).into()
}
crate::Constant::Float(f64) => f64.to_object(py),
crate::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
crate::Constant::Ellipsis => py.Ellipsis(),
};
Ok(value)
}
}
impl<T: ToPyo3Wrapper> ToPyo3Wrapper for Vec<T> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
pub mod located {
use super::*;
pub use crate::pyo3::AST;
include!("gen/pyo3_wrapper_located.rs");
}
pub mod ranged {
use super::*;
pub use crate::pyo3::AST;
include!("gen/pyo3_wrapper_ranged.rs");
}
fn init_type<P: pyo3::PyClass, N: Pyo3Node + Node>(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<P>()?;
let node = m.getattr(P::NAME)?;
if P::NAME != N::NAME {
// TODO: no idea how to escape rust keyword on #[pyclass]
m.setattr(P::NAME, node)?;
}
let names: Vec<&'static str> = N::FIELD_NAMES.to_vec();
let fields = PyTuple::new(py, names);
node.setattr("_fields", fields)?;
Ok(())
}
/// A Python module implemented in Rust.
fn init_module(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<AST>()?;
let ast = m.getattr("AST")?;
let fields = PyTuple::empty(py);
ast.setattr("_fields", fields)?;
Ok(())
}

4
scripts/update_asdl.sh
View file

@ -4,5 +4,5 @@ set -e
cd "$(dirname "$(dirname "$0")")"
# rm ast/src/gen/*.rs
python ast/asdl_rs.py --ast-dir ast/src/gen/ --module-file ../RustPython/vm/src/stdlib/ast/gen.rs ast/Python.asdl
rustfmt ast/src/gen/*.rs ../RustPython/vm/src/stdlib/ast/gen.rs
python ast/asdl_rs.py --ast-dir ast/src/gen/ --ast-pyo3-dir ast-pyo3/src/gen/ --module-file ../RustPython/vm/src/stdlib/ast/gen.rs ast/Python.asdl
rustfmt ast/src/gen/*.rs ast-pyo3/src/gen/*.rs ../RustPython/vm/src/stdlib/ast/gen.rs