ast::pyo3

This commit is contained in:
Jeong YunWon 2023-05-15 18:01:50 +09:00
parent 708d8061d3
commit 1a07454dc7
9 changed files with 9845 additions and 138 deletions

View file

@ -19,6 +19,8 @@ members = [
rustpython-ast = { path = "ast", default-features = false }
rustpython-parser-core = { path = "core", features = [] }
rustpython-literal = { path = "literal" }
rustpython-format = { path = "format" }
rustpython-parser = { path = "parser" }
ahash = "0.7.6"
anyhow = "1.0.45"
@ -30,6 +32,7 @@ log = "0.4.16"
num-complex = "0.4.0"
num-bigint = "0.4.3"
num-traits = "0.2"
pyo3 = { version = "0.18.3" }
rand = "0.8.5"
serde = "1.0"
static_assertions = "1.1"

View file

@ -23,3 +23,6 @@ rustpython-literal = { workspace = true, optional = true }
is-macro = { workspace = true }
num-bigint = { workspace = true }
static_assertions = "1.1.0"
num-complex = { workspace = true }
once_cell = { workspace = true }
pyo3 = { workspace = true, optional = true, features = ["num-bigint", "num-complex"] }

View file

@ -711,6 +711,583 @@ class VisitorModuleVisitor(EmitVisitor):
VisitorTraitDefVisitor(self.file, self.type_info).visit(mod, depth)
class ToPyo3AstVisitor(EmitVisitor):
"""Visitor to generate type-defs for AST."""
def __init__(self, namespace, *args, **kw):
super().__init__(*args, **kw)
self.namespace = namespace
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type, depth=0):
self.visit(type.value, type.name, depth)
def visitProduct(self, product, name, depth=0):
rust_name = rust_type_name(name)
self.emit(
f"""
// product
impl ToPyo3Ast for crate::{self.namespace}::{rust_name} {{
fn to_pyo3_ast(&self, _py: Python) -> PyResult<Py<PyAny>> {{
let class = ranged::{rust_name}::py_type_cell().get().unwrap();
let instance = class.call1(_py, (
""",
0,
)
for field in product.fields:
self.emit(f"""self.{field.name}.to_pyo3_ast(_py)?,""", depth + 1)
self.emit(
"""
))?;
Ok(instance.into())
}
}
""",
0,
)
def visitSum(self, sum, name, depth=0):
rust_name = rust_type_name(name)
self.emit(
f"""
impl ToPyo3Ast for crate::{self.namespace}::{rust_name} {{
fn to_pyo3_ast(&self, _py: Python) -> PyResult<Py<PyAny>> {{
let instance = match &self {{
""",
0,
)
for cons in sum.types:
if not is_simple(sum):
self.emit(
f"""crate::{rust_name}::{cons.name}(cons) => cons.to_pyo3_ast(_py)?,""",
depth,
)
else:
self.emit(
f"""crate::{rust_name}::{cons.name} => ranged::{rust_name}{cons.name}::py_type_cell().get().unwrap().clone(),""",
depth,
)
self.emit(
"""
};
Ok(instance)
}
}
""",
0,
)
if is_simple(sum):
return
for cons in sum.types:
self.visit(cons, rust_name, depth)
def visitConstructor(self, cons, parent, depth):
self.emit(
f"""
// constructor
impl ToPyo3Ast for crate::{self.namespace}::{parent}{cons.name} {{
fn to_pyo3_ast(&self, _py: Python) -> PyResult<Py<PyAny>> {{
let class = ranged::{parent}{cons.name}::py_type_cell().get().unwrap();
let instance = class.call1(_py, (
""",
depth,
)
for field in cons.fields:
self.emit(
f"self.{rust_field(field.name)}.to_pyo3_ast(_py)?,",
depth + 1,
)
self.emit(
"""
))?;
Ok(instance.into())
}
}
""",
depth,
)
class RangedDefVisitor(EmitVisitor):
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type, depth=0):
self.visit(type.value, type.name, depth)
def visitSum(self, sum, name, depth):
info = self.type_info[name]
self.emit_type_alias(info)
if info.is_simple:
return
sum_match_arms = ""
for ty in sum.types:
variant_info = self.type_info[ty.name]
sum_match_arms += (
f" Self::{variant_info.rust_name}(node) => node.range(),"
)
self.emit_type_alias(variant_info)
self.emit_ranged_impl(variant_info)
if not info.no_cfg(self.type_info):
self.emit('#[cfg(feature = "all-nodes-with-ranges")]', 0)
self.emit(
f"""
impl Ranged for crate::{info.rust_sum_name} {{
fn range(&self) -> TextRange {{
match self {{
{sum_match_arms}
}}
}}
}}
""".lstrip(),
0,
)
def visitProduct(self, product, name, depth):
info = self.type_info[name]
self.emit_type_alias(info)
self.emit_ranged_impl(info)
def emit_type_alias(self, info):
generics = "" if info.is_simple else "::<TextRange>"
self.emit(
f"pub type {info.rust_sum_name} = crate::generic::{info.rust_sum_name}{generics};",
0,
)
self.emit("", 0)
def emit_ranged_impl(self, info):
if not info.no_cfg(self.type_info):
self.emit('#[cfg(feature = "all-nodes-with-ranges")]', 0)
self.file.write(
f"""
impl Ranged for crate::generic::{info.rust_sum_name}::<TextRange> {{
fn range(&self) -> TextRange {{
self.range
}}
}}
""".strip()
)
class LocatedDefVisitor(EmitVisitor):
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type, depth=0):
self.visit(type.value, type.name, depth)
def visitSum(self, sum, name, depth):
info = self.type_info[name]
self.emit_type_alias(info)
if info.is_simple:
return
sum_match_arms = ""
for ty in sum.types:
variant_info = self.type_info[ty.name]
sum_match_arms += (
f" Self::{variant_info.rust_name}(node) => node.range(),"
)
self.emit_type_alias(variant_info)
self.emit_located_impl(variant_info)
if not info.no_cfg(self.type_info):
self.emit('#[cfg(feature = "all-nodes-with-ranges")]', 0)
self.emit(
f"""
impl Located for {info.rust_sum_name} {{
fn range(&self) -> SourceRange {{
match self {{
{sum_match_arms}
}}
}}
}}
""".lstrip(),
0,
)
def visitProduct(self, product, name, depth):
info = self.type_info[name]
self.emit_type_alias(info)
self.emit_located_impl(info)
def emit_type_alias(self, info):
generics = "" if info.is_simple else "::<SourceRange>"
self.emit(
f"pub type {info.rust_sum_name} = crate::generic::{info.rust_sum_name}{generics};",
0,
)
self.emit("", 0)
def emit_located_impl(self, info):
if not info.no_cfg(self.type_info):
self.emit('#[cfg(feature = "all-nodes-with-ranges")]', 0)
self.emit(
f"""
impl Located for {info.rust_sum_name} {{
fn range(&self) -> SourceRange {{
self.range
}}
}}
""",
0,
)
class Pyo3StructVisitor(EmitVisitor):
"""Visitor to generate type-defs for AST."""
def __init__(self, namespace, *args, borrow=False, **kw):
self.namespace = namespace
self.borrow = borrow
super().__init__(*args, **kw)
@property
def module_name(self):
name = f"rustpython_ast.{self.namespace}"
return name
@property
def ref_def(self):
return "&'static " if self.borrow else ""
@property
def ref(self):
return "&" if self.borrow else ""
def emit_class(self, name, rust_name, subclass, base="super::AST"):
if subclass:
subclass = ", subclass"
body = ""
into = f"{rust_name}"
else:
subclass = ""
body = f"(pub {self.ref_def} crate::{self.namespace}::{rust_name})"
into = f"{rust_name}(node)"
self.emit(
textwrap.dedent(
f"""
#[pyclass(module="{self.module_name}", name="_{name}", extends={base}{subclass})]
#[derive(Clone, Debug)]
pub struct {rust_name} {body};
impl {rust_name} {{
#[inline]
pub fn py_type_cell() -> &'static OnceCell<Py<PyAny>> {{
static PY_TYPE: OnceCell<Py<PyAny>> = OnceCell::new();
&PY_TYPE
}}
}}
impl From<{self.ref_def} crate::{self.namespace}::{rust_name}> for {rust_name} {{
fn from({"" if body else "_"}node: {self.ref_def} crate::{self.namespace}::{rust_name}) -> Self {{
{into}
}}
}}
"""
),
0,
)
if subclass:
self.emit(
textwrap.dedent(
f"""
#[pymethods]
impl {rust_name} {{
#[new]
fn new() -> PyClassInitializer<Self> {{
PyClassInitializer::from(AST)
.add_subclass(Self)
}}
}}
impl ToPyObject for {rust_name} {{
fn to_object(&self, py: Python) -> PyObject {{
let initializer = PyClassInitializer::from(AST)
.add_subclass(self.clone());
Py::new(py, initializer).unwrap().into_py(py)
}}
}}
"""
),
0,
)
else:
if base != "super::AST":
add_subclass = f".add_subclass({base})"
else:
add_subclass = ""
self.emit(
textwrap.dedent(
f"""
impl ToPyObject for {rust_name} {{
fn to_object(&self, py: Python) -> PyObject {{
let initializer = PyClassInitializer::from(AST)
{add_subclass}
.add_subclass(self.clone());
Py::new(py, initializer).unwrap().into_py(py)
}}
}}
"""
),
0,
)
if self.borrow and not subclass:
self.emit_wrapper(rust_name)
def emit_getter(self, owner, type_name):
self.emit(
textwrap.dedent(
f"""
#[pymethods]
impl {type_name} {{
"""
),
0,
)
for field in owner.fields:
self.emit(
textwrap.dedent(
f"""
#[getter]
#[inline]
fn get_{field.name}(&self, py: Python) -> PyResult<PyObject> {{
self.0.{rust_field(field.name)}.to_pyo3_wrapper(py)
}}
"""
),
3,
)
self.emit(
textwrap.dedent(
"""
}
"""
),
0,
)
def emit_getattr(self, owner, type_name):
self.emit(
textwrap.dedent(
f"""
#[pymethods]
impl {type_name} {{
fn __getattr__(&self, py: Python, key: &str) -> PyResult<PyObject> {{
let object: Py<PyAny> = match key {{
"""
),
0,
)
for field in owner.fields:
self.emit(
f'"{field.name}" => self.0.{rust_field(field.name)}.to_pyo3_wrapper(py)?,',
3,
)
self.emit(
textwrap.dedent(
"""
_ => todo!(),
};
Ok(object)
}
}
"""
),
0,
)
def emit_wrapper(self, rust_name):
self.emit(
f"""
impl ToPyo3Wrapper for crate::{self.namespace}::{rust_name} {{
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
Ok({rust_name}(self).to_object(py))
}}
}}
""",
0,
)
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type, depth=0):
self.visit(type.value, type.name, depth)
def visitSum(self, sum, name, depth=0):
rust_name = rust_type_name(name)
self.emit_class(name, rust_name, True)
simple = is_simple(sum)
if self.borrow:
self.emit(
f"""
impl ToPyo3Wrapper for crate::{self.namespace}::{rust_name} {{
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {{
match &self {{
""",
0,
)
for cons in sum.types:
if simple:
self.emit(
f"Self::{cons.name} => Ok({rust_name}{cons.name}.to_object(py)),",
3,
)
else:
self.emit(
f"Self::{cons.name}(cons) => cons.to_pyo3_wrapper(py),", 3
)
self.emit(
f"""
}}
}}
}}
""",
0,
)
for cons in sum.types:
self.visit(cons, rust_name, simple, depth + 1)
def visitProduct(self, product, name, depth=0):
rust_name = rust_type_name(name)
self.emit_class(name, rust_name, False)
if self.borrow:
self.emit_getter(product, rust_name)
def visitConstructor(self, cons, parent, simple, depth):
if simple:
self.emit(
f"""
#[pyclass(module="{self.module_name}", name="_{cons.name}", extends={parent})]
pub struct {parent}{cons.name};
impl {parent}{cons.name} {{
#[inline]
pub fn py_type_cell() -> &'static OnceCell<Py<PyAny>> {{
static PY_TYPE: OnceCell<Py<PyAny>> = OnceCell::new();
&PY_TYPE
}}
}}
impl ToPyObject for {parent}{cons.name} {{
fn to_object(&self, py: Python) -> PyObject {{
let initializer = PyClassInitializer::from(AST)
.add_subclass({parent})
.add_subclass(Self);
Py::new(py, initializer).unwrap().into_py(py)
}}
}}
""",
depth,
)
else:
self.emit_class(
cons.name, f"{parent}{cons.name}", subclass=False, base=parent
)
if self.borrow:
self.emit_getter(cons, f"{parent}{cons.name}")
class Pyo3PymoduleVisitor(EmitVisitor):
def __init__(self, namespace, *args, **kw):
self.namespace = namespace
super().__init__(*args, **kw)
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type, depth=0):
self.visit(type.value, type.name, depth)
def visitProduct(self, product, name, depth=0):
rust_name = rust_type_name(name)
self.emit_fields(name, rust_name, False, depth)
def visitSum(self, sum, name, depth):
rust_name = rust_type_name(name)
simple = is_simple(sum)
self.emit_fields(name, rust_name, True, depth)
for cons in sum.types:
self.visit(cons, name, simple, depth)
def visitConstructor(self, cons, parent, simple, depth):
rust_name = rust_type_name(parent) + rust_type_name(cons.name)
self.emit_fields(cons.name, rust_name, simple, depth)
def emit_fields(self, name, rust_name, simple, depth):
if simple:
call = ".call0().unwrap()"
else:
call = ""
self.emit(
f"""
{rust_name}::py_type_cell().get_or_init(|| {{
ast_module.getattr("{name}").unwrap(){call}.into_py(py)
}});
""",
depth,
)
if simple:
return
self.emit(
f"""
{{
m.add_class::<{rust_name}>()?;
let node = m.getattr("_{name}")?;
m.setattr("{name}", node)?;
let names: Vec<&'static str> = crate::{self.namespace}::{rust_name}::FIELD_NAMES.to_vec();
let fields = PyTuple::new(py, names);
node.setattr("_fields", fields)?;
}}
""",
depth,
)
class StdlibClassDefVisitor(EmitVisitor):
def visitModule(self, mod):
for dfn in mod.dfns:
@ -1011,138 +1588,6 @@ class StdlibTraitImplVisitor(EmitVisitor):
return f"Node::ast_from_object(_vm, get_node_field(_vm, &_object, {name}, {json.dumps(typename)})?)?"
class RangedDefVisitor(EmitVisitor):
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type, depth=0):
self.visit(type.value, type.name, depth)
def visitSum(self, sum, name, depth):
info = self.type_info[name]
if info.is_simple:
return
sum_match_arms = ""
for ty in sum.types:
variant_info = self.type_info[ty.name]
sum_match_arms += (
f" Self::{variant_info.rust_name}(node) => node.range(),"
)
self.emit_ranged_impl(variant_info)
if not info.no_cfg(self.type_info):
self.emit('#[cfg(feature = "all-nodes-with-ranges")]', 0)
self.emit(
f"""
impl Ranged for crate::{info.rust_sum_name} {{
fn range(&self) -> TextRange {{
match self {{
{sum_match_arms}
}}
}}
}}
""".lstrip(),
0,
)
def visitProduct(self, product, name, depth):
info = self.type_info[name]
self.emit_ranged_impl(info)
def emit_ranged_impl(self, info):
if not info.no_cfg(self.type_info):
self.emit('#[cfg(feature = "all-nodes-with-ranges")]', 0)
self.file.write(
f"""
impl Ranged for crate::generic::{info.rust_sum_name}::<TextRange> {{
fn range(&self) -> TextRange {{
self.range
}}
}}
""".strip()
)
class LocatedDefVisitor(EmitVisitor):
def visitModule(self, mod):
for dfn in mod.dfns:
self.visit(dfn)
def visitType(self, type, depth=0):
self.visit(type.value, type.name, depth)
def visitSum(self, sum, name, depth):
info = self.type_info[name]
self.emit_type_alias(info)
if info.is_simple:
return
sum_match_arms = ""
for ty in sum.types:
variant_info = self.type_info[ty.name]
sum_match_arms += (
f" Self::{variant_info.rust_name}(node) => node.range(),"
)
self.emit_type_alias(variant_info)
self.emit_located_impl(variant_info)
if not info.no_cfg(self.type_info):
self.emit('#[cfg(feature = "all-nodes-with-ranges")]', 0)
self.emit(
f"""
impl Located for {info.rust_sum_name} {{
fn range(&self) -> SourceRange {{
match self {{
{sum_match_arms}
}}
}}
}}
""".lstrip(),
0,
)
def visitProduct(self, product, name, depth):
info = self.type_info[name]
self.emit_type_alias(info)
self.emit_located_impl(info)
def emit_type_alias(self, info):
generics = "" if info.is_simple else "::<SourceRange>"
self.emit(
f"pub type {info.rust_sum_name} = crate::generic::{info.rust_sum_name}{generics};",
0,
)
self.emit("", 0)
def emit_located_impl(self, info):
if not info.no_cfg(self.type_info):
self.emit('#[cfg(feature = "all-nodes-with-ranges")]', 0)
self.emit(
f"""
impl Located for {info.rust_sum_name} {{
fn range(&self) -> SourceRange {{
self.range
}}
}}
""",
0,
)
class ChainOfVisitors:
def __init__(self, *visitors):
self.visitors = visitors
@ -1174,6 +1619,27 @@ def write_located_def(mod, type_info, f):
LocatedDefVisitor(f, type_info).visit(mod)
def write_ast_pyo3(mod, type_info, namespace, f):
ToPyo3AstVisitor(namespace, f, type_info).visit(mod)
def write_pyo3_def(mod, type_info, namespace, borrow, f):
Pyo3StructVisitor(namespace, f, type_info, borrow=borrow).visit(mod)
f.write(
"""
use once_cell::sync::OnceCell;
pub fn add_to_module(py: Python, m: &PyModule) -> PyResult<()> {
super::init_module(py, m)?;
let ast_module = PyModule::import(py, "_ast")?;
"""
)
Pyo3PymoduleVisitor(namespace, f, type_info).visit(mod)
f.write("Ok(())\n}")
def write_ast_mod(mod, type_info, f):
f.write(
textwrap.dedent(
@ -1211,16 +1677,29 @@ def main(
type_info = {}
FindUserDataTypesVisitor(type_info).visit(mod)
from functools import partial as p
for filename, write in [
("generic", write_ast_def),
("fold", write_fold_def),
("ranged", write_ranged_def),
("located", write_located_def),
("visitor", write_visitor_def),
("generic", p(write_ast_def, mod, type_info)),
("fold", p(write_fold_def, mod, type_info)),
("ranged", p(write_ranged_def, mod, type_info)),
("located", p(write_located_def, mod, type_info)),
("visitor", p(write_visitor_def, mod, type_info)),
("to_pyo3_located", p(write_ast_pyo3, mod, type_info, "located")),
("to_pyo3_ranged", p(write_ast_pyo3, mod, type_info, "ranged")),
# ("pyo3_located", p(write_pyo3_def, mod, type_info, "located", True)),
("pyo3_ranged", p(write_pyo3_def, mod, type_info, "ranged", True)),
]:
with (ast_dir / f"{filename}.rs").open("w") as f:
f.write(auto_gen_msg)
write(mod, type_info, f)
write(f)
# for filename, write in [
# ]:
# with (pyo3_dir / f"{filename}.rs").open("w") as f:
# f.write(auto_gen_msg)
# write(mod, type_info, f)
with module_filename.open("w") as module_file:
module_file.write(auto_gen_msg)

6267
ast/src/gen/pyo3_ranged.rs Normal file

File diff suppressed because it is too large Load diff

View file

@ -1,23 +1,33 @@
// File automatically generated by ast/asdl_rs.py.
pub type Mod = crate::generic::Mod<TextRange>;
pub type ModModule = crate::generic::ModModule<TextRange>;
#[cfg(feature = "all-nodes-with-ranges")]
impl Ranged for crate::generic::ModModule<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ModInteractive = crate::generic::ModInteractive<TextRange>;
#[cfg(feature = "all-nodes-with-ranges")]
impl Ranged for crate::generic::ModInteractive<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ModExpression = crate::generic::ModExpression<TextRange>;
#[cfg(feature = "all-nodes-with-ranges")]
impl Ranged for crate::generic::ModExpression<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ModFunctionType = crate::generic::ModFunctionType<TextRange>;
#[cfg(feature = "all-nodes-with-ranges")]
impl Ranged for crate::generic::ModFunctionType<TextRange> {
fn range(&self) -> TextRange {
@ -36,136 +46,192 @@ impl Ranged for crate::Mod {
}
}
pub type Stmt = crate::generic::Stmt<TextRange>;
pub type StmtFunctionDef = crate::generic::StmtFunctionDef<TextRange>;
impl Ranged for crate::generic::StmtFunctionDef<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtAsyncFunctionDef = crate::generic::StmtAsyncFunctionDef<TextRange>;
impl Ranged for crate::generic::StmtAsyncFunctionDef<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtClassDef = crate::generic::StmtClassDef<TextRange>;
impl Ranged for crate::generic::StmtClassDef<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtReturn = crate::generic::StmtReturn<TextRange>;
impl Ranged for crate::generic::StmtReturn<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtDelete = crate::generic::StmtDelete<TextRange>;
impl Ranged for crate::generic::StmtDelete<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtAssign = crate::generic::StmtAssign<TextRange>;
impl Ranged for crate::generic::StmtAssign<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtAugAssign = crate::generic::StmtAugAssign<TextRange>;
impl Ranged for crate::generic::StmtAugAssign<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtAnnAssign = crate::generic::StmtAnnAssign<TextRange>;
impl Ranged for crate::generic::StmtAnnAssign<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtFor = crate::generic::StmtFor<TextRange>;
impl Ranged for crate::generic::StmtFor<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtAsyncFor = crate::generic::StmtAsyncFor<TextRange>;
impl Ranged for crate::generic::StmtAsyncFor<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtWhile = crate::generic::StmtWhile<TextRange>;
impl Ranged for crate::generic::StmtWhile<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtIf = crate::generic::StmtIf<TextRange>;
impl Ranged for crate::generic::StmtIf<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtWith = crate::generic::StmtWith<TextRange>;
impl Ranged for crate::generic::StmtWith<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtAsyncWith = crate::generic::StmtAsyncWith<TextRange>;
impl Ranged for crate::generic::StmtAsyncWith<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtMatch = crate::generic::StmtMatch<TextRange>;
impl Ranged for crate::generic::StmtMatch<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtRaise = crate::generic::StmtRaise<TextRange>;
impl Ranged for crate::generic::StmtRaise<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtTry = crate::generic::StmtTry<TextRange>;
impl Ranged for crate::generic::StmtTry<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtTryStar = crate::generic::StmtTryStar<TextRange>;
impl Ranged for crate::generic::StmtTryStar<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtAssert = crate::generic::StmtAssert<TextRange>;
impl Ranged for crate::generic::StmtAssert<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtImport = crate::generic::StmtImport<TextRange>;
impl Ranged for crate::generic::StmtImport<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtImportFrom = crate::generic::StmtImportFrom<TextRange>;
impl Ranged for crate::generic::StmtImportFrom<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtGlobal = crate::generic::StmtGlobal<TextRange>;
impl Ranged for crate::generic::StmtGlobal<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtNonlocal = crate::generic::StmtNonlocal<TextRange>;
impl Ranged for crate::generic::StmtNonlocal<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtExpr = crate::generic::StmtExpr<TextRange>;
impl Ranged for crate::generic::StmtExpr<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtPass = crate::generic::StmtPass<TextRange>;
impl Ranged for crate::generic::StmtPass<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtBreak = crate::generic::StmtBreak<TextRange>;
impl Ranged for crate::generic::StmtBreak<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type StmtContinue = crate::generic::StmtContinue<TextRange>;
impl Ranged for crate::generic::StmtContinue<TextRange> {
fn range(&self) -> TextRange {
self.range
@ -205,136 +271,192 @@ impl Ranged for crate::Stmt {
}
}
pub type Expr = crate::generic::Expr<TextRange>;
pub type ExprBoolOp = crate::generic::ExprBoolOp<TextRange>;
impl Ranged for crate::generic::ExprBoolOp<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprNamedExpr = crate::generic::ExprNamedExpr<TextRange>;
impl Ranged for crate::generic::ExprNamedExpr<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprBinOp = crate::generic::ExprBinOp<TextRange>;
impl Ranged for crate::generic::ExprBinOp<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprUnaryOp = crate::generic::ExprUnaryOp<TextRange>;
impl Ranged for crate::generic::ExprUnaryOp<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprLambda = crate::generic::ExprLambda<TextRange>;
impl Ranged for crate::generic::ExprLambda<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprIfExp = crate::generic::ExprIfExp<TextRange>;
impl Ranged for crate::generic::ExprIfExp<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprDict = crate::generic::ExprDict<TextRange>;
impl Ranged for crate::generic::ExprDict<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprSet = crate::generic::ExprSet<TextRange>;
impl Ranged for crate::generic::ExprSet<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprListComp = crate::generic::ExprListComp<TextRange>;
impl Ranged for crate::generic::ExprListComp<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprSetComp = crate::generic::ExprSetComp<TextRange>;
impl Ranged for crate::generic::ExprSetComp<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprDictComp = crate::generic::ExprDictComp<TextRange>;
impl Ranged for crate::generic::ExprDictComp<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprGeneratorExp = crate::generic::ExprGeneratorExp<TextRange>;
impl Ranged for crate::generic::ExprGeneratorExp<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprAwait = crate::generic::ExprAwait<TextRange>;
impl Ranged for crate::generic::ExprAwait<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprYield = crate::generic::ExprYield<TextRange>;
impl Ranged for crate::generic::ExprYield<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprYieldFrom = crate::generic::ExprYieldFrom<TextRange>;
impl Ranged for crate::generic::ExprYieldFrom<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprCompare = crate::generic::ExprCompare<TextRange>;
impl Ranged for crate::generic::ExprCompare<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprCall = crate::generic::ExprCall<TextRange>;
impl Ranged for crate::generic::ExprCall<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprFormattedValue = crate::generic::ExprFormattedValue<TextRange>;
impl Ranged for crate::generic::ExprFormattedValue<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprJoinedStr = crate::generic::ExprJoinedStr<TextRange>;
impl Ranged for crate::generic::ExprJoinedStr<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprConstant = crate::generic::ExprConstant<TextRange>;
impl Ranged for crate::generic::ExprConstant<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprAttribute = crate::generic::ExprAttribute<TextRange>;
impl Ranged for crate::generic::ExprAttribute<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprSubscript = crate::generic::ExprSubscript<TextRange>;
impl Ranged for crate::generic::ExprSubscript<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprStarred = crate::generic::ExprStarred<TextRange>;
impl Ranged for crate::generic::ExprStarred<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprName = crate::generic::ExprName<TextRange>;
impl Ranged for crate::generic::ExprName<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprList = crate::generic::ExprList<TextRange>;
impl Ranged for crate::generic::ExprList<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprTuple = crate::generic::ExprTuple<TextRange>;
impl Ranged for crate::generic::ExprTuple<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type ExprSlice = crate::generic::ExprSlice<TextRange>;
impl Ranged for crate::generic::ExprSlice<TextRange> {
fn range(&self) -> TextRange {
self.range
@ -374,12 +496,28 @@ impl Ranged for crate::Expr {
}
}
pub type ExprContext = crate::generic::ExprContext;
pub type Boolop = crate::generic::Boolop;
pub type Operator = crate::generic::Operator;
pub type Unaryop = crate::generic::Unaryop;
pub type Cmpop = crate::generic::Cmpop;
pub type Comprehension = crate::generic::Comprehension<TextRange>;
#[cfg(feature = "all-nodes-with-ranges")]
impl Ranged for crate::generic::Comprehension<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type Excepthandler = crate::generic::Excepthandler<TextRange>;
pub type ExcepthandlerExceptHandler = crate::generic::ExcepthandlerExceptHandler<TextRange>;
impl Ranged for crate::generic::ExcepthandlerExceptHandler<TextRange> {
fn range(&self) -> TextRange {
self.range
@ -393,74 +531,104 @@ impl Ranged for crate::Excepthandler {
}
}
pub type Arguments = crate::generic::Arguments<TextRange>;
#[cfg(feature = "all-nodes-with-ranges")]
impl Ranged for crate::generic::Arguments<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type Arg = crate::generic::Arg<TextRange>;
impl Ranged for crate::generic::Arg<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type Keyword = crate::generic::Keyword<TextRange>;
impl Ranged for crate::generic::Keyword<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type Alias = crate::generic::Alias<TextRange>;
impl Ranged for crate::generic::Alias<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type Withitem = crate::generic::Withitem<TextRange>;
#[cfg(feature = "all-nodes-with-ranges")]
impl Ranged for crate::generic::Withitem<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type MatchCase = crate::generic::MatchCase<TextRange>;
#[cfg(feature = "all-nodes-with-ranges")]
impl Ranged for crate::generic::MatchCase<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type Pattern = crate::generic::Pattern<TextRange>;
pub type PatternMatchValue = crate::generic::PatternMatchValue<TextRange>;
impl Ranged for crate::generic::PatternMatchValue<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type PatternMatchSingleton = crate::generic::PatternMatchSingleton<TextRange>;
impl Ranged for crate::generic::PatternMatchSingleton<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type PatternMatchSequence = crate::generic::PatternMatchSequence<TextRange>;
impl Ranged for crate::generic::PatternMatchSequence<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type PatternMatchMapping = crate::generic::PatternMatchMapping<TextRange>;
impl Ranged for crate::generic::PatternMatchMapping<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type PatternMatchClass = crate::generic::PatternMatchClass<TextRange>;
impl Ranged for crate::generic::PatternMatchClass<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type PatternMatchStar = crate::generic::PatternMatchStar<TextRange>;
impl Ranged for crate::generic::PatternMatchStar<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type PatternMatchAs = crate::generic::PatternMatchAs<TextRange>;
impl Ranged for crate::generic::PatternMatchAs<TextRange> {
fn range(&self) -> TextRange {
self.range
}
}
pub type PatternMatchOr = crate::generic::PatternMatchOr<TextRange>;
impl Ranged for crate::generic::PatternMatchOr<TextRange> {
fn range(&self) -> TextRange {
self.range
@ -481,6 +649,10 @@ impl Ranged for crate::Pattern {
}
}
pub type TypeIgnore = crate::generic::TypeIgnore<TextRange>;
pub type TypeIgnoreTypeIgnore = crate::generic::TypeIgnoreTypeIgnore<TextRange>;
#[cfg(feature = "all-nodes-with-ranges")]
impl Ranged for crate::generic::TypeIgnoreTypeIgnore<TextRange> {
fn range(&self) -> TextRange {

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -47,3 +47,6 @@ pub use visitor::Visitor;
mod optimizer;
#[cfg(feature = "constant-optimization")]
pub use optimizer::ConstantOptimizer;
#[cfg(feature = "pyo3")]
pub mod pyo3;

350
ast/src/pyo3.rs Normal file
View file

@ -0,0 +1,350 @@
use crate::Node;
use num_complex::Complex64;
use pyo3::prelude::*;
use pyo3::types::{PyBytes, PyList, PyTuple};
pub trait ToPyo3Ast {
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>>;
}
impl<T: ToPyo3Ast> ToPyo3Ast for Box<T> {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
(**self).to_pyo3_ast(py)
}
}
impl<T: ToPyo3Ast> ToPyo3Ast for Option<T> {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
match self {
Some(ast) => ast.to_pyo3_ast(py),
None => Ok(py.None()),
}
}
}
impl<T: ToPyo3Ast> ToPyo3Ast for Vec<T> {
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_ast(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Ast for crate::Identifier {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyo3Ast for crate::String {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyo3Ast for crate::Int {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
Ok((self.to_u32()).to_object(py))
}
}
impl ToPyo3Ast for bool {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as u32).to_object(py))
}
}
impl ToPyo3Ast for crate::Constant {
#[inline]
fn to_pyo3_ast(&self, py: Python) -> PyResult<Py<PyAny>> {
let value = match self {
crate::Constant::None => py.None(),
crate::Constant::Bool(bool) => bool.to_object(py),
crate::Constant::Str(string) => string.to_object(py),
crate::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
crate::Constant::Int(int) => int.to_object(py),
crate::Constant::Tuple(elts) => {
let elts: PyResult<Vec<_>> = elts.iter().map(|c| c.to_pyo3_ast(py)).collect();
PyTuple::new(py, elts?).into()
}
crate::Constant::Float(f64) => f64.to_object(py),
crate::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
crate::Constant::Ellipsis => py.Ellipsis(),
};
Ok(value)
}
}
// pub trait FromAst<A> {
// fn from_ast(py: Python, ast: A) -> PyResult<Py<PyAny>>;
// }
pub trait ToPyo3Wrapper {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>>;
}
impl<T: ToPyo3Wrapper> ToPyo3Wrapper for Box<T> {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
(**self).to_pyo3_wrapper(py)
}
}
impl<T: ToPyo3Wrapper> ToPyo3Wrapper for Option<T> {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
match self {
Some(ast) => ast.to_pyo3_wrapper(py),
None => Ok(py.None()),
}
}
}
// for Vec needs refactoring
impl ToPyo3Wrapper for Vec<crate::ranged::Stmt> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for Vec<crate::ranged::Expr> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for Vec<crate::ranged::Arg> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for Vec<crate::ranged::Pattern> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for Vec<crate::Identifier> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for Vec<crate::ranged::Keyword> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for Vec<crate::ranged::Cmpop> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for Vec<crate::ranged::Comprehension> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for Vec<Option<crate::ranged::Expr>> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for Vec<crate::ranged::Alias> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for Vec<crate::ranged::Excepthandler> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for Vec<crate::ranged::MatchCase> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for Vec<crate::ranged::Withitem> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for Vec<crate::ranged::TypeIgnore> {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let list = PyList::empty(py);
for item in self {
let py_item = item.to_pyo3_wrapper(py)?;
list.append(py_item)?;
}
Ok(list.into())
}
}
impl ToPyo3Wrapper for crate::Identifier {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyo3Wrapper for crate::String {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok(self.as_str().to_object(py))
}
}
impl ToPyo3Wrapper for crate::Int {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((self.to_u32()).to_object(py))
}
}
impl ToPyo3Wrapper for bool {
#[inline]
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
Ok((*self as u32).to_object(py))
}
}
impl ToPyo3Wrapper for crate::Constant {
fn to_pyo3_wrapper(&'static self, py: Python) -> PyResult<Py<PyAny>> {
let value = match self {
crate::Constant::None => py.None(),
crate::Constant::Bool(bool) => bool.to_object(py),
crate::Constant::Str(string) => string.to_object(py),
crate::Constant::Bytes(bytes) => PyBytes::new(py, bytes).into(),
crate::Constant::Int(int) => int.to_object(py),
crate::Constant::Tuple(elts) => {
let elts: PyResult<Vec<_>> = elts.iter().map(|c| c.to_pyo3_wrapper(py)).collect();
PyTuple::new(py, elts?).into()
}
crate::Constant::Float(f64) => f64.to_object(py),
crate::Constant::Complex { real, imag } => Complex64::new(*real, *imag).to_object(py),
crate::Constant::Ellipsis => py.Ellipsis(),
};
Ok(value)
}
}
include!("gen/to_pyo3_ranged.rs");
// include!("gen/to_pyo3_located.rs");
pub mod located {
pub use super::AST;
use super::*;
// include!("gen/pyo3_located.rs");
}
pub mod ranged {
pub use super::AST;
use super::*;
include!("gen/pyo3_ranged.rs");
}
#[pyclass(module = "rustpython_ast", subclass)]
pub struct AST;
#[pymethods]
impl AST {
#[new]
fn new() -> Self {
Self
}
}
/// A Python module implemented in Rust.
fn init_module(py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<AST>()?;
let ast = m.getattr("AST")?;
let fields = PyTuple::empty(py);
ast.setattr("_fields", fields)?;
Ok(())
}